From 1d2c6c2bd6a4f2e299edf55e36a5885c8d51e4c4 Mon Sep 17 00:00:00 2001 From: yl231 Date: Tue, 19 May 2026 21:26:26 -0500 Subject: [PATCH 1/4] Sync leaderboard with RouterArena README (Weave, Auto, R2 entries) - leaderboard.json: add Weave Router (74.61) and Auto Router (70.05) entries; fix NotDiamond Arena 63.0 -> 57.29 (per RouterArena PR #34). - category_scores.json: add per-domain breakdowns for r2-router, auto_router, and weave-router. Computed locally from RouterArena full-split predictions joined with dataset Domain/Category/Difficulty metadata; robustness from the per-router robustness predictions. - routers.json: add Weave Router and Auto Router metadata. Drop the orphan universal_router entry (had no corresponding leaderboard row). - routers.yaml: append R2-Router (was only in routers.json), Weave Router, and Auto Router. - flip_labels/: add per-query flip labels for r2-router, auto_router, weave-router, and vllm-sr. Computed as model-selection equality between full and robustness predictions, universal-name normalized. Sqwish Router (RouterArena PR #94) is excluded; will be added in a follow-up once that submission's CI re-run completes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../flip_labels/flip_labels_auto_router.json | 1682 ++++++++++ .../flip_labels/flip_labels_r2-router.json | 1682 ++++++++++ src/data/flip_labels/flip_labels_vllm-sr.json | 1682 ++++++++++ .../flip_labels/flip_labels_weave-router.json | 1682 ++++++++++ src/data/routerMetrics/category_scores.json | 2796 +++++++++++++++++ src/data/routerMetrics/leaderboard.json | 185 +- src/data/routers.json | 41 +- src/data/routers.yaml | 26 + 8 files changed, 9663 insertions(+), 113 deletions(-) create mode 100644 src/data/flip_labels/flip_labels_auto_router.json create mode 100644 src/data/flip_labels/flip_labels_r2-router.json create mode 100644 src/data/flip_labels/flip_labels_vllm-sr.json create mode 100644 src/data/flip_labels/flip_labels_weave-router.json diff --git a/src/data/flip_labels/flip_labels_auto_router.json b/src/data/flip_labels/flip_labels_auto_router.json new file mode 100644 index 0000000..368efff --- /dev/null +++ b/src/data/flip_labels/flip_labels_auto_router.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 1 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 1 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 1 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 1 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 1 + }, + { + "global index": "AsDiv_472", + "flip": 1 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 1 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 1 + }, + { + "global index": "Ethics_virtue_51", + "flip": 1 + }, + { + "global index": "FinQA_149", + "flip": 1 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 1 + }, + { + "global index": "FinQA_60", + "flip": 1 + }, + { + "global index": "GSM8K_43", + "flip": 1 + }, + { + "global index": "GeoBench_1002", + "flip": 1 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 1 + }, + { + "global index": "GeoBench_1113", + "flip": 1 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 1 + }, + { + "global index": "GeoBench_526", + "flip": 1 + }, + { + "global index": "GeoBench_591", + "flip": 1 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 1 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 1 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 1 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 1 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 1 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 1 + }, + { + "global index": "MATH_53", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 1 + }, + { + "global index": "MMLUPro_business_294", + "flip": 1 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 1 + }, + { + "global index": "MMLUPro_business_503", + "flip": 1 + }, + { + "global index": "MMLUPro_business_507", + "flip": 1 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 1 + }, + { + "global index": "MMLUPro_law_899", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 1 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 1 + }, + { + "global index": "MathQA_1742", + "flip": 1 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 1 + }, + { + "global index": "MathQA_2102", + "flip": 1 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 1 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 1 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 1 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 1 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 1 + }, + { + "global index": "MedMCQA_2366", + "flip": 1 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 1 + }, + { + "global index": "NarrativeQA_1683", + "flip": 1 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 1 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 1 + }, + { + "global index": "NarrativeQA_5022", + "flip": 1 + }, + { + "global index": "NarrativeQA_5259", + "flip": 1 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 1 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 1 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 1 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 1 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 1 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 1 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 1 + }, + { + "global index": "PubMedQA_362", + "flip": 1 + }, + { + "global index": "PubMedQA_437", + "flip": 1 + }, + { + "global index": "PubMedQA_510", + "flip": 1 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 1 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 1 + }, + { + "global index": "PubMedQA_722", + "flip": 1 + }, + { + "global index": "PubMedQA_73", + "flip": 1 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 1 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 1 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 1 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 1 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] \ No newline at end of file diff --git a/src/data/flip_labels/flip_labels_r2-router.json b/src/data/flip_labels/flip_labels_r2-router.json new file mode 100644 index 0000000..0056bc5 --- /dev/null +++ b/src/data/flip_labels/flip_labels_r2-router.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 1 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 1 + }, + { + "global index": "ArcMMLU_230", + "flip": 1 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 1 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 1 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 1 + }, + { + "global index": "ChessInstruct_144", + "flip": 1 + }, + { + "global index": "ChessInstruct_42", + "flip": 1 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 1 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 1 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 1 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 1 + }, + { + "global index": "Ethics_virtue_48", + "flip": 1 + }, + { + "global index": "Ethics_virtue_51", + "flip": 1 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 1 + }, + { + "global index": "GeoBench_1094", + "flip": 1 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 1 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 1 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 1 + }, + { + "global index": "GeoBench_944", + "flip": 1 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 1 + }, + { + "global index": "LiveCodeBench_105", + "flip": 1 + }, + { + "global index": "LiveCodeBench_114", + "flip": 1 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 1 + }, + { + "global index": "LiveCodeBench_181", + "flip": 1 + }, + { + "global index": "LiveCodeBench_237", + "flip": 1 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 1 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 1 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 1 + }, + { + "global index": "LiveCodeBench_485", + "flip": 1 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 1 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 1 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 1 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 1 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 1 + }, + { + "global index": "MathQA_2102", + "flip": 1 + }, + { + "global index": "MathQA_2851", + "flip": 1 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 1 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 1 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 1 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 1 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 1 + }, + { + "global index": "PubMedQA_0", + "flip": 1 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 1 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 1 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 1 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 1 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 1 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 1 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 1 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 1 + }, + { + "global index": "QANTA_History_473", + "flip": 1 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 1 + }, + { + "global index": "QANTA_Literature_386", + "flip": 1 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 1 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 1 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 1 + }, + { + "global index": "QANTA_Science_308", + "flip": 1 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 1 + }, + { + "global index": "SocialiQA_22095", + "flip": 1 + }, + { + "global index": "SocialiQA_26846", + "flip": 1 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 1 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 1 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 1 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 1 + } +] \ No newline at end of file diff --git a/src/data/flip_labels/flip_labels_vllm-sr.json b/src/data/flip_labels/flip_labels_vllm-sr.json new file mode 100644 index 0000000..fe6e3a5 --- /dev/null +++ b/src/data/flip_labels/flip_labels_vllm-sr.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 1 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 1 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 1 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 1 + }, + { + "global index": "NarrativeQA_1683", + "flip": 1 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 1 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 1 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 1 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 1 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] \ No newline at end of file diff --git a/src/data/flip_labels/flip_labels_weave-router.json b/src/data/flip_labels/flip_labels_weave-router.json new file mode 100644 index 0000000..ba790c1 --- /dev/null +++ b/src/data/flip_labels/flip_labels_weave-router.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 1 + }, + { + "global index": "ChessInstruct_107", + "flip": 1 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 1 + }, + { + "global index": "ChessInstruct_84", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 1 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 1 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 1 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 1 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 1 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 1 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 1 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 1 + }, + { + "global index": "PubMedQA_250", + "flip": 1 + }, + { + "global index": "PubMedQA_337", + "flip": 1 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 1 + }, + { + "global index": "PubMedQA_510", + "flip": 1 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 1 + }, + { + "global index": "PubMedQA_687", + "flip": 1 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 1 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 1 + }, + { + "global index": "PubMedQA_905", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 1 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 1 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 1 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 1 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 1 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] \ No newline at end of file diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index a0e1d63..1ccaaec 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -12018,5 +12018,2801 @@ } } } + }, + "r2-router": { + "metrics": { + "easy": { + "accuracy": 95.8, + "cost": 0.0001, + "robustness": 47.0 + }, + "medium": { + "accuracy": 67.3, + "cost": 0.0001, + "robustness": 50.4 + }, + "hard": { + "accuracy": 26.2, + "cost": 0.0001, + "robustness": 38.5 + }, + "all": { + "accuracy": 71.2, + "cost": 0.0001, + "robustness": 45.7 + } + }, + "categories": { + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 42.3 + }, + "medium": { + "accuracy": 80.7, + "cost": 0.0001, + "robustness": 42.9 + }, + "hard": { + "accuracy": 32.2, + "cost": 0.0003, + "robustness": 53.3 + }, + "all": { + "accuracy": 80.4, + "cost": 0.0001, + "robustness": 45.2 + } + }, + "subcategories": { + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0, + "robustness": 38.5 + }, + "medium": { + "accuracy": 62.4, + "cost": 0.0, + "robustness": 0.0 + }, + "hard": { + "accuracy": 2.8, + "cost": 0.0, + "robustness": 50.0 + }, + "all": { + "accuracy": 80.3, + "cost": 0.0, + "robustness": 31.6 + } + } + }, + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 46.2 + }, + "medium": { + "accuracy": 84.9, + "cost": 0.0001, + "robustness": 52.9 + }, + "hard": { + "accuracy": 37.3, + "cost": 0.0004, + "robustness": 53.8 + }, + "all": { + "accuracy": 80.5, + "cost": 0.0002, + "robustness": 51.2 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0, + "robustness": 22.2 + }, + "medium": { + "accuracy": 62.6, + "cost": 0.0001, + "robustness": 40.0 + }, + "hard": { + "accuracy": 13.6, + "cost": 0.0001, + "robustness": 42.9 + }, + "all": { + "accuracy": 74.0, + "cost": 0.0001, + "robustness": 29.5 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 85.9, + "cost": 0.0, + "robustness": 21.4 + }, + "medium": { + "accuracy": 47.1, + "cost": 0.0001, + "robustness": 20.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 70.3, + "cost": 0.0001, + "robustness": 28.6 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 52.5, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 13.3, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 50.8, + "cost": 0.0001, + "robustness": 12.5 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0, + "robustness": 33.3 + }, + "medium": { + "accuracy": 81.2, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": 50.0 + }, + "all": { + "accuracy": 85.0, + "cost": 0.0, + "robustness": 37.5 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0001, + "robustness": 25.0 + }, + "medium": { + "accuracy": 84.2, + "cost": 0.0001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0003, + "robustness": null + }, + "all": { + "accuracy": 90.7, + "cost": 0.0001, + "robustness": 42.9 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0, + "robustness": 55.6 + }, + "medium": { + "accuracy": 60.5, + "cost": 0.0001, + "robustness": 83.3 + }, + "hard": { + "accuracy": 8.7, + "cost": 0.0001, + "robustness": 28.6 + }, + "all": { + "accuracy": 64.3, + "cost": 0.0001, + "robustness": 59.5 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0, + "robustness": 66.7 + }, + "medium": { + "accuracy": 66.3, + "cost": 0.0001, + "robustness": 83.3 + }, + "hard": { + "accuracy": 12.5, + "cost": 0.0001, + "robustness": 66.7 + }, + "all": { + "accuracy": 69.7, + "cost": 0.0001, + "robustness": 71.4 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 87.7, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 54.0, + "cost": 0.0001, + "robustness": 75.0 + }, + "hard": { + "accuracy": 6.7, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 58.9, + "cost": 0.0001, + "robustness": 55.6 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": null + }, + "medium": { + "accuracy": 61.1, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 25.5, + "cost": 0.0, + "robustness": 66.7 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 20.0, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 75.4, + "cost": 0.0, + "robustness": 0.0 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 90.4, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 61.7, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 58.0, + "cost": 0.0, + "robustness": 20.0 + }, + "all": { + "accuracy": 68.6, + "cost": 0.0, + "robustness": 11.1 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 90.4, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 61.7, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 58.0, + "cost": 0.0, + "robustness": 20.0 + }, + "all": { + "accuracy": 68.6, + "cost": 0.0, + "robustness": 11.1 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.6, + "cost": 0.0, + "robustness": 61.5 + }, + "medium": { + "accuracy": 76.2, + "cost": 0.0001, + "robustness": 57.1 + }, + "hard": { + "accuracy": 11.3, + "cost": 0.0001, + "robustness": 44.4 + }, + "all": { + "accuracy": 79.6, + "cost": 0.0001, + "robustness": 58.0 + } + }, + "subcategories": { + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 72.7 + }, + "medium": { + "accuracy": 79.2, + "cost": 0.0001, + "robustness": 63.6 + }, + "hard": { + "accuracy": 12.4, + "cost": 0.0002, + "robustness": 80.0 + }, + "all": { + "accuracy": 77.5, + "cost": 0.0001, + "robustness": 70.4 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0, + "robustness": 50.0 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 7.7, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 90.0, + "cost": 0.0, + "robustness": 46.7 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0, + "robustness": 71.4 + }, + "medium": { + "accuracy": 64.7, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 89.5, + "cost": 0.0, + "robustness": 71.4 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 79.4, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 7.7, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 78.9, + "cost": 0.0001, + "robustness": 60.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 27.3, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 84.5, + "cost": 0.0, + "robustness": 80.0 + } + } + }, + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 80.0, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": null, + "cost": null, + "robustness": null + }, + "all": { + "accuracy": 96.0, + "cost": 0.0, + "robustness": 0.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 40.0 + }, + "medium": { + "accuracy": 44.0, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 8.5, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 40.4, + "cost": 0.0, + "robustness": 25.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 94.7, + "cost": 0.0001, + "robustness": 55.8 + }, + "medium": { + "accuracy": 64.3, + "cost": 0.0001, + "robustness": 57.1 + }, + "hard": { + "accuracy": 13.4, + "cost": 0.0001, + "robustness": 71.4 + }, + "all": { + "accuracy": 76.4, + "cost": 0.0001, + "robustness": 57.7 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 95.2, + "cost": 0.0001, + "robustness": 20.0 + }, + "medium": { + "accuracy": 79.0, + "cost": 0.0001, + "robustness": 53.8 + }, + "hard": { + "accuracy": 31.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 79.1, + "cost": 0.0001, + "robustness": 47.4 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.2, + "cost": 0.0001, + "robustness": 58.8 + }, + "medium": { + "accuracy": 53.3, + "cost": 0.0001, + "robustness": 71.4 + }, + "hard": { + "accuracy": 7.1, + "cost": 0.0001, + "robustness": 66.7 + }, + "all": { + "accuracy": 74.3, + "cost": 0.0001, + "robustness": 61.7 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0, + "robustness": 75.0 + }, + "medium": { + "accuracy": 81.8, + "cost": 0.0, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 93.2, + "cost": 0.0, + "robustness": 60.0 + } + } + } + } + }, + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 53.7, + "cost": 0.0002, + "robustness": 72.7 + }, + "hard": { + "accuracy": 6.1, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 61.1, + "cost": 0.0001, + "robustness": 50.0 + } + }, + "subcategories": { + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 92.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 54.0, + "cost": 0.0001, + "robustness": 75.0 + }, + "hard": { + "accuracy": 6.1, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 61.1, + "cost": 0.0001, + "robustness": 62.5 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 98.0, + "cost": 0.0001, + "robustness": 40.0 + }, + "medium": { + "accuracy": 43.1, + "cost": 0.0002, + "robustness": 60.0 + }, + "hard": { + "accuracy": 8.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "all": { + "accuracy": 62.1, + "cost": 0.0002, + "robustness": 45.5 + } + } + }, + "Arts": { + "metrics": { + "easy": { + "accuracy": 94.9, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 84.6, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 5.4, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 59.7, + "cost": 0.0, + "robustness": 40.0 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 66.7 + }, + "medium": { + "accuracy": 56.2, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 30.1, + "cost": 0.0001, + "robustness": 50.0 + }, + "all": { + "accuracy": 44.0, + "cost": 0.0001, + "robustness": 51.7 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 66.7 + }, + "medium": { + "accuracy": 56.2, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 30.1, + "cost": 0.0001, + "robustness": 50.0 + }, + "all": { + "accuracy": 44.0, + "cost": 0.0001, + "robustness": 51.7 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.8, + "cost": 0.0, + "robustness": 52.2 + }, + "medium": { + "accuracy": 62.1, + "cost": 0.0001, + "robustness": 16.7 + }, + "hard": { + "accuracy": 10.1, + "cost": 0.0001, + "robustness": 30.0 + }, + "all": { + "accuracy": 69.6, + "cost": 0.0001, + "robustness": 41.0 + } + }, + "subcategories": { + "Geography": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 37.5 + }, + "medium": { + "accuracy": 51.7, + "cost": 0.0, + "robustness": 0.0 + }, + "hard": { + "accuracy": 8.3, + "cost": 0.0001, + "robustness": null + }, + "all": { + "accuracy": 83.3, + "cost": 0.0, + "robustness": 33.3 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 54.5 + }, + "medium": { + "accuracy": 63.3, + "cost": 0.0001, + "robustness": 20.0 + }, + "hard": { + "accuracy": 10.3, + "cost": 0.0001, + "robustness": 30.0 + }, + "all": { + "accuracy": 63.5, + "cost": 0.0001, + "robustness": 38.5 + } + } + }, + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0, + "robustness": 75.0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 88.5, + "cost": 0.0, + "robustness": 75.0 + } + } + } + } + } + } + }, + "auto_router": { + "metrics": { + "easy": { + "accuracy": 94.9, + "cost": 0.0001, + "robustness": 48.0 + }, + "medium": { + "accuracy": 65.2, + "cost": 0.0001, + "robustness": 45.1 + }, + "hard": { + "accuracy": 25.1, + "cost": 0.0002, + "robustness": 56.9 + }, + "all": { + "accuracy": 69.9, + "cost": 0.0001, + "robustness": 49.5 + } + }, + "categories": { + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 42.3 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0002, + "robustness": 57.1 + }, + "hard": { + "accuracy": 31.4, + "cost": 0.0005, + "robustness": 60.0 + }, + "all": { + "accuracy": 79.4, + "cost": 0.0002, + "robustness": 51.6 + } + }, + "subcategories": { + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0001, + "robustness": 38.5 + }, + "medium": { + "accuracy": 55.3, + "cost": 0.0001, + "robustness": 25.0 + }, + "hard": { + "accuracy": 5.6, + "cost": 0.0001, + "robustness": 50.0 + }, + "all": { + "accuracy": 80.1, + "cost": 0.0001, + "robustness": 36.8 + } + } + }, + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0001, + "robustness": 46.2 + }, + "medium": { + "accuracy": 82.5, + "cost": 0.0002, + "robustness": 64.7 + }, + "hard": { + "accuracy": 35.9, + "cost": 0.0006, + "robustness": 61.5 + }, + "all": { + "accuracy": 79.2, + "cost": 0.0002, + "robustness": 58.1 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0001, + "robustness": 51.9 + }, + "medium": { + "accuracy": 62.1, + "cost": 0.0001, + "robustness": 30.0 + }, + "hard": { + "accuracy": 12.3, + "cost": 0.0001, + "robustness": 28.6 + }, + "all": { + "accuracy": 74.3, + "cost": 0.0001, + "robustness": 43.2 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 89.2, + "cost": 0.0, + "robustness": 50.0 + }, + "medium": { + "accuracy": 32.9, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 68.2, + "cost": 0.0001, + "robustness": 33.3 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 86.5, + "cost": 0.0, + "robustness": 33.3 + }, + "medium": { + "accuracy": 70.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 17.8, + "cost": 0.0001, + "robustness": 66.7 + }, + "all": { + "accuracy": 55.7, + "cost": 0.0001, + "robustness": 37.5 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 83.3 + }, + "medium": { + "accuracy": 85.4, + "cost": 0.0001, + "robustness": null + }, + "hard": { + "accuracy": 5.6, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 85.0, + "cost": 0.0001, + "robustness": 62.5 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 25.0 + }, + "medium": { + "accuracy": 89.5, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 20.0, + "cost": 0.0001, + "robustness": null + }, + "all": { + "accuracy": 92.6, + "cost": 0.0001, + "robustness": 57.1 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0001, + "robustness": 25.0 + }, + "hard": { + "accuracy": 7.4, + "cost": 0.0002, + "robustness": 42.9 + }, + "all": { + "accuracy": 64.7, + "cost": 0.0001, + "robustness": 40.5 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 41.7 + }, + "medium": { + "accuracy": 73.6, + "cost": 0.0002, + "robustness": 33.3 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0003, + "robustness": 33.3 + }, + "all": { + "accuracy": 71.6, + "cost": 0.0001, + "robustness": 38.1 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 87.7, + "cost": 0.0002, + "robustness": 33.3 + }, + "medium": { + "accuracy": 47.6, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 6.7, + "cost": 0.0002, + "robustness": 50.0 + }, + "all": { + "accuracy": 56.3, + "cost": 0.0002, + "robustness": 22.2 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 66.7, + "cost": 0.0, + "robustness": null + }, + "medium": { + "accuracy": 44.4, + "cost": 0.0, + "robustness": 50.0 + }, + "hard": { + "accuracy": 5.9, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 21.8, + "cost": 0.0, + "robustness": 33.3 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 33.3, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 88.5, + "cost": 0.0001, + "robustness": 63.6 + }, + "medium": { + "accuracy": 44.9, + "cost": 0.0001, + "robustness": 44.4 + }, + "hard": { + "accuracy": 48.4, + "cost": 0.0002, + "robustness": 80.0 + }, + "all": { + "accuracy": 59.6, + "cost": 0.0001, + "robustness": 68.9 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 88.5, + "cost": 0.0001, + "robustness": 63.6 + }, + "medium": { + "accuracy": 44.9, + "cost": 0.0001, + "robustness": 44.4 + }, + "hard": { + "accuracy": 48.4, + "cost": 0.0002, + "robustness": 80.0 + }, + "all": { + "accuracy": 59.6, + "cost": 0.0001, + "robustness": 68.9 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 97.2, + "cost": 0.0001, + "robustness": 43.6 + }, + "medium": { + "accuracy": 75.2, + "cost": 0.0001, + "robustness": 47.6 + }, + "hard": { + "accuracy": 18.0, + "cost": 0.0003, + "robustness": 55.6 + }, + "all": { + "accuracy": 79.4, + "cost": 0.0001, + "robustness": 46.4 + } + }, + "subcategories": { + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 96.2, + "cost": 0.0001, + "robustness": 18.2 + }, + "medium": { + "accuracy": 79.6, + "cost": 0.0001, + "robustness": 63.6 + }, + "hard": { + "accuracy": 28.1, + "cost": 0.0006, + "robustness": 20.0 + }, + "all": { + "accuracy": 79.2, + "cost": 0.0002, + "robustness": 37.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 37.5 + }, + "medium": { + "accuracy": 67.8, + "cost": 0.0001, + "robustness": 16.7 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 88.2, + "cost": 0.0001, + "robustness": 33.3 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 71.4 + }, + "medium": { + "accuracy": 58.8, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0002, + "robustness": null + }, + "all": { + "accuracy": 88.1, + "cost": 0.0001, + "robustness": 71.4 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 82.4, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0002, + "robustness": null + }, + "all": { + "accuracy": 81.1, + "cost": 0.0001, + "robustness": 20.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0, + "robustness": 66.7 + }, + "medium": { + "accuracy": 73.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 80.4, + "cost": 0.0001, + "robustness": 80.0 + } + } + }, + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0, + "robustness": 50.0 + }, + "medium": { + "accuracy": 80.0, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": null, + "cost": null, + "robustness": null + }, + "all": { + "accuracy": 92.0, + "cost": 0.0, + "robustness": 50.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0, + "robustness": 60.0 + }, + "medium": { + "accuracy": 52.0, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 5.1, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 39.5, + "cost": 0.0, + "robustness": 75.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 93.8, + "cost": 0.0001, + "robustness": 55.8 + }, + "medium": { + "accuracy": 62.9, + "cost": 0.0001, + "robustness": 42.9 + }, + "hard": { + "accuracy": 18.5, + "cost": 0.0002, + "robustness": 14.3 + }, + "all": { + "accuracy": 76.1, + "cost": 0.0001, + "robustness": 47.9 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 60.0 + }, + "medium": { + "accuracy": 80.2, + "cost": 0.0002, + "robustness": 46.2 + }, + "hard": { + "accuracy": 42.9, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 81.5, + "cost": 0.0002, + "robustness": 52.6 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0001, + "robustness": 52.9 + }, + "medium": { + "accuracy": 50.8, + "cost": 0.0001, + "robustness": 42.9 + }, + "hard": { + "accuracy": 9.7, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 73.2, + "cost": 0.0001, + "robustness": 44.7 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 75.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 90.4, + "cost": 0.0001, + "robustness": 60.0 + } + } + } + } + }, + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 93.0, + "cost": 0.0, + "robustness": 62.5 + }, + "medium": { + "accuracy": 48.6, + "cost": 0.0001, + "robustness": 63.6 + }, + "hard": { + "accuracy": 11.6, + "cost": 0.0001, + "robustness": 80.0 + }, + "all": { + "accuracy": 59.7, + "cost": 0.0001, + "robustness": 66.7 + } + }, + "subcategories": { + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 90.7, + "cost": 0.0, + "robustness": 50.0 + }, + "medium": { + "accuracy": 58.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 12.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 63.1, + "cost": 0.0001, + "robustness": 87.5 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 93.9, + "cost": 0.0, + "robustness": 80.0 + }, + "medium": { + "accuracy": 31.9, + "cost": 0.0001, + "robustness": 40.0 + }, + "hard": { + "accuracy": 24.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 56.7, + "cost": 0.0001, + "robustness": 54.5 + } + } + }, + "Arts": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 74.4, + "cost": 0.0, + "robustness": 50.0 + }, + "hard": { + "accuracy": 5.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 57.8, + "cost": 0.0001, + "robustness": 60.0 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 67.2, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 27.4, + "cost": 0.0002, + "robustness": 45.8 + }, + "all": { + "accuracy": 42.9, + "cost": 0.0002, + "robustness": 44.8 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 67.2, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 27.4, + "cost": 0.0002, + "robustness": 45.8 + }, + "all": { + "accuracy": 42.9, + "cost": 0.0002, + "robustness": 44.8 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 94.1, + "cost": 0.0001, + "robustness": 30.4 + }, + "medium": { + "accuracy": 62.1, + "cost": 0.0001, + "robustness": 33.3 + }, + "hard": { + "accuracy": 9.4, + "cost": 0.0001, + "robustness": 70.0 + }, + "all": { + "accuracy": 68.0, + "cost": 0.0001, + "robustness": 41.0 + } + }, + "subcategories": { + "Geography": { + "metrics": { + "easy": { + "accuracy": 97.2, + "cost": 0.0, + "robustness": 37.5 + }, + "medium": { + "accuracy": 48.3, + "cost": 0.0, + "robustness": 0.0 + }, + "hard": { + "accuracy": 16.7, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 81.3, + "cost": 0.0, + "robustness": 33.3 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0001, + "robustness": 27.3 + }, + "medium": { + "accuracy": 62.6, + "cost": 0.0001, + "robustness": 40.0 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0001, + "robustness": 70.0 + }, + "all": { + "accuracy": 61.0, + "cost": 0.0001, + "robustness": 46.2 + } + } + }, + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0, + "robustness": 25.0 + }, + "medium": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": null + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": null + }, + "all": { + "accuracy": 96.2, + "cost": 0.0, + "robustness": 25.0 + } + } + } + } + } + } + }, + "weave-router": { + "metrics": { + "easy": { + "accuracy": 97.1, + "cost": 0.0006, + "robustness": 79.3 + }, + "medium": { + "accuracy": 76.6, + "cost": 0.0011, + "robustness": 78.8 + }, + "hard": { + "accuracy": 38.5, + "cost": 0.0014, + "robustness": 78.9 + }, + "all": { + "accuracy": 78.4, + "cost": 0.0009, + "robustness": 79.0 + } + }, + "categories": { + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 95.2, + "cost": 0.0004, + "robustness": 88.5 + }, + "medium": { + "accuracy": 71.2, + "cost": 0.0011, + "robustness": 85.7 + }, + "hard": { + "accuracy": 32.1, + "cost": 0.0022, + "robustness": 73.3 + }, + "all": { + "accuracy": 78.1, + "cost": 0.0009, + "robustness": 83.9 + } + }, + "subcategories": { + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0003, + "robustness": 92.3 + }, + "medium": { + "accuracy": 67.9, + "cost": 0.0007, + "robustness": 100.0 + }, + "hard": { + "accuracy": 11.8, + "cost": 0.0006, + "robustness": 100.0 + }, + "all": { + "accuracy": 83.1, + "cost": 0.0004, + "robustness": 94.7 + } + } + }, + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 94.2, + "cost": 0.0004, + "robustness": 84.6 + }, + "medium": { + "accuracy": 72.0, + "cost": 0.0012, + "robustness": 82.4 + }, + "hard": { + "accuracy": 36.5, + "cost": 0.0025, + "robustness": 69.2 + }, + "all": { + "accuracy": 76.1, + "cost": 0.0012, + "robustness": 79.1 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 96.1, + "cost": 0.0004, + "robustness": 81.5 + }, + "medium": { + "accuracy": 72.5, + "cost": 0.0007, + "robustness": 90.0 + }, + "hard": { + "accuracy": 38.2, + "cost": 0.0012, + "robustness": 85.7 + }, + "all": { + "accuracy": 82.6, + "cost": 0.0006, + "robustness": 84.1 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 93.5, + "cost": 0.0003, + "robustness": 92.9 + }, + "medium": { + "accuracy": 40.0, + "cost": 0.0007, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0004, + "robustness": 100.0 + }, + "all": { + "accuracy": 73.1, + "cost": 0.0004, + "robustness": 95.2 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0004, + "robustness": 100.0 + }, + "medium": { + "accuracy": 97.4, + "cost": 0.0007, + "robustness": 50.0 + }, + "hard": { + "accuracy": 47.5, + "cost": 0.0014, + "robustness": 66.7 + }, + "all": { + "accuracy": 81.0, + "cost": 0.0009, + "robustness": 75.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0006, + "robustness": 50.0 + }, + "medium": { + "accuracy": 91.3, + "cost": 0.0009, + "robustness": null + }, + "hard": { + "accuracy": 33.3, + "cost": 0.0011, + "robustness": 100.0 + }, + "all": { + "accuracy": 89.7, + "cost": 0.0007, + "robustness": 62.5 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 98.5, + "cost": 0.0005, + "robustness": 75.0 + }, + "medium": { + "accuracy": 97.3, + "cost": 0.0007, + "robustness": 100.0 + }, + "hard": { + "accuracy": 80.0, + "cost": 0.0015, + "robustness": null + }, + "all": { + "accuracy": 97.2, + "cost": 0.0006, + "robustness": 85.7 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 98.0, + "cost": 0.0007, + "robustness": 94.4 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0012, + "robustness": 83.3 + }, + "hard": { + "accuracy": 21.3, + "cost": 0.0013, + "robustness": 71.4 + }, + "all": { + "accuracy": 74.6, + "cost": 0.001, + "robustness": 86.5 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0006, + "robustness": 91.7 + }, + "medium": { + "accuracy": 81.8, + "cost": 0.0012, + "robustness": 100.0 + }, + "hard": { + "accuracy": 21.3, + "cost": 0.0015, + "robustness": 100.0 + }, + "all": { + "accuracy": 78.1, + "cost": 0.001, + "robustness": 95.2 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 95.1, + "cost": 0.0012, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0017, + "robustness": 75.0 + }, + "hard": { + "accuracy": 20.7, + "cost": 0.0014, + "robustness": 50.0 + }, + "all": { + "accuracy": 71.8, + "cost": 0.0014, + "robustness": 77.8 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0004, + "robustness": null + }, + "medium": { + "accuracy": 72.2, + "cost": 0.0005, + "robustness": 50.0 + }, + "hard": { + "accuracy": 25.0, + "cost": 0.0011, + "robustness": 0.0 + }, + "all": { + "accuracy": 45.3, + "cost": 0.0009, + "robustness": 33.3 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 + }, + "medium": { + "accuracy": 44.4, + "cost": 0.0005, + "robustness": null + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 83.6, + "cost": 0.0003, + "robustness": 100.0 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0013, + "robustness": 27.3 + }, + "medium": { + "accuracy": 53.3, + "cost": 0.0011, + "robustness": 44.4 + }, + "hard": { + "accuracy": 53.0, + "cost": 0.0011, + "robustness": 84.0 + }, + "all": { + "accuracy": 65.4, + "cost": 0.0012, + "robustness": 62.2 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 94.3, + "cost": 0.0013, + "robustness": 27.3 + }, + "medium": { + "accuracy": 53.3, + "cost": 0.0011, + "robustness": 44.4 + }, + "hard": { + "accuracy": 53.0, + "cost": 0.0011, + "robustness": 84.0 + }, + "all": { + "accuracy": 65.4, + "cost": 0.0012, + "robustness": 62.2 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 99.2, + "cost": 0.0005, + "robustness": 82.1 + }, + "medium": { + "accuracy": 87.1, + "cost": 0.0009, + "robustness": 85.7 + }, + "hard": { + "accuracy": 33.1, + "cost": 0.0016, + "robustness": 88.9 + }, + "all": { + "accuracy": 87.7, + "cost": 0.0008, + "robustness": 84.1 + } + }, + "subcategories": { + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 99.6, + "cost": 0.0004, + "robustness": 81.8 + }, + "medium": { + "accuracy": 91.3, + "cost": 0.0009, + "robustness": 90.9 + }, + "hard": { + "accuracy": 49.2, + "cost": 0.002, + "robustness": 100.0 + }, + "all": { + "accuracy": 90.3, + "cost": 0.0009, + "robustness": 88.9 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 98.8, + "cost": 0.0003, + "robustness": 87.5 + }, + "medium": { + "accuracy": 76.3, + "cost": 0.0006, + "robustness": 100.0 + }, + "hard": { + "accuracy": 25.0, + "cost": 0.0009, + "robustness": 100.0 + }, + "all": { + "accuracy": 92.1, + "cost": 0.0004, + "robustness": 93.3 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0008, + "robustness": 85.7 + }, + "medium": { + "accuracy": 75.0, + "cost": 0.0011, + "robustness": null + }, + "hard": { + "accuracy": 62.5, + "cost": 0.0013, + "robustness": null + }, + "all": { + "accuracy": 94.2, + "cost": 0.0009, + "robustness": 85.7 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0007, + "robustness": 100.0 + }, + "medium": { + "accuracy": 93.8, + "cost": 0.001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0015, + "robustness": null + }, + "all": { + "accuracy": 89.5, + "cost": 0.0009, + "robustness": 80.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0004, + "robustness": 66.7 + }, + "medium": { + "accuracy": 89.2, + "cost": 0.0014, + "robustness": 50.0 + }, + "hard": { + "accuracy": 40.0, + "cost": 0.0019, + "robustness": null + }, + "all": { + "accuracy": 87.9, + "cost": 0.001, + "robustness": 60.0 + } + } + }, + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 50.0 + }, + "medium": { + "accuracy": 80.0, + "cost": 0.0002, + "robustness": null + }, + "hard": { + "accuracy": null, + "cost": null, + "robustness": null + }, + "all": { + "accuracy": 96.0, + "cost": 0.0003, + "robustness": 50.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 80.0 + }, + "medium": { + "accuracy": 70.8, + "cost": 0.0004, + "robustness": null + }, + "hard": { + "accuracy": 7.8, + "cost": 0.0012, + "robustness": 66.7 + }, + "all": { + "accuracy": 48.6, + "cost": 0.0008, + "robustness": 75.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 96.1, + "cost": 0.0006, + "robustness": 72.1 + }, + "medium": { + "accuracy": 76.9, + "cost": 0.0012, + "robustness": 90.5 + }, + "hard": { + "accuracy": 31.5, + "cost": 0.0014, + "robustness": 85.7 + }, + "all": { + "accuracy": 83.7, + "cost": 0.0009, + "robustness": 78.9 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0006, + "robustness": 100.0 + }, + "medium": { + "accuracy": 93.3, + "cost": 0.0015, + "robustness": 92.3 + }, + "hard": { + "accuracy": 67.6, + "cost": 0.0023, + "robustness": 100.0 + }, + "all": { + "accuracy": 92.5, + "cost": 0.0012, + "robustness": 94.7 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 95.3, + "cost": 0.0007, + "robustness": 64.7 + }, + "medium": { + "accuracy": 66.2, + "cost": 0.001, + "robustness": 85.7 + }, + "hard": { + "accuracy": 20.6, + "cost": 0.0011, + "robustness": 83.3 + }, + "all": { + "accuracy": 80.1, + "cost": 0.0008, + "robustness": 70.2 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 + }, + "medium": { + "accuracy": 81.8, + "cost": 0.0005, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0003, + "robustness": null + }, + "all": { + "accuracy": 94.5, + "cost": 0.0003, + "robustness": 100.0 + } + } + } + } + }, + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0006, + "robustness": 87.5 + }, + "medium": { + "accuracy": 80.2, + "cost": 0.0017, + "robustness": 54.5 + }, + "hard": { + "accuracy": 36.1, + "cost": 0.0026, + "robustness": 80.0 + }, + "all": { + "accuracy": 83.5, + "cost": 0.0014, + "robustness": 70.8 + } + }, + "subcategories": { + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 99.2, + "cost": 0.0009, + "robustness": 50.0 + }, + "medium": { + "accuracy": 92.1, + "cost": 0.002, + "robustness": 50.0 + }, + "hard": { + "accuracy": 63.6, + "cost": 0.0035, + "robustness": 50.0 + }, + "all": { + "accuracy": 94.8, + "cost": 0.0018, + "robustness": 50.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 95.9, + "cost": 0.0004, + "robustness": 100.0 + }, + "medium": { + "accuracy": 63.3, + "cost": 0.002, + "robustness": 80.0 + }, + "hard": { + "accuracy": 27.8, + "cost": 0.0024, + "robustness": 100.0 + }, + "all": { + "accuracy": 75.6, + "cost": 0.0014, + "robustness": 90.9 + } + } + }, + "Arts": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 100.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "hard": { + "accuracy": 32.6, + "cost": 0.0016, + "robustness": 100.0 + }, + "all": { + "accuracy": 79.4, + "cost": 0.0008, + "robustness": 60.0 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 + }, + "medium": { + "accuracy": 92.2, + "cost": 0.0008, + "robustness": 50.0 + }, + "hard": { + "accuracy": 44.2, + "cost": 0.001, + "robustness": 70.8 + }, + "all": { + "accuracy": 58.3, + "cost": 0.0009, + "robustness": 72.4 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 100.0 + }, + "medium": { + "accuracy": 92.2, + "cost": 0.0008, + "robustness": 50.0 + }, + "hard": { + "accuracy": 44.2, + "cost": 0.001, + "robustness": 70.8 + }, + "all": { + "accuracy": 58.3, + "cost": 0.0009, + "robustness": 72.4 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0005, + "robustness": 82.6 + }, + "medium": { + "accuracy": 81.1, + "cost": 0.001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 26.2, + "cost": 0.0012, + "robustness": 80.0 + }, + "all": { + "accuracy": 79.0, + "cost": 0.0008, + "robustness": 79.5 + } + }, + "subcategories": { + "Geography": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0004, + "robustness": 87.5 + }, + "medium": { + "accuracy": 75.9, + "cost": 0.0005, + "robustness": 100.0 + }, + "hard": { + "accuracy": 50.0, + "cost": 0.0009, + "robustness": null + }, + "all": { + "accuracy": 90.0, + "cost": 0.0004, + "robustness": 88.9 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 98.7, + "cost": 0.0006, + "robustness": 72.7 + }, + "medium": { + "accuracy": 81.0, + "cost": 0.0012, + "robustness": 60.0 + }, + "hard": { + "accuracy": 24.2, + "cost": 0.0013, + "robustness": 80.0 + }, + "all": { + "accuracy": 73.8, + "cost": 0.0009, + "robustness": 73.1 + } + } + }, + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": null + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0002, + "robustness": null + }, + "all": { + "accuracy": 96.2, + "cost": 0.0002, + "robustness": 100.0 + } + } + } + } + } + } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index c798a2e..c7f3618 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -4,7 +4,7 @@ "Arena Score": 74.61, "Optimal Selection Score": 1.38, "Optimal Cost Score": 12.27, - "Optimal Acc. Score": 100.00, + "Optimal Acc. Score": 100.0, "Robustness Score": 79.05, "Latency Score": null, "Accuracy": 78.43, @@ -12,25 +12,47 @@ }, { "Router Name": "R2-Router", - "Arena Score": 71.60, + "Arena Score": 71.6, "Optimal Selection Score": 32.38, "Optimal Cost Score": 74.16, - "Optimal Acc. Score": 100.00, + "Optimal Acc. Score": 100.0, "Robustness Score": 45.71, "Latency Score": null, "Accuracy": 71.23, "Cost per 1k": 0.06 }, { - "Router Name": "RouterDC", - "Arena Score": 33.75, - "Optimal Selection Score": 39.84, - "Optimal Cost Score": 72.998, - "Optimal Acc. Score": 49.05, - "Robustness Score": 85.24, - "Latency Score": 10.7527, - "Accuracy": 32.01, - "Cost per 1k": 0.07 + "Router Name": "Auto Router", + "Arena Score": 70.05, + "Optimal Selection Score": 43.77, + "Optimal Cost Score": 42.29, + "Optimal Acc. Score": 100.0, + "Robustness Score": 49.52, + "Latency Score": null, + "Accuracy": 70.17, + "Cost per 1k": 0.12 + }, + { + "Router Name": "vllm", + "Arena Score": 67.23, + "Optimal Selection Score": 94.1, + "Optimal Cost Score": 90.12, + "Optimal Acc. Score": 100.0, + "Robustness Score": 90.95, + "Latency Score": null, + "Accuracy": 66.53, + "Cost per 1k": 0.06 + }, + { + "Router Name": "mirt_bert", + "Arena Score": 66.89, + "Optimal Selection Score": 3.44, + "Optimal Cost Score": 19.6178, + "Optimal Acc. Score": 78.18, + "Robustness Score": 61.19, + "Latency Score": 27.027, + "Accuracy": 66.88, + "Cost per 1k": 0.15 }, { "Router Name": "azure", @@ -44,15 +66,15 @@ "Cost per 1k": 0.54 }, { - "Router Name": "carrot", - "Arena Score": 63.87, - "Optimal Selection Score": 2.68, - "Optimal Cost Score": 6.7697, - "Optimal Acc. Score": 78.63, - "Robustness Score": 89.05, - "Latency Score": 1.4993, - "Accuracy": 67.21, - "Cost per 1k": 2.06 + "Router Name": "nirt_bert", + "Arena Score": 66.12, + "Optimal Selection Score": 3.83, + "Optimal Cost Score": 14.039, + "Optimal Acc. Score": 77.88, + "Robustness Score": 49.29, + "Latency Score": 10.4167, + "Accuracy": 66.34, + "Cost per 1k": 0.21 }, { "Router Name": "gpt5", @@ -66,41 +88,41 @@ "Cost per 1k": 10.02 }, { - "Router Name": "graphrouter", - "Arena Score": 57.22, - "Optimal Selection Score": 4.73, - "Optimal Cost Score": 38.3347, - "Optimal Acc. Score": 74.25, - "Robustness Score": 94.29, - "Latency Score": 2.6954, - "Accuracy": 57.0, - "Cost per 1k": 0.34 + "Router Name": "carrot", + "Arena Score": 63.87, + "Optimal Selection Score": 2.68, + "Optimal Cost Score": 6.7697, + "Optimal Acc. Score": 78.63, + "Robustness Score": 89.05, + "Latency Score": 1.4993, + "Accuracy": 67.21, + "Cost per 1k": 2.06 }, { - "Router Name": "mirt_bert", - "Arena Score": 66.89, - "Optimal Selection Score": 3.44, - "Optimal Cost Score": 19.6178, - "Optimal Acc. Score": 78.18, - "Robustness Score": 61.19, - "Latency Score": 27.027, - "Accuracy": 66.88, - "Cost per 1k": 0.15 + "Router Name": "chayan", + "Arena Score": 63.83, + "Optimal Selection Score": 43.03, + "Optimal Cost Score": 43.75, + "Optimal Acc. Score": 88.74, + "Robustness Score": null, + "Latency Score": null, + "Accuracy": 64.89, + "Cost per 1k": 0.56 }, { - "Router Name": "nirt_bert", - "Arena Score": 66.12, - "Optimal Selection Score": 3.83, - "Optimal Cost Score": 14.039, - "Optimal Acc. Score": 77.88, - "Robustness Score": 49.29, - "Latency Score": 10.4167, - "Accuracy": 66.34, - "Cost per 1k": 0.21 + "Router Name": "routerbench_mlp", + "Arena Score": 57.56, + "Optimal Selection Score": 13.39, + "Optimal Cost Score": 24.4499, + "Optimal Acc. Score": 83.32, + "Robustness Score": 80.0, + "Latency Score": 90.9091, + "Accuracy": 61.62, + "Cost per 1k": 4.83 }, { "Router Name": "notdiamond", - "Arena Score": 63.0, + "Arena Score": 57.29, "Optimal Selection Score": 1.55, "Optimal Cost Score": 2.1367, "Optimal Acc. Score": 76.81, @@ -110,15 +132,15 @@ "Cost per 1k": 4.1 }, { - "Router Name": "routellm", - "Arena Score": 48.07, - "Optimal Selection Score": 99.72, - "Optimal Cost Score": 99.6314, - "Optimal Acc. Score": 68.76, - "Robustness Score": 100.00, - "Latency Score": 0.4016, - "Accuracy": 47.04, - "Cost per 1k": 0.27 + "Router Name": "graphrouter", + "Arena Score": 57.22, + "Optimal Selection Score": 4.73, + "Optimal Cost Score": 38.3347, + "Optimal Acc. Score": 74.25, + "Robustness Score": 94.29, + "Latency Score": 2.6954, + "Accuracy": 57.0, + "Cost per 1k": 0.34 }, { "Router Name": "routerbench_knn", @@ -132,36 +154,25 @@ "Cost per 1k": 4.27 }, { - "Router Name": "routerbench_mlp", - "Arena Score": 57.56, - "Optimal Selection Score": 13.39, - "Optimal Cost Score": 24.4499, - "Optimal Acc. Score": 83.32, - "Robustness Score": 80.00, - "Latency Score": 90.9091, - "Accuracy": 61.62, - "Cost per 1k": 4.83 - }, - { - "Router Name": "vllm", - "Arena Score": 67.23, - "Optimal Selection Score": 94.10, - "Optimal Cost Score": 90.12, - "Optimal Acc. Score": 100.00, - "Robustness Score": 90.95, - "Latency Score": null, - "Accuracy": 66.53, - "Cost per 1k": 0.06 + "Router Name": "routellm", + "Arena Score": 48.07, + "Optimal Selection Score": 99.72, + "Optimal Cost Score": 99.6314, + "Optimal Acc. Score": 68.76, + "Robustness Score": 100.0, + "Latency Score": 0.4016, + "Accuracy": 47.04, + "Cost per 1k": 0.27 }, { - "Router Name": "chayan", - "Arena Score": 63.83, - "Optimal Selection Score": 43.03, - "Optimal Cost Score": 43.75, - "Optimal Acc. Score": 88.74, - "Robustness Score": null, - "Latency Score": null, - "Accuracy": 64.89, - "Cost per 1k": 0.56 + "Router Name": "RouterDC", + "Arena Score": 33.75, + "Optimal Selection Score": 39.84, + "Optimal Cost Score": 72.998, + "Optimal Acc. Score": 49.05, + "Robustness Score": 85.24, + "Latency Score": 10.7527, + "Accuracy": 32.01, + "Cost per 1k": 0.07 } ] diff --git a/src/data/routers.json b/src/data/routers.json index ab38356..e9df9f6 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -103,7 +103,9 @@ "type": "closed-source", "description": "OpenAI's internal routing system for GPT model family", "affiliation": "OpenAI", - "modelPool": ["gpt-5-chat"], + "modelPool": [ + "gpt-5-chat" + ], "paperUrl": "https://openai.com/index/introducing-gpt-5/", "websiteUrl": "https://openai.com/index/introducing-gpt-5/" }, @@ -128,27 +130,6 @@ "websiteUrl": "https://arxiv.org/abs/2410.03834", "githubUrl": "https://github.com/ulab-uiuc/GraphRouter" }, - "universal_router": { - "name": "Universal Router", - "type": "open-source", - "description": "Universal baseline that mirrors the RouterBench model mix", - "affiliation": "RouterArena", - "modelPool": [ - "WizardLM/WizardLM-13B-V1.2", - "claude-instant-v1", - "claude-v1", - "claude-v2", - "gpt-3.5-turbo-1106", - "gpt-4-1106-preview", - "meta/codellama-34b-instruct", - "meta/llama-2-70b-chat", - "mistralai/mistral-7b-chat", - "mistralai/mixtral-8x7b-chat", - "zero-one-ai/Yi-34B-Chat" - ], - "paperUrl": "https://huggingface.co/datasets/withmartian/routerbench", - "websiteUrl": "https://huggingface.co/datasets/withmartian/routerbench" - }, "mirt_bert": { "name": "MIRT-BERT", "type": "open-source", @@ -226,18 +207,15 @@ "openai/gpt-4.1-mini", "openai/gpt-4.1-nano", "openai/o1-mini", - "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-haiku-20240307", "anthropic/claude-sonnet-4-20250514", - "google/gemini-1.5-pro-latest", "google/gemini-1.5-flash-latest", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash", "google/gemini-2.5-pro", - "mistral/codestral-latest", "mistral/open-mixtral-8x7b", "mistral/mistral-large-latest", @@ -245,7 +223,6 @@ "mistral/mistral-small-latest", "mistral/open-mistral-7b", "mistral/open-mistral-nemo", - "togetherai/Meta-Llama-3.1-8B-Instruct-Turbo", "togetherai/Meta-Llama-3.1-70B-Instruct-Turbo", "togetherai/Meta-Llama-3.1-405B-Instruct-Turbo" @@ -347,5 +324,17 @@ "paperUrl": "https://huggingface.co/adaptive-classifier/chayan", "websiteUrl": "https://huggingface.co/adaptive-classifier/chayan", "huggingfaceUrl": "https://huggingface.co/adaptive-classifier/chayan" + }, + "Auto Router": { + "name": "Auto Router", + "type": "open-source", + "description": "Cyclic baseline that rotates through its 4-model pool.", + "affiliation": "RouterArena", + "modelPool": [ + "deepseek/deepseek-v3.2", + "qwen/qwen3-235b-a22b-2507", + "qwen/qwen3.5-9b", + "openai/gpt-4o" + ] } } diff --git a/src/data/routers.yaml b/src/data/routers.yaml index 3d13c46..92f2bc5 100644 --- a/src/data/routers.yaml +++ b/src/data/routers.yaml @@ -195,3 +195,29 @@ chayan: paperUrl: https://huggingface.co/adaptive-classifier/chayan websiteUrl: https://huggingface.co/adaptive-classifier/chayan huggingfaceUrl: https://huggingface.co/adaptive-classifier/chayan + +R2-Router: + name: R2-Router + type: open-source + description: "R2-Router: Global KNN (K=80, cosine, distance-weighted) trained on sub_10 split. 6 models, 4 budgets. Routes via risk=(1-lambda)*quality - lambda*tokens*price. Lambda=0.999." + affiliation: UCF + modelPool: + - qwen/qwen3-235b-a22b-2507 + - qwen/qwen3-next-80b-a3b-instruct + - qwen/qwen3-30b-a3b-instruct-2507 + - Qwen/Qwen3-Coder-Next + - gemini-2.5-flash + - claude-3-haiku-20240307 + paperUrl: https://arxiv.org/abs/2602.02823 + websiteUrl: https://arxiv.org/abs/2602.02823 + +Auto Router: + name: Auto Router + type: open-source + description: Cyclic baseline that rotates through its 4-model pool. + affiliation: RouterArena + modelPool: + - deepseek/deepseek-v3.2 + - qwen/qwen3-235b-a22b-2507 + - qwen/qwen3.5-9b + - openai/gpt-4o From fb334ff7e35f74c0abddf85dbef8d7e3d29a1035 Mon Sep 17 00:00:00 2001 From: yl231 Date: Tue, 19 May 2026 21:51:51 -0500 Subject: [PATCH 2/4] Add Sqwish Router entry; default null per-domain cells to 0 - leaderboard.json: add Sqwish Router (RouterArena PR #94) at rank 1. Metrics from the /evaluate CI run on 2026-05-20: Arena 75.27, Accuracy 76.40, Cost/1K $0.18, Robustness 100.00, Opt 14.92/17.17/100.00. - category_scores.json: add per-domain entry for sqwish-router. Also refresh entries for r2-router, auto_router, weave-router so that difficulty-bucket cells with no samples default to 0 (matching the existing TypeScript schema) instead of null. The TS build was rejecting the prior null cells against CompareSubcategory.metrics. - routers.json + routers.yaml: add Sqwish Router metadata (closed-source, affiliation Sqwish, 5-model pool from the PR config). - flip_labels/flip_labels_sqwish-router.json: new (all-stable, since Sqwish picked the same model on every robustness pair). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../flip_labels_sqwish-router.json | 1682 +++++++++++++++++ src/data/routerMetrics/category_scores.json | 1034 +++++++++- src/data/routerMetrics/leaderboard.json | 11 + src/data/routers.json | 13 + src/data/routers.yaml | 12 + 5 files changed, 2701 insertions(+), 51 deletions(-) create mode 100644 src/data/flip_labels/flip_labels_sqwish-router.json diff --git a/src/data/flip_labels/flip_labels_sqwish-router.json b/src/data/flip_labels/flip_labels_sqwish-router.json new file mode 100644 index 0000000..87b217c --- /dev/null +++ b/src/data/flip_labels/flip_labels_sqwish-router.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 0 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 0 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 0 + }, + { + "global index": "ArcMMLU_646", + "flip": 0 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 0 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 0 + }, + { + "global index": "ArcMMLU_702", + "flip": 0 + }, + { + "global index": "ArcMMLU_713", + "flip": 0 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 0 + }, + { + "global index": "ChessInstruct_71", + "flip": 0 + }, + { + "global index": "ChessInstruct_84", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 0 + }, + { + "global index": "Ethics_deontology_2", + "flip": 0 + }, + { + "global index": "Ethics_deontology_31", + "flip": 0 + }, + { + "global index": "Ethics_deontology_32", + "flip": 0 + }, + { + "global index": "Ethics_deontology_56", + "flip": 0 + }, + { + "global index": "Ethics_justice_1", + "flip": 0 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 0 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 0 + }, + { + "global index": "Ethics_virtue_26", + "flip": 0 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 0 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 0 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 0 + }, + { + "global index": "GeoBench_124", + "flip": 0 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 0 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 0 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 0 + }, + { + "global index": "LiveCodeBench_131", + "flip": 0 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 0 + }, + { + "global index": "LiveCodeBench_431", + "flip": 0 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 0 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 0 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 0 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 0 + }, + { + "global index": "MMLUPro_business_784", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 0 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 0 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 0 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 0 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 0 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 0 + }, + { + "global index": "MedMCQA_511", + "flip": 0 + }, + { + "global index": "MedMCQA_59", + "flip": 0 + }, + { + "global index": "MedMCQA_643", + "flip": 0 + }, + { + "global index": "MedMCQA_853", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 0 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 0 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 0 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 0 + }, + { + "global index": "NarrativeQA_6829", + "flip": 0 + }, + { + "global index": "NarrativeQA_7678", + "flip": 0 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 0 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 0 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 0 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 0 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 0 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 0 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 0 + }, + { + "global index": "PubMedQA_8", + "flip": 0 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 0 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 0 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 0 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 0 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 0 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 0 + } +] \ No newline at end of file diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 1ccaaec..32e7796 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -12199,7 +12199,7 @@ "medium": { "accuracy": 81.2, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 11.1, @@ -12228,7 +12228,7 @@ "hard": { "accuracy": 60.0, "cost": 0.0003, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 90.7, @@ -12316,7 +12316,7 @@ "easy": { "accuracy": 100.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "medium": { "accuracy": 61.1, @@ -12345,7 +12345,7 @@ "medium": { "accuracy": 0.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 20.0, @@ -12493,12 +12493,12 @@ "medium": { "accuracy": 64.7, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 11.1, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 89.5, @@ -12522,7 +12522,7 @@ "hard": { "accuracy": 7.7, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 78.9, @@ -12546,7 +12546,7 @@ "hard": { "accuracy": 27.3, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 84.5, @@ -12565,12 +12565,12 @@ "medium": { "accuracy": 80.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { - "accuracy": null, - "cost": null, - "robustness": null + "accuracy": 0, + "cost": 0, + "robustness": 0 }, "all": { "accuracy": 96.0, @@ -12589,7 +12589,7 @@ "medium": { "accuracy": 44.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 8.5, @@ -12692,7 +12692,7 @@ "hard": { "accuracy": 0.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 93.2, @@ -12890,7 +12890,7 @@ "hard": { "accuracy": 8.3, "cost": 0.0001, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 83.3, @@ -12933,12 +12933,12 @@ "medium": { "accuracy": 77.8, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 0.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 88.5, @@ -13131,7 +13131,7 @@ "medium": { "accuracy": 85.4, "cost": 0.0001, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 5.6, @@ -13160,7 +13160,7 @@ "hard": { "accuracy": 20.0, "cost": 0.0001, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 92.6, @@ -13248,7 +13248,7 @@ "easy": { "accuracy": 66.7, "cost": 0.0, - "robustness": null + "robustness": 0 }, "medium": { "accuracy": 44.4, @@ -13277,7 +13277,7 @@ "medium": { "accuracy": 33.3, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 0.0, @@ -13425,12 +13425,12 @@ "medium": { "accuracy": 58.8, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 11.1, "cost": 0.0002, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 88.1, @@ -13454,7 +13454,7 @@ "hard": { "accuracy": 15.4, "cost": 0.0002, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 81.1, @@ -13478,7 +13478,7 @@ "hard": { "accuracy": 36.4, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 80.4, @@ -13497,12 +13497,12 @@ "medium": { "accuracy": 80.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { - "accuracy": null, - "cost": null, - "robustness": null + "accuracy": 0, + "cost": 0, + "robustness": 0 }, "all": { "accuracy": 92.0, @@ -13521,7 +13521,7 @@ "medium": { "accuracy": 52.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 5.1, @@ -13624,7 +13624,7 @@ "hard": { "accuracy": 0.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 90.4, @@ -13822,7 +13822,7 @@ "hard": { "accuracy": 16.7, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 81.3, @@ -13865,12 +13865,12 @@ "medium": { "accuracy": 100.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 0.0, "cost": 0.0, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 96.2, @@ -14063,7 +14063,7 @@ "medium": { "accuracy": 91.3, "cost": 0.0009, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 33.3, @@ -14092,7 +14092,7 @@ "hard": { "accuracy": 80.0, "cost": 0.0015, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 97.2, @@ -14180,7 +14180,7 @@ "easy": { "accuracy": 100.0, "cost": 0.0004, - "robustness": null + "robustness": 0 }, "medium": { "accuracy": 72.2, @@ -14209,7 +14209,7 @@ "medium": { "accuracy": 44.4, "cost": 0.0005, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 0.0, @@ -14357,12 +14357,12 @@ "medium": { "accuracy": 75.0, "cost": 0.0011, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 62.5, "cost": 0.0013, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 94.2, @@ -14386,7 +14386,7 @@ "hard": { "accuracy": 36.4, "cost": 0.0015, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 89.5, @@ -14410,7 +14410,7 @@ "hard": { "accuracy": 40.0, "cost": 0.0019, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 87.9, @@ -14429,12 +14429,12 @@ "medium": { "accuracy": 80.0, "cost": 0.0002, - "robustness": null + "robustness": 0 }, "hard": { - "accuracy": null, - "cost": null, - "robustness": null + "accuracy": 0, + "cost": 0, + "robustness": 0 }, "all": { "accuracy": 96.0, @@ -14453,7 +14453,7 @@ "medium": { "accuracy": 70.8, "cost": 0.0004, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 7.8, @@ -14556,7 +14556,7 @@ "hard": { "accuracy": 0.0, "cost": 0.0003, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 94.5, @@ -14754,7 +14754,7 @@ "hard": { "accuracy": 50.0, "cost": 0.0009, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 90.0, @@ -14797,12 +14797,12 @@ "medium": { "accuracy": 100.0, "cost": 0.0003, - "robustness": null + "robustness": 0 }, "hard": { "accuracy": 0.0, "cost": 0.0002, - "robustness": null + "robustness": 0 }, "all": { "accuracy": 96.2, @@ -14814,5 +14814,937 @@ } } } + }, + "sqwish-router": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 77.4, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 34.1, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 76.4, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "categories": { + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 84.0, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 34.7, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 82.1, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "subcategories": { + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 70.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 82.6, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 98.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 87.0, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 38.8, + "cost": 0.0004, + "robustness": 100.0 + }, + "all": { + "accuracy": 81.9, + "cost": 0.0002, + "robustness": 100.0 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 84.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 38.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 86.1, + "cost": 0.0001, + "robustness": 100.0 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 95.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 78.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 84.6, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 89.8, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 26.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 64.8, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 89.6, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 87.7, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 97.4, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 80.0, + "cost": 0.0005, + "robustness": 0 + }, + "all": { + "accuracy": 98.1, + "cost": 0.0002, + "robustness": 100.0 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.3, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 17.4, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 71.7, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 79.1, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 18.8, + "cost": 0.0004, + "robustness": 100.0 + }, + "all": { + "accuracy": 75.8, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 95.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 71.4, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 26.7, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.8, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 + }, + "medium": { + "accuracy": 61.1, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 30.9, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 33.3, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.9, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 59.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.7, + "cost": 0.0001, + "robustness": 100.0 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.9, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 59.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.7, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 99.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 82.6, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 30.9, + "cost": 0.0005, + "robustness": 100.0 + }, + "all": { + "accuracy": 84.5, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "subcategories": { + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.9, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 84.8, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 47.2, + "cost": 0.001, + "robustness": 100.0 + }, + "all": { + "accuracy": 85.4, + "cost": 0.0004, + "robustness": 100.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 98.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 90.9, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 76.5, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 44.4, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 93.7, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 91.2, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 84.4, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 88.1, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 45.5, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 87.6, + "cost": 0.0003, + "robustness": 100.0 + } + } + }, + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 92.0, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 6.8, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 45.6, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 30.6, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 81.5, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 88.0, + "cost": 0.0005, + "robustness": 100.0 + }, + "hard": { + "accuracy": 57.1, + "cost": 0.0009, + "robustness": 100.0 + }, + "all": { + "accuracy": 88.1, + "cost": 0.0005, + "robustness": 100.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 62.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 21.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 78.6, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 90.4, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 94.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 65.1, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 14.3, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 66.7, + "cost": 0.0003, + "robustness": 100.0 + } + }, + "subcategories": { + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 92.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 70.0, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 16.7, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 68.6, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 93.9, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 53.4, + "cost": 0.0007, + "robustness": 100.0 + }, + "hard": { + "accuracy": 12.0, + "cost": 0.0013, + "robustness": 100.0 + }, + "all": { + "accuracy": 65.8, + "cost": 0.0005, + "robustness": 100.0 + } + } + }, + "Arts": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 87.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 12.5, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 64.3, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 35.9, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 50.3, + "cost": 0.0001, + "robustness": 100.0 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 35.9, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 50.3, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 14.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 73.6, + "cost": 0.0001, + "robustness": 100.0 + } + }, + "subcategories": { + "Geography": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 62.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 25.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 85.3, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 76.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 13.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 68.1, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 92.3, + "cost": 0.0001, + "robustness": 100.0 + } + } + } + } + } + } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index c7f3618..1c77ac4 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -1,4 +1,15 @@ [ + { + "Router Name": "Sqwish Router", + "Arena Score": 75.27, + "Optimal Selection Score": 14.92, + "Optimal Cost Score": 17.17, + "Optimal Acc. Score": 100.0, + "Robustness Score": 100.0, + "Latency Score": null, + "Accuracy": 76.4, + "Cost per 1k": 0.18 + }, { "Router Name": "Weave Router", "Arena Score": 74.61, diff --git a/src/data/routers.json b/src/data/routers.json index e9df9f6..488c8a8 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -1,4 +1,17 @@ { + "Sqwish Router": { + "name": "Sqwish Router", + "type": "closed-source", + "description": "Sqwish Router final submission.", + "affiliation": "Sqwish", + "modelPool": [ + "qwen/qwen3-235b-a22b-2507", + "qwen/qwen3-next-80b-a3b-instruct", + "Qwen/Qwen3-Coder-Next", + "google/gemini-3.1-flash-lite", + "deepseek/deepseek-v4-flash" + ] + }, "Weave Router": { "name": "Weave Router", "type": "open-source", diff --git a/src/data/routers.yaml b/src/data/routers.yaml index 92f2bc5..1836215 100644 --- a/src/data/routers.yaml +++ b/src/data/routers.yaml @@ -221,3 +221,15 @@ Auto Router: - qwen/qwen3-235b-a22b-2507 - qwen/qwen3.5-9b - openai/gpt-4o + +Sqwish Router: + name: Sqwish Router + type: closed-source + description: Sqwish Router final submission. + affiliation: Sqwish + modelPool: + - qwen/qwen3-235b-a22b-2507 + - qwen/qwen3-next-80b-a3b-instruct + - Qwen/Qwen3-Coder-Next + - google/gemini-3.1-flash-lite + - deepseek/deepseek-v4-flash From c16a1cbd609def4fe81c376eff721623ced52ab1 Mon Sep 17 00:00:00 2001 From: yl231 Date: Tue, 19 May 2026 22:11:17 -0500 Subject: [PATCH 3/4] Apply PR #97 optimality fix to all workflow-evaluated routers RouterArena PR #97 (merged into main 2026-05-19) re-applied the PR #95 optimality fix and re-ran the /evaluate workflow for the four routers with for_optimality entries. The earlier sync used the pre-PR-#97 buggy values; this commit updates them to match origin/main README. - Weave Router: OptSel 1.38 -> 1.27, OptCost 12.27 -> 12.10, OptAcc 100.00 -> 89.56 - R2-Router: OptSel 32.38 -> 24.51, OptCost 74.16 -> 48.70, OptAcc 100.00 -> 99.85 - Auto Router: OptSel 43.77 -> 37.58, OptCost 42.29 -> 40.02, OptAcc 100.00 -> 86.04 - vLLM-SR: OptSel 94.10 -> 84.66, OptCost 90.12 -> 90.71, OptAcc 100.00 -> 89.24 Sqwish Router was evaluated by CI against PR #94's pre-#97 worktree, so its bot-reported numbers used the buggy formula. Recomputing locally with the merged main run.py on the same prediction file: - Sqwish Router: OptSel 14.92 -> 7.41, OptCost 17.17 -> 25.10, OptAcc 100.00 -> 90.47 Other rows (paper-era + Chayan) are unaffected since PR #97 only re-ran workflow-evaluated routers. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/data/routerMetrics/leaderboard.json | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index 1c77ac4..0c1e958 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -2,9 +2,9 @@ { "Router Name": "Sqwish Router", "Arena Score": 75.27, - "Optimal Selection Score": 14.92, - "Optimal Cost Score": 17.17, - "Optimal Acc. Score": 100.0, + "Optimal Selection Score": 7.41, + "Optimal Cost Score": 25.1, + "Optimal Acc. Score": 90.47, "Robustness Score": 100.0, "Latency Score": null, "Accuracy": 76.4, @@ -13,9 +13,9 @@ { "Router Name": "Weave Router", "Arena Score": 74.61, - "Optimal Selection Score": 1.38, - "Optimal Cost Score": 12.27, - "Optimal Acc. Score": 100.0, + "Optimal Selection Score": 1.27, + "Optimal Cost Score": 12.1, + "Optimal Acc. Score": 89.56, "Robustness Score": 79.05, "Latency Score": null, "Accuracy": 78.43, @@ -24,9 +24,9 @@ { "Router Name": "R2-Router", "Arena Score": 71.6, - "Optimal Selection Score": 32.38, - "Optimal Cost Score": 74.16, - "Optimal Acc. Score": 100.0, + "Optimal Selection Score": 24.51, + "Optimal Cost Score": 48.7, + "Optimal Acc. Score": 99.85, "Robustness Score": 45.71, "Latency Score": null, "Accuracy": 71.23, @@ -35,9 +35,9 @@ { "Router Name": "Auto Router", "Arena Score": 70.05, - "Optimal Selection Score": 43.77, - "Optimal Cost Score": 42.29, - "Optimal Acc. Score": 100.0, + "Optimal Selection Score": 37.58, + "Optimal Cost Score": 40.02, + "Optimal Acc. Score": 86.04, "Robustness Score": 49.52, "Latency Score": null, "Accuracy": 70.17, @@ -46,9 +46,9 @@ { "Router Name": "vllm", "Arena Score": 67.23, - "Optimal Selection Score": 94.1, - "Optimal Cost Score": 90.12, - "Optimal Acc. Score": 100.0, + "Optimal Selection Score": 84.66, + "Optimal Cost Score": 90.71, + "Optimal Acc. Score": 89.24, "Robustness Score": 90.95, "Latency Score": null, "Accuracy": 66.53, From 6aee9b47ef43aa09abda32bb323c7da0d1168f77 Mon Sep 17 00:00:00 2001 From: yl231 Date: Tue, 19 May 2026 23:08:23 -0500 Subject: [PATCH 4/4] Format data files with prettier (trailing newlines, inline single-element arrays) Co-Authored-By: Claude Opus 4.7 (1M context) --- src/data/flip_labels/flip_labels_auto_router.json | 2 +- src/data/flip_labels/flip_labels_r2-router.json | 2 +- src/data/flip_labels/flip_labels_sqwish-router.json | 2 +- src/data/flip_labels/flip_labels_vllm-sr.json | 2 +- src/data/flip_labels/flip_labels_weave-router.json | 2 +- src/data/routers.json | 9 ++------- src/data/routers.yaml | 2 +- 7 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/data/flip_labels/flip_labels_auto_router.json b/src/data/flip_labels/flip_labels_auto_router.json index 368efff..19ac384 100644 --- a/src/data/flip_labels/flip_labels_auto_router.json +++ b/src/data/flip_labels/flip_labels_auto_router.json @@ -1679,4 +1679,4 @@ "global index": "WMT19-zh-en_59", "flip": 0 } -] \ No newline at end of file +] diff --git a/src/data/flip_labels/flip_labels_r2-router.json b/src/data/flip_labels/flip_labels_r2-router.json index 0056bc5..41fee9c 100644 --- a/src/data/flip_labels/flip_labels_r2-router.json +++ b/src/data/flip_labels/flip_labels_r2-router.json @@ -1679,4 +1679,4 @@ "global index": "WMT19-zh-en_59", "flip": 1 } -] \ No newline at end of file +] diff --git a/src/data/flip_labels/flip_labels_sqwish-router.json b/src/data/flip_labels/flip_labels_sqwish-router.json index 87b217c..9450917 100644 --- a/src/data/flip_labels/flip_labels_sqwish-router.json +++ b/src/data/flip_labels/flip_labels_sqwish-router.json @@ -1679,4 +1679,4 @@ "global index": "WMT19-zh-en_59", "flip": 0 } -] \ No newline at end of file +] diff --git a/src/data/flip_labels/flip_labels_vllm-sr.json b/src/data/flip_labels/flip_labels_vllm-sr.json index fe6e3a5..67fb612 100644 --- a/src/data/flip_labels/flip_labels_vllm-sr.json +++ b/src/data/flip_labels/flip_labels_vllm-sr.json @@ -1679,4 +1679,4 @@ "global index": "WMT19-zh-en_59", "flip": 0 } -] \ No newline at end of file +] diff --git a/src/data/flip_labels/flip_labels_weave-router.json b/src/data/flip_labels/flip_labels_weave-router.json index ba790c1..699ecfc 100644 --- a/src/data/flip_labels/flip_labels_weave-router.json +++ b/src/data/flip_labels/flip_labels_weave-router.json @@ -1679,4 +1679,4 @@ "global index": "WMT19-zh-en_59", "flip": 0 } -] \ No newline at end of file +] diff --git a/src/data/routers.json b/src/data/routers.json index 488c8a8..7e85b57 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -116,9 +116,7 @@ "type": "closed-source", "description": "OpenAI's internal routing system for GPT model family", "affiliation": "OpenAI", - "modelPool": [ - "gpt-5-chat" - ], + "modelPool": ["gpt-5-chat"], "paperUrl": "https://openai.com/index/introducing-gpt-5/", "websiteUrl": "https://openai.com/index/introducing-gpt-5/" }, @@ -248,10 +246,7 @@ "type": "open-source", "description": "Binary selection between strong and weak models", "affiliation": "Berkeley", - "modelPool": [ - "openai-gpt-4o", - "mixtral-8x7b-instruct" - ], + "modelPool": ["openai-gpt-4o", "mixtral-8x7b-instruct"], "paperUrl": "https://arxiv.org/abs/2406.18665", "websiteUrl": "https://arxiv.org/abs/2406.18665", "githubUrl": "https://github.com/lm-sys/RouteLLM", diff --git a/src/data/routers.yaml b/src/data/routers.yaml index 1836215..dd7d89b 100644 --- a/src/data/routers.yaml +++ b/src/data/routers.yaml @@ -199,7 +199,7 @@ chayan: R2-Router: name: R2-Router type: open-source - description: "R2-Router: Global KNN (K=80, cosine, distance-weighted) trained on sub_10 split. 6 models, 4 budgets. Routes via risk=(1-lambda)*quality - lambda*tokens*price. Lambda=0.999." + description: 'R2-Router: Global KNN (K=80, cosine, distance-weighted) trained on sub_10 split. 6 models, 4 budgets. Routes via risk=(1-lambda)*quality - lambda*tokens*price. Lambda=0.999.' affiliation: UCF modelPool: - qwen/qwen3-235b-a22b-2507