From 5b3cd265dcae4049eb382c4b46fe03b7ecc51d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Wed, 25 Jun 2025 14:29:52 +0200 Subject: [PATCH 01/15] added instruct specific evals --- src/lighteval/tasks/extended/instruct.py | 346 +++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 src/lighteval/tasks/extended/instruct.py diff --git a/src/lighteval/tasks/extended/instruct.py b/src/lighteval/tasks/extended/instruct.py new file mode 100644 index 000000000..b58ebec9e --- /dev/null +++ b/src/lighteval/tasks/extended/instruct.py @@ -0,0 +1,346 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from functools import partial + +import numpy as np +from langcodes import standardize_tag + +from lighteval.metrics.dynamic_metrics import ( + IndicesExtractionConfig, + multilingual_extractive_match_metric, +) +from lighteval.metrics.metrics import MetricCategory, MetricUseCase, SampleLevelMetric +from lighteval.metrics.metrics_sample import ( + PassAtK, +) +from lighteval.tasks.default_prompts import LETTER_INDICES +from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.requests import Doc +from lighteval.utils.language import Language + + +TASKS_TABLE = [] + +lang_to_literal = { + "deu": Language.GERMAN, + "fra": Language.FRENCH, + "ita": Language.ITALIAN, + "por": Language.PORTUGUESE, + "spa": Language.SPANISH, +} + + +def belebele_prompt(line, task_name: str = None): + lang_to_template = { + "eng_Latn": "Given the following passage, query, and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of A, B, C, or D. Think step by step before answering.\n\n###\nPassage:\n{Passage}\n###\nQuery:\n{Question}\n###\nChoices:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "deu_Latn": "Gib basierend auf dem folgenden Textabschnitt, der Frage und den Antwortmöglichkeiten den Buchstaben aus, der der richtigen Antwort entspricht. Die letzte Zeile deiner Antwort sollte folgendes Format haben: 'Antwort: $BUCHSTABE' (ohne Anführungszeichen), wobei BUCHSTABE einer der folgenden ist: A, B, C oder D. Denke Schritt für Schritt, bevor du antwortest.\n\n###\nTextabschnitt:\n{Passage}\n###\nFrage:\n{Question}\n###\nAntwortmöglichkeiten:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "fra_Latn": "A partir du passage suivant, de la question et des choix de réponses, indiquez la lettre correspondant à la bonne réponse. La dernière ligne de votre réponse doit avoir le format suivant : 'Réponse: '$LETTRE' (sans les guillemets) où LETTRE est l'une des lettres: A, B, C ou D. Réfléchissez étape par étape avant de répondre.\n\n###\nPassage:\n{Passage}\n###\nRequête:\n{Question}\n###\nChoix:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "ita_Latn": "Dato il seguente passaggio, un quesito e le diverse opzioni per una risposta, indicare la lettera corrispondente alla risposta corretta. L'ultima riga della risposta deve avere il seguente formato: 'Risposta: $LETTERA' (senza virgolette), e LETTERA è necessariamente una tra A, B, C, D. Prima di rispondere, è importante che si ragioni passo per passo.\n\n###\nPassaggio:\n{Passage}\n###\nQuesito:\n{Question}\n###\nOpzioni:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "por_Latn": "Tendo em conta a seguinte passagem, pergunta e opções de resposta, indique a letra correspondente à resposta correta. A última linha da sua resposta deve ter o seguinte formato: 'Resposta: $LETRA' (sem aspas) em que LETRA é uma de A, B, C ou D. Pense passo a passo antes de responder.\n\n###\nPassagem:\n{Passage}\n###\nPergunta:\n{Question}\n###\nOpções:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "spa_Latn": "Dado el siguiente contexto, pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\n###\nContexto:\n{Passage}\n###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + } + + gold_index = int(line["correct_answer_num"]) - 1 + choices = [line["mc_answer1"], line["mc_answer2"], line["mc_answer3"], line["mc_answer4"]] + query_template = lang_to_template.get(line["dialect"], "eng_Latn") + query = query_template.format( + A=choices[0], + B=choices[1], + C=choices[2], + D=choices[3], + Passage=line["flores_passage"], + Question=line["question"], + ) + instruction = query_template.split("\n\n###")[0] + + return Doc( + task_name=task_name, + query=query, + choices=LETTER_INDICES[: len(choices)], + gold_index=gold_index, + instruction=instruction, + ) + + +BELEBELE_TASKS = [ + LightevalTaskConfig( + name=f"belebele_instruct_{lang}_Latn", + prompt_function=belebele_prompt, + suite=["extended"], + hf_repo="facebook/belebele", + hf_subset=f"{lang}_Latn", + evaluation_splits=["test"], + hf_avail_splits=["test"], + few_shots_split=None, + few_shots_select=None, + generation_size=32768, # needed for reasoning models like R1 + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=lang_to_literal[lang], + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], + stop_sequence=[], # no stop sequence, will use eos token + trust_dataset=True, + version=1, + ) + for lang in [ + "deu", + "fra", + "ita", + "por", + "spa", + ] +] + +TASKS_TABLE.extend(BELEBELE_TASKS) + + +MMLU_SUBSETS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +class GlobalMMLUPrompt: + def __init__(self, lang): + self.lang = lang + self.lang_to_template = { + "eng": "Given the following query and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of A, B, C, or D. Think step by step before answering.\n\n###\nQuery:\n{Question}\n###\nChoices:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "deu": "Gib basierend auf der folgenden Frage und den Antwortmöglichkeiten den Buchstaben aus, der der richtigen Antwort entspricht. Die letzte Zeile deiner Antwort sollte folgendes Format haben: 'Antwort: $BUCHSTABE' (ohne Anführungszeichen), wobei BUCHSTABE einer der folgenden ist: A, B, C oder D. Denke Schritt für Schritt, bevor du antwortest.\n\n###\nFrage:\n{Question}\n###\nAntwortmöglichkeiten:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "fra": "A partir de la question et des choix de réponses suivants, indiquez la lettre correspondant à la bonne réponse. La dernière ligne de votre réponse doit avoir le format suivant : 'Réponse: '$LETTRE' (sans les guillemets) où LETTRE est l'une des lettres: A, B, C ou D. Réfléchissez étape par étape avant de répondre.\n\n###\nRequête:\n{Question}\n###\nChoix:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "ita": "Dato il seguente quesito e le diverse opzioni per una risposta, indicare la lettera corrispondente alla risposta corretta. L'ultima riga della risposta deve avere il seguente formato: 'Risposta: $LETTERA' (senza virgolette), e LETTERA è necessariamente una tra A, B, C, D. Prima di rispondere, è importante che si ragioni passo per passo.\n\n###\nQuesito:\n{Question}\n###\nOpzioni:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "por": "Tendo em conta a seguinte pergunta e opções de resposta, indique a letra correspondente à resposta correta. A última linha da sua resposta deve ter o seguinte formato: 'Resposta: $LETRA' (sem aspas) em que LETRA é uma de A, B, C ou D. Pense passo a passo antes de responder.\n\n###\nPergunta:\n{Question}\n###\nOpções:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\\###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + } + + def prompt(self, line, task_name: str = None): + gold_index = LETTER_INDICES.index(line["answer"]) + choices = [line["option_a"], line["option_b"], line["option_c"], line["option_d"]] + query_template = self.lang_to_template.get(self.lang, "eng") + query = query_template.format( + A=choices[0], + B=choices[1], + C=choices[2], + D=choices[3], + Question=line["question"], + ) + instruction = query_template.split("\n\n###")[0] + + return Doc( + task_name=task_name, + query=query, + choices=LETTER_INDICES[: len(choices)], + gold_index=gold_index, + instruction=instruction, + ) + + +global_mmlu_tasks = [ + LightevalTaskConfig( + name=f"global_mmlu_instruct_{sensitivity_label.lower()}_{language.value}:{subset}", + prompt_function=GlobalMMLUPrompt(language).prompt, + suite=("extended"), + hf_repo="CohereForAI/Global-MMLU", + hf_subset=standardize_tag(language.value), + evaluation_splits=("test",), + few_shots_split="dev", + hf_filter=partial( + lambda subset, sensitivity_label, x: x["subject"].lower() == subset + and ( + sensitivity_label == "ALL" or sensitivity_label in x["cultural_sensitivity_label"].replace("-", "UNK") + ) + and all(x[f"option_{opt}"] is not None and x[f"option_{opt}"].strip() for opt in "abcd"), + subset, + sensitivity_label, + ), + metric=SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=language, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ), + generation_size=32768, # needed for reasoning models like R1 + stop_sequence=[], # no stop sequence, will use eos token + ) + for subset in MMLU_SUBSETS + for language in [ + Language.GERMAN, + Language.ENGLISH, + Language.SPANISH, + Language.FRENCH, + Language.HEBREW, + Language.HINDI, + Language.INDONESIAN, + Language.ITALIAN, + Language.JAPANESE, + Language.KOREAN, + Language.MALAY, + Language.DUTCH, + Language.NORWEGIAN, + Language.POLISH, + Language.PORTUGUESE, + Language.ROMANIAN, + Language.RUSSIAN, + Language.SERBIAN, + Language.SWEDISH, + Language.SWAHILI, + Language.TAMIL, + Language.TELUGU, + Language.THAI, + Language.TURKISH, + Language.UKRAINIAN, + Language.URDU, + Language.VIETNAMESE, + Language.YORUBA, + Language.ZULU, + ] + for sensitivity_label in ["ALL", "CA", "CS", "UNK"] +] + + +def mmlu_pro(line, task_name: str = None): + instruction = f"Given the following question about {line['category']} and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of {' ,'.join(LETTER_INDICES[: len(line['choices'] - 1)])}, or {LETTER_INDICES[len(line['choices'])]}. Think step by step before answering.\n\n" + query = f"{instruction}###\nQuery:\n{line['question']}\n###\nChoices:\n" + query += "".join([f"\n{key}) {choice}" for key, choice in zip(LETTER_INDICES, line["choices"])]) + + return Doc( + task_name=task_name, + query=query, + choices=LETTER_INDICES[: len(line["choices"])], + gold_index=line["answer_index"], + instruction=instruction, + ) + + +mmlu_pro = LightevalTaskConfig( + name="mmlu_pro", + suite=["lighteval"], + prompt_function=mmlu_pro, + hf_repo="TIGER-Lab/MMLU-Pro", + hf_subset="default", + hf_avail_splits=["validation", "test"], + evaluation_splits=["test"], + few_shots_split="validation", + few_shots_select=None, + generation_size=32768, # needed for reasoning models like R1 + stop_sequence=[], # no stop sequence, will use eos token + metric=SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=Language.ENGLISH, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ), + trust_dataset=True, + version=0, +) From 75af52df57957a9a54c48970881f8ec04771a22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Wed, 25 Jun 2025 14:32:11 +0200 Subject: [PATCH 02/15] tmp --- src/lighteval/tasks/extended/instruct.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lighteval/tasks/extended/instruct.py b/src/lighteval/tasks/extended/instruct.py index b58ebec9e..1b1c08ed8 100644 --- a/src/lighteval/tasks/extended/instruct.py +++ b/src/lighteval/tasks/extended/instruct.py @@ -126,8 +126,6 @@ def belebele_prompt(line, task_name: str = None): ] ] -TASKS_TABLE.extend(BELEBELE_TASKS) - MMLU_SUBSETS = [ "abstract_algebra", @@ -224,7 +222,7 @@ def prompt(self, line, task_name: str = None): ) -global_mmlu_tasks = [ +GLOBAL_MMLU_TASKS = [ LightevalTaskConfig( name=f"global_mmlu_instruct_{sensitivity_label.lower()}_{language.value}:{subset}", prompt_function=GlobalMMLUPrompt(language).prompt, @@ -314,7 +312,7 @@ def mmlu_pro(line, task_name: str = None): mmlu_pro = LightevalTaskConfig( name="mmlu_pro", - suite=["lighteval"], + suite=["extended"], prompt_function=mmlu_pro, hf_repo="TIGER-Lab/MMLU-Pro", hf_subset="default", @@ -344,3 +342,7 @@ def mmlu_pro(line, task_name: str = None): trust_dataset=True, version=0, ) + +TASKS_TABLE.extend(BELEBELE_TASKS) +TASKS_TABLE.extend(GLOBAL_MMLU_TASKS) +TASKS_TABLE.append(mmlu_pro) From ffa43ccb357245760e2b47f46362be27ec48be42 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Wed, 25 Jun 2025 13:55:00 +0000 Subject: [PATCH 03/15] fix --- src/lighteval/tasks/extended/__init__.py | 8 +- .../tasks/extended/{ => misc}/instruct.py | 101 +++++++++--------- 2 files changed, 57 insertions(+), 52 deletions(-) rename src/lighteval/tasks/extended/{ => misc}/instruct.py (90%) diff --git a/src/lighteval/tasks/extended/__init__.py b/src/lighteval/tasks/extended/__init__.py index 39963eac1..b2e44433a 100644 --- a/src/lighteval/tasks/extended/__init__.py +++ b/src/lighteval/tasks/extended/__init__.py @@ -20,9 +20,12 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import lighteval.tasks.extended.misc.instruct as instruct from lighteval.utils.imports import can_load_extended_tasks +AVAILABLE_EXTENDED_TASKS_MODULES = [instruct] + if can_load_extended_tasks(): import lighteval.tasks.extended.hle.main as hle import lighteval.tasks.extended.ifeval.main as ifeval @@ -32,7 +35,4 @@ import lighteval.tasks.extended.olympiade_bench.main as olympiad_bench import lighteval.tasks.extended.tiny_benchmarks.main as tiny_benchmarks - AVAILABLE_EXTENDED_TASKS_MODULES = [ifeval, tiny_benchmarks, mt_bench, mix_eval, olympiad_bench, hle, lcb] - -else: - AVAILABLE_EXTENDED_TASKS_MODULES = [] + AVAILABLE_EXTENDED_TASKS_MODULES.extend([ifeval, tiny_benchmarks, mt_bench, mix_eval, olympiad_bench, hle, lcb]) diff --git a/src/lighteval/tasks/extended/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py similarity index 90% rename from src/lighteval/tasks/extended/instruct.py rename to src/lighteval/tasks/extended/misc/instruct.py index 1b1c08ed8..5eac5b1fe 100644 --- a/src/lighteval/tasks/extended/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -125,6 +125,7 @@ def belebele_prompt(line, task_name: str = None): "spa", ] ] +TASKS_TABLE.extend(BELEBELE_TASKS) MMLU_SUBSETS = [ @@ -225,8 +226,8 @@ def prompt(self, line, task_name: str = None): GLOBAL_MMLU_TASKS = [ LightevalTaskConfig( name=f"global_mmlu_instruct_{sensitivity_label.lower()}_{language.value}:{subset}", - prompt_function=GlobalMMLUPrompt(language).prompt, - suite=("extended"), + prompt_function=GlobalMMLUPrompt(language.value).prompt, + suite=["extended"], hf_repo="CohereForAI/Global-MMLU", hf_subset=standardize_tag(language.value), evaluation_splits=("test",), @@ -240,23 +241,25 @@ def prompt(self, line, task_name: str = None): subset, sensitivity_label, ), - metric=SampleLevelMetric( - metric_name="pass@1:1_samples", - sample_level_fn=PassAtK( - k=1, - n=1, - sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( - language=language, - gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - precision=6, - ).sample_level_fn([ref], [pred], doc), - ).compute, - category=MetricCategory.GENERATIVE_SAMPLING, - use_case=MetricUseCase.REASONING, - corpus_level_fn=np.mean, - higher_is_better=True, - ), + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=language, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], generation_size=32768, # needed for reasoning models like R1 stop_sequence=[], # no stop sequence, will use eos token ) @@ -266,45 +269,47 @@ def prompt(self, line, task_name: str = None): Language.ENGLISH, Language.SPANISH, Language.FRENCH, - Language.HEBREW, + # Language.HEBREW, Language.HINDI, Language.INDONESIAN, Language.ITALIAN, Language.JAPANESE, - Language.KOREAN, - Language.MALAY, + # Language.KOREAN, + # Language.MALAY, Language.DUTCH, Language.NORWEGIAN, Language.POLISH, Language.PORTUGUESE, - Language.ROMANIAN, + # Language.ROMANIAN, Language.RUSSIAN, Language.SERBIAN, Language.SWEDISH, Language.SWAHILI, - Language.TAMIL, + # Language.TAMIL, Language.TELUGU, Language.THAI, Language.TURKISH, Language.UKRAINIAN, Language.URDU, Language.VIETNAMESE, - Language.YORUBA, - Language.ZULU, + # Language.YORUBA, + # Language.ZULU, missing literals ] for sensitivity_label in ["ALL", "CA", "CS", "UNK"] ] +TASKS_TABLE.extend(GLOBAL_MMLU_TASKS) def mmlu_pro(line, task_name: str = None): - instruction = f"Given the following question about {line['category']} and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of {' ,'.join(LETTER_INDICES[: len(line['choices'] - 1)])}, or {LETTER_INDICES[len(line['choices'])]}. Think step by step before answering.\n\n" + num_choices = len(line["options"]) + instruction = f"Given the following question about {line['category']} and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of {' ,'.join(LETTER_INDICES[: num_choices - 1])}, or {LETTER_INDICES[num_choices]}. Think step by step before answering.\n\n" query = f"{instruction}###\nQuery:\n{line['question']}\n###\nChoices:\n" - query += "".join([f"\n{key}) {choice}" for key, choice in zip(LETTER_INDICES, line["choices"])]) + query += "".join([f"\n{key}) {choice}" for key, choice in zip(LETTER_INDICES, line["options"])]) return Doc( task_name=task_name, query=query, - choices=LETTER_INDICES[: len(line["choices"])], + choices=LETTER_INDICES[:num_choices], gold_index=line["answer_index"], instruction=instruction, ) @@ -322,27 +327,27 @@ def mmlu_pro(line, task_name: str = None): few_shots_select=None, generation_size=32768, # needed for reasoning models like R1 stop_sequence=[], # no stop sequence, will use eos token - metric=SampleLevelMetric( - metric_name="pass@1:1_samples", - sample_level_fn=PassAtK( - k=1, - n=1, - sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( - language=Language.ENGLISH, - gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - precision=6, - ).sample_level_fn([ref], [pred], doc), - ).compute, - category=MetricCategory.GENERATIVE_SAMPLING, - use_case=MetricUseCase.REASONING, - corpus_level_fn=np.mean, - higher_is_better=True, - ), + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=Language.ENGLISH, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], trust_dataset=True, version=0, ) -TASKS_TABLE.extend(BELEBELE_TASKS) -TASKS_TABLE.extend(GLOBAL_MMLU_TASKS) TASKS_TABLE.append(mmlu_pro) From a94973a28c06d24fbf42c283a0c11134a8c2beff Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Wed, 25 Jun 2025 14:21:02 +0000 Subject: [PATCH 04/15] fix --- src/lighteval/tasks/extended/misc/instruct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py index 5eac5b1fe..2dcb15b85 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -303,7 +303,7 @@ def prompt(self, line, task_name: str = None): def mmlu_pro(line, task_name: str = None): num_choices = len(line["options"]) instruction = f"Given the following question about {line['category']} and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of {' ,'.join(LETTER_INDICES[: num_choices - 1])}, or {LETTER_INDICES[num_choices]}. Think step by step before answering.\n\n" - query = f"{instruction}###\nQuery:\n{line['question']}\n###\nChoices:\n" + query = f"{instruction}###\nQuery:\n{line['question']}\n###\nChoices:" query += "".join([f"\n{key}) {choice}" for key, choice in zip(LETTER_INDICES, line["options"])]) return Doc( From 2852db8dc91e9e89d13cf95dba6267c5608b455d Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Wed, 25 Jun 2025 14:51:09 +0000 Subject: [PATCH 05/15] tmp --- examples/tasks/instruct_multilingual.txt | 5477 +++++++++++++++++ src/lighteval/tasks/extended/misc/instruct.py | 4 +- 2 files changed, 5480 insertions(+), 1 deletion(-) create mode 100644 examples/tasks/instruct_multilingual.txt diff --git a/examples/tasks/instruct_multilingual.txt b/examples/tasks/instruct_multilingual.txt new file mode 100644 index 000000000..cd65763b6 --- /dev/null +++ b/examples/tasks/instruct_multilingual.txt @@ -0,0 +1,5477 @@ +extended|'belebele_instruct_deu_Latn|0|0 + extended|belebele_instruct_fra_Latn|0|0 + extended|belebele_instruct_ita_Latn|0|0 + extended|belebele_instruct_por_Latn|0|0 + extended|belebele_instruct_spa_Latn|0|0 + extended|global_mmlu_instruct_all_ara:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_ara:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_ara:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_ara:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_ben:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_ben:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_ben:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_ben:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_deu:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_deu:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_deu:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_deu:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_spa:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_spa:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_spa:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_spa:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_eng:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_eng:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_eng:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_eng:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_fra:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_fra:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_fra:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_fra:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_hin:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_hin:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_hin:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_hin:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_ind:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_ind:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_ind:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_ind:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_ita:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_ita:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_ita:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_ita:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_jpn:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_jpn:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_jpn:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_jpn:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_nld:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_nld:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_nld:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_nld:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_nor:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_nor:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_nor:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_nor:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_pol:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_pol:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_pol:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_pol:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_por:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_por:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_por:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_por:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_rus:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_rus:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_rus:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_rus:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_srp:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_srp:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_srp:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_srp:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_swe:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_swe:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_swe:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_swe:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_swa:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_swa:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_swa:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_swa:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_tel:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_tel:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_tel:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_tel:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_tha:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_tha:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_tha:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_tha:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_tur:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_tur:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_tur:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_tur:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_ukr:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_ukr:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_ukr:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_ukr:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_urd:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_urd:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_urd:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_urd:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_vie:abstract_algebra|0|0 + extended|global_mmlu_instruct_ca_vie:abstract_algebra|0|0 + extended|global_mmlu_instruct_cs_vie:abstract_algebra|0|0 + extended|global_mmlu_instruct_unk_vie:abstract_algebra|0|0 + extended|global_mmlu_instruct_all_ara:anatomy|0|0 + extended|global_mmlu_instruct_ca_ara:anatomy|0|0 + extended|global_mmlu_instruct_cs_ara:anatomy|0|0 + extended|global_mmlu_instruct_unk_ara:anatomy|0|0 + extended|global_mmlu_instruct_all_ben:anatomy|0|0 + extended|global_mmlu_instruct_ca_ben:anatomy|0|0 + extended|global_mmlu_instruct_cs_ben:anatomy|0|0 + extended|global_mmlu_instruct_unk_ben:anatomy|0|0 + extended|global_mmlu_instruct_all_deu:anatomy|0|0 + extended|global_mmlu_instruct_ca_deu:anatomy|0|0 + extended|global_mmlu_instruct_cs_deu:anatomy|0|0 + extended|global_mmlu_instruct_unk_deu:anatomy|0|0 + extended|global_mmlu_instruct_all_spa:anatomy|0|0 + extended|global_mmlu_instruct_ca_spa:anatomy|0|0 + extended|global_mmlu_instruct_cs_spa:anatomy|0|0 + extended|global_mmlu_instruct_unk_spa:anatomy|0|0 + extended|global_mmlu_instruct_all_eng:anatomy|0|0 + extended|global_mmlu_instruct_ca_eng:anatomy|0|0 + extended|global_mmlu_instruct_cs_eng:anatomy|0|0 + extended|global_mmlu_instruct_unk_eng:anatomy|0|0 + extended|global_mmlu_instruct_all_fra:anatomy|0|0 + extended|global_mmlu_instruct_ca_fra:anatomy|0|0 + extended|global_mmlu_instruct_cs_fra:anatomy|0|0 + extended|global_mmlu_instruct_unk_fra:anatomy|0|0 + extended|global_mmlu_instruct_all_hin:anatomy|0|0 + extended|global_mmlu_instruct_ca_hin:anatomy|0|0 + extended|global_mmlu_instruct_cs_hin:anatomy|0|0 + extended|global_mmlu_instruct_unk_hin:anatomy|0|0 + extended|global_mmlu_instruct_all_ind:anatomy|0|0 + extended|global_mmlu_instruct_ca_ind:anatomy|0|0 + extended|global_mmlu_instruct_cs_ind:anatomy|0|0 + extended|global_mmlu_instruct_unk_ind:anatomy|0|0 + extended|global_mmlu_instruct_all_ita:anatomy|0|0 + extended|global_mmlu_instruct_ca_ita:anatomy|0|0 + extended|global_mmlu_instruct_cs_ita:anatomy|0|0 + extended|global_mmlu_instruct_unk_ita:anatomy|0|0 + extended|global_mmlu_instruct_all_jpn:anatomy|0|0 + extended|global_mmlu_instruct_ca_jpn:anatomy|0|0 + extended|global_mmlu_instruct_cs_jpn:anatomy|0|0 + extended|global_mmlu_instruct_unk_jpn:anatomy|0|0 + extended|global_mmlu_instruct_all_nld:anatomy|0|0 + extended|global_mmlu_instruct_ca_nld:anatomy|0|0 + extended|global_mmlu_instruct_cs_nld:anatomy|0|0 + extended|global_mmlu_instruct_unk_nld:anatomy|0|0 + extended|global_mmlu_instruct_all_nor:anatomy|0|0 + extended|global_mmlu_instruct_ca_nor:anatomy|0|0 + extended|global_mmlu_instruct_cs_nor:anatomy|0|0 + extended|global_mmlu_instruct_unk_nor:anatomy|0|0 + extended|global_mmlu_instruct_all_pol:anatomy|0|0 + extended|global_mmlu_instruct_ca_pol:anatomy|0|0 + extended|global_mmlu_instruct_cs_pol:anatomy|0|0 + extended|global_mmlu_instruct_unk_pol:anatomy|0|0 + extended|global_mmlu_instruct_all_por:anatomy|0|0 + extended|global_mmlu_instruct_ca_por:anatomy|0|0 + extended|global_mmlu_instruct_cs_por:anatomy|0|0 + extended|global_mmlu_instruct_unk_por:anatomy|0|0 + extended|global_mmlu_instruct_all_rus:anatomy|0|0 + extended|global_mmlu_instruct_ca_rus:anatomy|0|0 + extended|global_mmlu_instruct_cs_rus:anatomy|0|0 + extended|global_mmlu_instruct_unk_rus:anatomy|0|0 + extended|global_mmlu_instruct_all_srp:anatomy|0|0 + extended|global_mmlu_instruct_ca_srp:anatomy|0|0 + extended|global_mmlu_instruct_cs_srp:anatomy|0|0 + extended|global_mmlu_instruct_unk_srp:anatomy|0|0 + extended|global_mmlu_instruct_all_swe:anatomy|0|0 + extended|global_mmlu_instruct_ca_swe:anatomy|0|0 + extended|global_mmlu_instruct_cs_swe:anatomy|0|0 + extended|global_mmlu_instruct_unk_swe:anatomy|0|0 + extended|global_mmlu_instruct_all_swa:anatomy|0|0 + extended|global_mmlu_instruct_ca_swa:anatomy|0|0 + extended|global_mmlu_instruct_cs_swa:anatomy|0|0 + extended|global_mmlu_instruct_unk_swa:anatomy|0|0 + extended|global_mmlu_instruct_all_tel:anatomy|0|0 + extended|global_mmlu_instruct_ca_tel:anatomy|0|0 + extended|global_mmlu_instruct_cs_tel:anatomy|0|0 + extended|global_mmlu_instruct_unk_tel:anatomy|0|0 + extended|global_mmlu_instruct_all_tha:anatomy|0|0 + extended|global_mmlu_instruct_ca_tha:anatomy|0|0 + extended|global_mmlu_instruct_cs_tha:anatomy|0|0 + extended|global_mmlu_instruct_unk_tha:anatomy|0|0 + extended|global_mmlu_instruct_all_tur:anatomy|0|0 + extended|global_mmlu_instruct_ca_tur:anatomy|0|0 + extended|global_mmlu_instruct_cs_tur:anatomy|0|0 + extended|global_mmlu_instruct_unk_tur:anatomy|0|0 + extended|global_mmlu_instruct_all_ukr:anatomy|0|0 + extended|global_mmlu_instruct_ca_ukr:anatomy|0|0 + extended|global_mmlu_instruct_cs_ukr:anatomy|0|0 + extended|global_mmlu_instruct_unk_ukr:anatomy|0|0 + extended|global_mmlu_instruct_all_urd:anatomy|0|0 + extended|global_mmlu_instruct_ca_urd:anatomy|0|0 + extended|global_mmlu_instruct_cs_urd:anatomy|0|0 + extended|global_mmlu_instruct_unk_urd:anatomy|0|0 + extended|global_mmlu_instruct_all_vie:anatomy|0|0 + extended|global_mmlu_instruct_ca_vie:anatomy|0|0 + extended|global_mmlu_instruct_cs_vie:anatomy|0|0 + extended|global_mmlu_instruct_unk_vie:anatomy|0|0 + extended|global_mmlu_instruct_all_ara:astronomy|0|0 + extended|global_mmlu_instruct_ca_ara:astronomy|0|0 + extended|global_mmlu_instruct_cs_ara:astronomy|0|0 + extended|global_mmlu_instruct_unk_ara:astronomy|0|0 + extended|global_mmlu_instruct_all_ben:astronomy|0|0 + extended|global_mmlu_instruct_ca_ben:astronomy|0|0 + extended|global_mmlu_instruct_cs_ben:astronomy|0|0 + extended|global_mmlu_instruct_unk_ben:astronomy|0|0 + extended|global_mmlu_instruct_all_deu:astronomy|0|0 + extended|global_mmlu_instruct_ca_deu:astronomy|0|0 + extended|global_mmlu_instruct_cs_deu:astronomy|0|0 + extended|global_mmlu_instruct_unk_deu:astronomy|0|0 + extended|global_mmlu_instruct_all_spa:astronomy|0|0 + extended|global_mmlu_instruct_ca_spa:astronomy|0|0 + extended|global_mmlu_instruct_cs_spa:astronomy|0|0 + extended|global_mmlu_instruct_unk_spa:astronomy|0|0 + extended|global_mmlu_instruct_all_eng:astronomy|0|0 + extended|global_mmlu_instruct_ca_eng:astronomy|0|0 + extended|global_mmlu_instruct_cs_eng:astronomy|0|0 + extended|global_mmlu_instruct_unk_eng:astronomy|0|0 + extended|global_mmlu_instruct_all_fra:astronomy|0|0 + extended|global_mmlu_instruct_ca_fra:astronomy|0|0 + extended|global_mmlu_instruct_cs_fra:astronomy|0|0 + extended|global_mmlu_instruct_unk_fra:astronomy|0|0 + extended|global_mmlu_instruct_all_hin:astronomy|0|0 + extended|global_mmlu_instruct_ca_hin:astronomy|0|0 + extended|global_mmlu_instruct_cs_hin:astronomy|0|0 + extended|global_mmlu_instruct_unk_hin:astronomy|0|0 + extended|global_mmlu_instruct_all_ind:astronomy|0|0 + extended|global_mmlu_instruct_ca_ind:astronomy|0|0 + extended|global_mmlu_instruct_cs_ind:astronomy|0|0 + extended|global_mmlu_instruct_unk_ind:astronomy|0|0 + extended|global_mmlu_instruct_all_ita:astronomy|0|0 + extended|global_mmlu_instruct_ca_ita:astronomy|0|0 + extended|global_mmlu_instruct_cs_ita:astronomy|0|0 + extended|global_mmlu_instruct_unk_ita:astronomy|0|0 + extended|global_mmlu_instruct_all_jpn:astronomy|0|0 + extended|global_mmlu_instruct_ca_jpn:astronomy|0|0 + extended|global_mmlu_instruct_cs_jpn:astronomy|0|0 + extended|global_mmlu_instruct_unk_jpn:astronomy|0|0 + extended|global_mmlu_instruct_all_nld:astronomy|0|0 + extended|global_mmlu_instruct_ca_nld:astronomy|0|0 + extended|global_mmlu_instruct_cs_nld:astronomy|0|0 + extended|global_mmlu_instruct_unk_nld:astronomy|0|0 + extended|global_mmlu_instruct_all_nor:astronomy|0|0 + extended|global_mmlu_instruct_ca_nor:astronomy|0|0 + extended|global_mmlu_instruct_cs_nor:astronomy|0|0 + extended|global_mmlu_instruct_unk_nor:astronomy|0|0 + extended|global_mmlu_instruct_all_pol:astronomy|0|0 + extended|global_mmlu_instruct_ca_pol:astronomy|0|0 + extended|global_mmlu_instruct_cs_pol:astronomy|0|0 + extended|global_mmlu_instruct_unk_pol:astronomy|0|0 + extended|global_mmlu_instruct_all_por:astronomy|0|0 + extended|global_mmlu_instruct_ca_por:astronomy|0|0 + extended|global_mmlu_instruct_cs_por:astronomy|0|0 + extended|global_mmlu_instruct_unk_por:astronomy|0|0 + extended|global_mmlu_instruct_all_rus:astronomy|0|0 + extended|global_mmlu_instruct_ca_rus:astronomy|0|0 + extended|global_mmlu_instruct_cs_rus:astronomy|0|0 + extended|global_mmlu_instruct_unk_rus:astronomy|0|0 + extended|global_mmlu_instruct_all_srp:astronomy|0|0 + extended|global_mmlu_instruct_ca_srp:astronomy|0|0 + extended|global_mmlu_instruct_cs_srp:astronomy|0|0 + extended|global_mmlu_instruct_unk_srp:astronomy|0|0 + extended|global_mmlu_instruct_all_swe:astronomy|0|0 + extended|global_mmlu_instruct_ca_swe:astronomy|0|0 + extended|global_mmlu_instruct_cs_swe:astronomy|0|0 + extended|global_mmlu_instruct_unk_swe:astronomy|0|0 + extended|global_mmlu_instruct_all_swa:astronomy|0|0 + extended|global_mmlu_instruct_ca_swa:astronomy|0|0 + extended|global_mmlu_instruct_cs_swa:astronomy|0|0 + extended|global_mmlu_instruct_unk_swa:astronomy|0|0 + extended|global_mmlu_instruct_all_tel:astronomy|0|0 + extended|global_mmlu_instruct_ca_tel:astronomy|0|0 + extended|global_mmlu_instruct_cs_tel:astronomy|0|0 + extended|global_mmlu_instruct_unk_tel:astronomy|0|0 + extended|global_mmlu_instruct_all_tha:astronomy|0|0 + extended|global_mmlu_instruct_ca_tha:astronomy|0|0 + extended|global_mmlu_instruct_cs_tha:astronomy|0|0 + extended|global_mmlu_instruct_unk_tha:astronomy|0|0 + extended|global_mmlu_instruct_all_tur:astronomy|0|0 + extended|global_mmlu_instruct_ca_tur:astronomy|0|0 + extended|global_mmlu_instruct_cs_tur:astronomy|0|0 + extended|global_mmlu_instruct_unk_tur:astronomy|0|0 + extended|global_mmlu_instruct_all_ukr:astronomy|0|0 + extended|global_mmlu_instruct_ca_ukr:astronomy|0|0 + extended|global_mmlu_instruct_cs_ukr:astronomy|0|0 + extended|global_mmlu_instruct_unk_ukr:astronomy|0|0 + extended|global_mmlu_instruct_all_urd:astronomy|0|0 + extended|global_mmlu_instruct_ca_urd:astronomy|0|0 + extended|global_mmlu_instruct_cs_urd:astronomy|0|0 + extended|global_mmlu_instruct_unk_urd:astronomy|0|0 + extended|global_mmlu_instruct_all_vie:astronomy|0|0 + extended|global_mmlu_instruct_ca_vie:astronomy|0|0 + extended|global_mmlu_instruct_cs_vie:astronomy|0|0 + extended|global_mmlu_instruct_unk_vie:astronomy|0|0 + extended|global_mmlu_instruct_all_ara:business_ethics|0|0 + extended|global_mmlu_instruct_ca_ara:business_ethics|0|0 + extended|global_mmlu_instruct_cs_ara:business_ethics|0|0 + extended|global_mmlu_instruct_unk_ara:business_ethics|0|0 + extended|global_mmlu_instruct_all_ben:business_ethics|0|0 + extended|global_mmlu_instruct_ca_ben:business_ethics|0|0 + extended|global_mmlu_instruct_cs_ben:business_ethics|0|0 + extended|global_mmlu_instruct_unk_ben:business_ethics|0|0 + extended|global_mmlu_instruct_all_deu:business_ethics|0|0 + extended|global_mmlu_instruct_ca_deu:business_ethics|0|0 + extended|global_mmlu_instruct_cs_deu:business_ethics|0|0 + extended|global_mmlu_instruct_unk_deu:business_ethics|0|0 + extended|global_mmlu_instruct_all_spa:business_ethics|0|0 + extended|global_mmlu_instruct_ca_spa:business_ethics|0|0 + extended|global_mmlu_instruct_cs_spa:business_ethics|0|0 + extended|global_mmlu_instruct_unk_spa:business_ethics|0|0 + extended|global_mmlu_instruct_all_eng:business_ethics|0|0 + extended|global_mmlu_instruct_ca_eng:business_ethics|0|0 + extended|global_mmlu_instruct_cs_eng:business_ethics|0|0 + extended|global_mmlu_instruct_unk_eng:business_ethics|0|0 + extended|global_mmlu_instruct_all_fra:business_ethics|0|0 + extended|global_mmlu_instruct_ca_fra:business_ethics|0|0 + extended|global_mmlu_instruct_cs_fra:business_ethics|0|0 + extended|global_mmlu_instruct_unk_fra:business_ethics|0|0 + extended|global_mmlu_instruct_all_hin:business_ethics|0|0 + extended|global_mmlu_instruct_ca_hin:business_ethics|0|0 + extended|global_mmlu_instruct_cs_hin:business_ethics|0|0 + extended|global_mmlu_instruct_unk_hin:business_ethics|0|0 + extended|global_mmlu_instruct_all_ind:business_ethics|0|0 + extended|global_mmlu_instruct_ca_ind:business_ethics|0|0 + extended|global_mmlu_instruct_cs_ind:business_ethics|0|0 + extended|global_mmlu_instruct_unk_ind:business_ethics|0|0 + extended|global_mmlu_instruct_all_ita:business_ethics|0|0 + extended|global_mmlu_instruct_ca_ita:business_ethics|0|0 + extended|global_mmlu_instruct_cs_ita:business_ethics|0|0 + extended|global_mmlu_instruct_unk_ita:business_ethics|0|0 + extended|global_mmlu_instruct_all_jpn:business_ethics|0|0 + extended|global_mmlu_instruct_ca_jpn:business_ethics|0|0 + extended|global_mmlu_instruct_cs_jpn:business_ethics|0|0 + extended|global_mmlu_instruct_unk_jpn:business_ethics|0|0 + extended|global_mmlu_instruct_all_nld:business_ethics|0|0 + extended|global_mmlu_instruct_ca_nld:business_ethics|0|0 + extended|global_mmlu_instruct_cs_nld:business_ethics|0|0 + extended|global_mmlu_instruct_unk_nld:business_ethics|0|0 + extended|global_mmlu_instruct_all_nor:business_ethics|0|0 + extended|global_mmlu_instruct_ca_nor:business_ethics|0|0 + extended|global_mmlu_instruct_cs_nor:business_ethics|0|0 + extended|global_mmlu_instruct_unk_nor:business_ethics|0|0 + extended|global_mmlu_instruct_all_pol:business_ethics|0|0 + extended|global_mmlu_instruct_ca_pol:business_ethics|0|0 + extended|global_mmlu_instruct_cs_pol:business_ethics|0|0 + extended|global_mmlu_instruct_unk_pol:business_ethics|0|0 + extended|global_mmlu_instruct_all_por:business_ethics|0|0 + extended|global_mmlu_instruct_ca_por:business_ethics|0|0 + extended|global_mmlu_instruct_cs_por:business_ethics|0|0 + extended|global_mmlu_instruct_unk_por:business_ethics|0|0 + extended|global_mmlu_instruct_all_rus:business_ethics|0|0 + extended|global_mmlu_instruct_ca_rus:business_ethics|0|0 + extended|global_mmlu_instruct_cs_rus:business_ethics|0|0 + extended|global_mmlu_instruct_unk_rus:business_ethics|0|0 + extended|global_mmlu_instruct_all_srp:business_ethics|0|0 + extended|global_mmlu_instruct_ca_srp:business_ethics|0|0 + extended|global_mmlu_instruct_cs_srp:business_ethics|0|0 + extended|global_mmlu_instruct_unk_srp:business_ethics|0|0 + extended|global_mmlu_instruct_all_swe:business_ethics|0|0 + extended|global_mmlu_instruct_ca_swe:business_ethics|0|0 + extended|global_mmlu_instruct_cs_swe:business_ethics|0|0 + extended|global_mmlu_instruct_unk_swe:business_ethics|0|0 + extended|global_mmlu_instruct_all_swa:business_ethics|0|0 + extended|global_mmlu_instruct_ca_swa:business_ethics|0|0 + extended|global_mmlu_instruct_cs_swa:business_ethics|0|0 + extended|global_mmlu_instruct_unk_swa:business_ethics|0|0 + extended|global_mmlu_instruct_all_tel:business_ethics|0|0 + extended|global_mmlu_instruct_ca_tel:business_ethics|0|0 + extended|global_mmlu_instruct_cs_tel:business_ethics|0|0 + extended|global_mmlu_instruct_unk_tel:business_ethics|0|0 + extended|global_mmlu_instruct_all_tha:business_ethics|0|0 + extended|global_mmlu_instruct_ca_tha:business_ethics|0|0 + extended|global_mmlu_instruct_cs_tha:business_ethics|0|0 + extended|global_mmlu_instruct_unk_tha:business_ethics|0|0 + extended|global_mmlu_instruct_all_tur:business_ethics|0|0 + extended|global_mmlu_instruct_ca_tur:business_ethics|0|0 + extended|global_mmlu_instruct_cs_tur:business_ethics|0|0 + extended|global_mmlu_instruct_unk_tur:business_ethics|0|0 + extended|global_mmlu_instruct_all_ukr:business_ethics|0|0 + extended|global_mmlu_instruct_ca_ukr:business_ethics|0|0 + extended|global_mmlu_instruct_cs_ukr:business_ethics|0|0 + extended|global_mmlu_instruct_unk_ukr:business_ethics|0|0 + extended|global_mmlu_instruct_all_urd:business_ethics|0|0 + extended|global_mmlu_instruct_ca_urd:business_ethics|0|0 + extended|global_mmlu_instruct_cs_urd:business_ethics|0|0 + extended|global_mmlu_instruct_unk_urd:business_ethics|0|0 + extended|global_mmlu_instruct_all_vie:business_ethics|0|0 + extended|global_mmlu_instruct_ca_vie:business_ethics|0|0 + extended|global_mmlu_instruct_cs_vie:business_ethics|0|0 + extended|global_mmlu_instruct_unk_vie:business_ethics|0|0 + extended|global_mmlu_instruct_all_ara:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_ara:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_ara:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_ara:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_ben:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_ben:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_ben:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_ben:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_deu:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_deu:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_deu:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_deu:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_spa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_spa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_spa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_spa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_eng:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_eng:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_eng:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_eng:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_fra:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_fra:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_fra:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_fra:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_hin:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_hin:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_hin:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_hin:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_ind:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_ind:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_ind:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_ind:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_ita:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_ita:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_ita:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_ita:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_jpn:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_jpn:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_jpn:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_jpn:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_nld:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_nld:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_nld:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_nld:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_nor:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_nor:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_nor:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_nor:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_pol:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_pol:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_pol:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_pol:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_por:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_por:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_por:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_por:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_rus:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_rus:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_rus:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_rus:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_srp:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_srp:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_srp:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_srp:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_swe:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_swe:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_swe:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_swe:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_swa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_swa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_swa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_swa:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_tel:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_tel:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_tel:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_tel:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_tha:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_tha:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_tha:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_tha:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_tur:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_tur:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_tur:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_tur:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_ukr:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_ukr:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_ukr:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_ukr:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_urd:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_urd:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_urd:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_urd:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_vie:clinical_knowledge|0|0 + extended|global_mmlu_instruct_ca_vie:clinical_knowledge|0|0 + extended|global_mmlu_instruct_cs_vie:clinical_knowledge|0|0 + extended|global_mmlu_instruct_unk_vie:clinical_knowledge|0|0 + extended|global_mmlu_instruct_all_ara:college_biology|0|0 + extended|global_mmlu_instruct_ca_ara:college_biology|0|0 + extended|global_mmlu_instruct_cs_ara:college_biology|0|0 + extended|global_mmlu_instruct_unk_ara:college_biology|0|0 + extended|global_mmlu_instruct_all_ben:college_biology|0|0 + extended|global_mmlu_instruct_ca_ben:college_biology|0|0 + extended|global_mmlu_instruct_cs_ben:college_biology|0|0 + extended|global_mmlu_instruct_unk_ben:college_biology|0|0 + extended|global_mmlu_instruct_all_deu:college_biology|0|0 + extended|global_mmlu_instruct_ca_deu:college_biology|0|0 + extended|global_mmlu_instruct_cs_deu:college_biology|0|0 + extended|global_mmlu_instruct_unk_deu:college_biology|0|0 + extended|global_mmlu_instruct_all_spa:college_biology|0|0 + extended|global_mmlu_instruct_ca_spa:college_biology|0|0 + extended|global_mmlu_instruct_cs_spa:college_biology|0|0 + extended|global_mmlu_instruct_unk_spa:college_biology|0|0 + extended|global_mmlu_instruct_all_eng:college_biology|0|0 + extended|global_mmlu_instruct_ca_eng:college_biology|0|0 + extended|global_mmlu_instruct_cs_eng:college_biology|0|0 + extended|global_mmlu_instruct_unk_eng:college_biology|0|0 + extended|global_mmlu_instruct_all_fra:college_biology|0|0 + extended|global_mmlu_instruct_ca_fra:college_biology|0|0 + extended|global_mmlu_instruct_cs_fra:college_biology|0|0 + extended|global_mmlu_instruct_unk_fra:college_biology|0|0 + extended|global_mmlu_instruct_all_hin:college_biology|0|0 + extended|global_mmlu_instruct_ca_hin:college_biology|0|0 + extended|global_mmlu_instruct_cs_hin:college_biology|0|0 + extended|global_mmlu_instruct_unk_hin:college_biology|0|0 + extended|global_mmlu_instruct_all_ind:college_biology|0|0 + extended|global_mmlu_instruct_ca_ind:college_biology|0|0 + extended|global_mmlu_instruct_cs_ind:college_biology|0|0 + extended|global_mmlu_instruct_unk_ind:college_biology|0|0 + extended|global_mmlu_instruct_all_ita:college_biology|0|0 + extended|global_mmlu_instruct_ca_ita:college_biology|0|0 + extended|global_mmlu_instruct_cs_ita:college_biology|0|0 + extended|global_mmlu_instruct_unk_ita:college_biology|0|0 + extended|global_mmlu_instruct_all_jpn:college_biology|0|0 + extended|global_mmlu_instruct_ca_jpn:college_biology|0|0 + extended|global_mmlu_instruct_cs_jpn:college_biology|0|0 + extended|global_mmlu_instruct_unk_jpn:college_biology|0|0 + extended|global_mmlu_instruct_all_nld:college_biology|0|0 + extended|global_mmlu_instruct_ca_nld:college_biology|0|0 + extended|global_mmlu_instruct_cs_nld:college_biology|0|0 + extended|global_mmlu_instruct_unk_nld:college_biology|0|0 + extended|global_mmlu_instruct_all_nor:college_biology|0|0 + extended|global_mmlu_instruct_ca_nor:college_biology|0|0 + extended|global_mmlu_instruct_cs_nor:college_biology|0|0 + extended|global_mmlu_instruct_unk_nor:college_biology|0|0 + extended|global_mmlu_instruct_all_pol:college_biology|0|0 + extended|global_mmlu_instruct_ca_pol:college_biology|0|0 + extended|global_mmlu_instruct_cs_pol:college_biology|0|0 + extended|global_mmlu_instruct_unk_pol:college_biology|0|0 + extended|global_mmlu_instruct_all_por:college_biology|0|0 + extended|global_mmlu_instruct_ca_por:college_biology|0|0 + extended|global_mmlu_instruct_cs_por:college_biology|0|0 + extended|global_mmlu_instruct_unk_por:college_biology|0|0 + extended|global_mmlu_instruct_all_rus:college_biology|0|0 + extended|global_mmlu_instruct_ca_rus:college_biology|0|0 + extended|global_mmlu_instruct_cs_rus:college_biology|0|0 + extended|global_mmlu_instruct_unk_rus:college_biology|0|0 + extended|global_mmlu_instruct_all_srp:college_biology|0|0 + extended|global_mmlu_instruct_ca_srp:college_biology|0|0 + extended|global_mmlu_instruct_cs_srp:college_biology|0|0 + extended|global_mmlu_instruct_unk_srp:college_biology|0|0 + extended|global_mmlu_instruct_all_swe:college_biology|0|0 + extended|global_mmlu_instruct_ca_swe:college_biology|0|0 + extended|global_mmlu_instruct_cs_swe:college_biology|0|0 + extended|global_mmlu_instruct_unk_swe:college_biology|0|0 + extended|global_mmlu_instruct_all_swa:college_biology|0|0 + extended|global_mmlu_instruct_ca_swa:college_biology|0|0 + extended|global_mmlu_instruct_cs_swa:college_biology|0|0 + extended|global_mmlu_instruct_unk_swa:college_biology|0|0 + extended|global_mmlu_instruct_all_tel:college_biology|0|0 + extended|global_mmlu_instruct_ca_tel:college_biology|0|0 + extended|global_mmlu_instruct_cs_tel:college_biology|0|0 + extended|global_mmlu_instruct_unk_tel:college_biology|0|0 + extended|global_mmlu_instruct_all_tha:college_biology|0|0 + extended|global_mmlu_instruct_ca_tha:college_biology|0|0 + extended|global_mmlu_instruct_cs_tha:college_biology|0|0 + extended|global_mmlu_instruct_unk_tha:college_biology|0|0 + extended|global_mmlu_instruct_all_tur:college_biology|0|0 + extended|global_mmlu_instruct_ca_tur:college_biology|0|0 + extended|global_mmlu_instruct_cs_tur:college_biology|0|0 + extended|global_mmlu_instruct_unk_tur:college_biology|0|0 + extended|global_mmlu_instruct_all_ukr:college_biology|0|0 + extended|global_mmlu_instruct_ca_ukr:college_biology|0|0 + extended|global_mmlu_instruct_cs_ukr:college_biology|0|0 + extended|global_mmlu_instruct_unk_ukr:college_biology|0|0 + extended|global_mmlu_instruct_all_urd:college_biology|0|0 + extended|global_mmlu_instruct_ca_urd:college_biology|0|0 + extended|global_mmlu_instruct_cs_urd:college_biology|0|0 + extended|global_mmlu_instruct_unk_urd:college_biology|0|0 + extended|global_mmlu_instruct_all_vie:college_biology|0|0 + extended|global_mmlu_instruct_ca_vie:college_biology|0|0 + extended|global_mmlu_instruct_cs_vie:college_biology|0|0 + extended|global_mmlu_instruct_unk_vie:college_biology|0|0 + extended|global_mmlu_instruct_all_ara:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_ara:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_ara:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_ara:college_chemistry|0|0 + extended|global_mmlu_instruct_all_ben:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_ben:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_ben:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_ben:college_chemistry|0|0 + extended|global_mmlu_instruct_all_deu:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_deu:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_deu:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_deu:college_chemistry|0|0 + extended|global_mmlu_instruct_all_spa:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_spa:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_spa:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_spa:college_chemistry|0|0 + extended|global_mmlu_instruct_all_eng:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_eng:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_eng:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_eng:college_chemistry|0|0 + extended|global_mmlu_instruct_all_fra:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_fra:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_fra:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_fra:college_chemistry|0|0 + extended|global_mmlu_instruct_all_hin:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_hin:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_hin:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_hin:college_chemistry|0|0 + extended|global_mmlu_instruct_all_ind:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_ind:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_ind:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_ind:college_chemistry|0|0 + extended|global_mmlu_instruct_all_ita:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_ita:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_ita:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_ita:college_chemistry|0|0 + extended|global_mmlu_instruct_all_jpn:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_jpn:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_jpn:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_jpn:college_chemistry|0|0 + extended|global_mmlu_instruct_all_nld:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_nld:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_nld:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_nld:college_chemistry|0|0 + extended|global_mmlu_instruct_all_nor:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_nor:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_nor:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_nor:college_chemistry|0|0 + extended|global_mmlu_instruct_all_pol:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_pol:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_pol:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_pol:college_chemistry|0|0 + extended|global_mmlu_instruct_all_por:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_por:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_por:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_por:college_chemistry|0|0 + extended|global_mmlu_instruct_all_rus:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_rus:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_rus:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_rus:college_chemistry|0|0 + extended|global_mmlu_instruct_all_srp:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_srp:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_srp:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_srp:college_chemistry|0|0 + extended|global_mmlu_instruct_all_swe:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_swe:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_swe:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_swe:college_chemistry|0|0 + extended|global_mmlu_instruct_all_swa:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_swa:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_swa:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_swa:college_chemistry|0|0 + extended|global_mmlu_instruct_all_tel:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_tel:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_tel:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_tel:college_chemistry|0|0 + extended|global_mmlu_instruct_all_tha:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_tha:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_tha:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_tha:college_chemistry|0|0 + extended|global_mmlu_instruct_all_tur:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_tur:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_tur:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_tur:college_chemistry|0|0 + extended|global_mmlu_instruct_all_ukr:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_ukr:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_ukr:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_ukr:college_chemistry|0|0 + extended|global_mmlu_instruct_all_urd:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_urd:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_urd:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_urd:college_chemistry|0|0 + extended|global_mmlu_instruct_all_vie:college_chemistry|0|0 + extended|global_mmlu_instruct_ca_vie:college_chemistry|0|0 + extended|global_mmlu_instruct_cs_vie:college_chemistry|0|0 + extended|global_mmlu_instruct_unk_vie:college_chemistry|0|0 + extended|global_mmlu_instruct_all_ara:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_ara:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_ara:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_ara:college_computer_science|0|0 + extended|global_mmlu_instruct_all_ben:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_ben:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_ben:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_ben:college_computer_science|0|0 + extended|global_mmlu_instruct_all_deu:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_deu:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_deu:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_deu:college_computer_science|0|0 + extended|global_mmlu_instruct_all_spa:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_spa:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_spa:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_spa:college_computer_science|0|0 + extended|global_mmlu_instruct_all_eng:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_eng:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_eng:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_eng:college_computer_science|0|0 + extended|global_mmlu_instruct_all_fra:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_fra:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_fra:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_fra:college_computer_science|0|0 + extended|global_mmlu_instruct_all_hin:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_hin:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_hin:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_hin:college_computer_science|0|0 + extended|global_mmlu_instruct_all_ind:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_ind:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_ind:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_ind:college_computer_science|0|0 + extended|global_mmlu_instruct_all_ita:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_ita:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_ita:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_ita:college_computer_science|0|0 + extended|global_mmlu_instruct_all_jpn:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_jpn:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_jpn:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_jpn:college_computer_science|0|0 + extended|global_mmlu_instruct_all_nld:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_nld:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_nld:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_nld:college_computer_science|0|0 + extended|global_mmlu_instruct_all_nor:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_nor:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_nor:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_nor:college_computer_science|0|0 + extended|global_mmlu_instruct_all_pol:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_pol:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_pol:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_pol:college_computer_science|0|0 + extended|global_mmlu_instruct_all_por:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_por:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_por:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_por:college_computer_science|0|0 + extended|global_mmlu_instruct_all_rus:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_rus:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_rus:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_rus:college_computer_science|0|0 + extended|global_mmlu_instruct_all_srp:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_srp:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_srp:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_srp:college_computer_science|0|0 + extended|global_mmlu_instruct_all_swe:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_swe:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_swe:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_swe:college_computer_science|0|0 + extended|global_mmlu_instruct_all_swa:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_swa:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_swa:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_swa:college_computer_science|0|0 + extended|global_mmlu_instruct_all_tel:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_tel:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_tel:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_tel:college_computer_science|0|0 + extended|global_mmlu_instruct_all_tha:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_tha:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_tha:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_tha:college_computer_science|0|0 + extended|global_mmlu_instruct_all_tur:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_tur:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_tur:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_tur:college_computer_science|0|0 + extended|global_mmlu_instruct_all_ukr:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_ukr:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_ukr:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_ukr:college_computer_science|0|0 + extended|global_mmlu_instruct_all_urd:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_urd:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_urd:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_urd:college_computer_science|0|0 + extended|global_mmlu_instruct_all_vie:college_computer_science|0|0 + extended|global_mmlu_instruct_ca_vie:college_computer_science|0|0 + extended|global_mmlu_instruct_cs_vie:college_computer_science|0|0 + extended|global_mmlu_instruct_unk_vie:college_computer_science|0|0 + extended|global_mmlu_instruct_all_ara:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_ara:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_ara:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_ara:college_mathematics|0|0 + extended|global_mmlu_instruct_all_ben:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_ben:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_ben:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_ben:college_mathematics|0|0 + extended|global_mmlu_instruct_all_deu:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_deu:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_deu:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_deu:college_mathematics|0|0 + extended|global_mmlu_instruct_all_spa:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_spa:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_spa:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_spa:college_mathematics|0|0 + extended|global_mmlu_instruct_all_eng:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_eng:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_eng:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_eng:college_mathematics|0|0 + extended|global_mmlu_instruct_all_fra:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_fra:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_fra:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_fra:college_mathematics|0|0 + extended|global_mmlu_instruct_all_hin:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_hin:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_hin:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_hin:college_mathematics|0|0 + extended|global_mmlu_instruct_all_ind:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_ind:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_ind:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_ind:college_mathematics|0|0 + extended|global_mmlu_instruct_all_ita:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_ita:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_ita:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_ita:college_mathematics|0|0 + extended|global_mmlu_instruct_all_jpn:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_jpn:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_jpn:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_jpn:college_mathematics|0|0 + extended|global_mmlu_instruct_all_nld:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_nld:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_nld:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_nld:college_mathematics|0|0 + extended|global_mmlu_instruct_all_nor:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_nor:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_nor:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_nor:college_mathematics|0|0 + extended|global_mmlu_instruct_all_pol:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_pol:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_pol:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_pol:college_mathematics|0|0 + extended|global_mmlu_instruct_all_por:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_por:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_por:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_por:college_mathematics|0|0 + extended|global_mmlu_instruct_all_rus:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_rus:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_rus:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_rus:college_mathematics|0|0 + extended|global_mmlu_instruct_all_srp:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_srp:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_srp:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_srp:college_mathematics|0|0 + extended|global_mmlu_instruct_all_swe:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_swe:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_swe:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_swe:college_mathematics|0|0 + extended|global_mmlu_instruct_all_swa:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_swa:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_swa:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_swa:college_mathematics|0|0 + extended|global_mmlu_instruct_all_tel:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_tel:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_tel:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_tel:college_mathematics|0|0 + extended|global_mmlu_instruct_all_tha:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_tha:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_tha:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_tha:college_mathematics|0|0 + extended|global_mmlu_instruct_all_tur:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_tur:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_tur:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_tur:college_mathematics|0|0 + extended|global_mmlu_instruct_all_ukr:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_ukr:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_ukr:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_ukr:college_mathematics|0|0 + extended|global_mmlu_instruct_all_urd:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_urd:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_urd:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_urd:college_mathematics|0|0 + extended|global_mmlu_instruct_all_vie:college_mathematics|0|0 + extended|global_mmlu_instruct_ca_vie:college_mathematics|0|0 + extended|global_mmlu_instruct_cs_vie:college_mathematics|0|0 + extended|global_mmlu_instruct_unk_vie:college_mathematics|0|0 + extended|global_mmlu_instruct_all_ara:college_medicine|0|0 + extended|global_mmlu_instruct_ca_ara:college_medicine|0|0 + extended|global_mmlu_instruct_cs_ara:college_medicine|0|0 + extended|global_mmlu_instruct_unk_ara:college_medicine|0|0 + extended|global_mmlu_instruct_all_ben:college_medicine|0|0 + extended|global_mmlu_instruct_ca_ben:college_medicine|0|0 + extended|global_mmlu_instruct_cs_ben:college_medicine|0|0 + extended|global_mmlu_instruct_unk_ben:college_medicine|0|0 + extended|global_mmlu_instruct_all_deu:college_medicine|0|0 + extended|global_mmlu_instruct_ca_deu:college_medicine|0|0 + extended|global_mmlu_instruct_cs_deu:college_medicine|0|0 + extended|global_mmlu_instruct_unk_deu:college_medicine|0|0 + extended|global_mmlu_instruct_all_spa:college_medicine|0|0 + extended|global_mmlu_instruct_ca_spa:college_medicine|0|0 + extended|global_mmlu_instruct_cs_spa:college_medicine|0|0 + extended|global_mmlu_instruct_unk_spa:college_medicine|0|0 + extended|global_mmlu_instruct_all_eng:college_medicine|0|0 + extended|global_mmlu_instruct_ca_eng:college_medicine|0|0 + extended|global_mmlu_instruct_cs_eng:college_medicine|0|0 + extended|global_mmlu_instruct_unk_eng:college_medicine|0|0 + extended|global_mmlu_instruct_all_fra:college_medicine|0|0 + extended|global_mmlu_instruct_ca_fra:college_medicine|0|0 + extended|global_mmlu_instruct_cs_fra:college_medicine|0|0 + extended|global_mmlu_instruct_unk_fra:college_medicine|0|0 + extended|global_mmlu_instruct_all_hin:college_medicine|0|0 + extended|global_mmlu_instruct_ca_hin:college_medicine|0|0 + extended|global_mmlu_instruct_cs_hin:college_medicine|0|0 + extended|global_mmlu_instruct_unk_hin:college_medicine|0|0 + extended|global_mmlu_instruct_all_ind:college_medicine|0|0 + extended|global_mmlu_instruct_ca_ind:college_medicine|0|0 + extended|global_mmlu_instruct_cs_ind:college_medicine|0|0 + extended|global_mmlu_instruct_unk_ind:college_medicine|0|0 + extended|global_mmlu_instruct_all_ita:college_medicine|0|0 + extended|global_mmlu_instruct_ca_ita:college_medicine|0|0 + extended|global_mmlu_instruct_cs_ita:college_medicine|0|0 + extended|global_mmlu_instruct_unk_ita:college_medicine|0|0 + extended|global_mmlu_instruct_all_jpn:college_medicine|0|0 + extended|global_mmlu_instruct_ca_jpn:college_medicine|0|0 + extended|global_mmlu_instruct_cs_jpn:college_medicine|0|0 + extended|global_mmlu_instruct_unk_jpn:college_medicine|0|0 + extended|global_mmlu_instruct_all_nld:college_medicine|0|0 + extended|global_mmlu_instruct_ca_nld:college_medicine|0|0 + extended|global_mmlu_instruct_cs_nld:college_medicine|0|0 + extended|global_mmlu_instruct_unk_nld:college_medicine|0|0 + extended|global_mmlu_instruct_all_nor:college_medicine|0|0 + extended|global_mmlu_instruct_ca_nor:college_medicine|0|0 + extended|global_mmlu_instruct_cs_nor:college_medicine|0|0 + extended|global_mmlu_instruct_unk_nor:college_medicine|0|0 + extended|global_mmlu_instruct_all_pol:college_medicine|0|0 + extended|global_mmlu_instruct_ca_pol:college_medicine|0|0 + extended|global_mmlu_instruct_cs_pol:college_medicine|0|0 + extended|global_mmlu_instruct_unk_pol:college_medicine|0|0 + extended|global_mmlu_instruct_all_por:college_medicine|0|0 + extended|global_mmlu_instruct_ca_por:college_medicine|0|0 + extended|global_mmlu_instruct_cs_por:college_medicine|0|0 + extended|global_mmlu_instruct_unk_por:college_medicine|0|0 + extended|global_mmlu_instruct_all_rus:college_medicine|0|0 + extended|global_mmlu_instruct_ca_rus:college_medicine|0|0 + extended|global_mmlu_instruct_cs_rus:college_medicine|0|0 + extended|global_mmlu_instruct_unk_rus:college_medicine|0|0 + extended|global_mmlu_instruct_all_srp:college_medicine|0|0 + extended|global_mmlu_instruct_ca_srp:college_medicine|0|0 + extended|global_mmlu_instruct_cs_srp:college_medicine|0|0 + extended|global_mmlu_instruct_unk_srp:college_medicine|0|0 + extended|global_mmlu_instruct_all_swe:college_medicine|0|0 + extended|global_mmlu_instruct_ca_swe:college_medicine|0|0 + extended|global_mmlu_instruct_cs_swe:college_medicine|0|0 + extended|global_mmlu_instruct_unk_swe:college_medicine|0|0 + extended|global_mmlu_instruct_all_swa:college_medicine|0|0 + extended|global_mmlu_instruct_ca_swa:college_medicine|0|0 + extended|global_mmlu_instruct_cs_swa:college_medicine|0|0 + extended|global_mmlu_instruct_unk_swa:college_medicine|0|0 + extended|global_mmlu_instruct_all_tel:college_medicine|0|0 + extended|global_mmlu_instruct_ca_tel:college_medicine|0|0 + extended|global_mmlu_instruct_cs_tel:college_medicine|0|0 + extended|global_mmlu_instruct_unk_tel:college_medicine|0|0 + extended|global_mmlu_instruct_all_tha:college_medicine|0|0 + extended|global_mmlu_instruct_ca_tha:college_medicine|0|0 + extended|global_mmlu_instruct_cs_tha:college_medicine|0|0 + extended|global_mmlu_instruct_unk_tha:college_medicine|0|0 + extended|global_mmlu_instruct_all_tur:college_medicine|0|0 + extended|global_mmlu_instruct_ca_tur:college_medicine|0|0 + extended|global_mmlu_instruct_cs_tur:college_medicine|0|0 + extended|global_mmlu_instruct_unk_tur:college_medicine|0|0 + extended|global_mmlu_instruct_all_ukr:college_medicine|0|0 + extended|global_mmlu_instruct_ca_ukr:college_medicine|0|0 + extended|global_mmlu_instruct_cs_ukr:college_medicine|0|0 + extended|global_mmlu_instruct_unk_ukr:college_medicine|0|0 + extended|global_mmlu_instruct_all_urd:college_medicine|0|0 + extended|global_mmlu_instruct_ca_urd:college_medicine|0|0 + extended|global_mmlu_instruct_cs_urd:college_medicine|0|0 + extended|global_mmlu_instruct_unk_urd:college_medicine|0|0 + extended|global_mmlu_instruct_all_vie:college_medicine|0|0 + extended|global_mmlu_instruct_ca_vie:college_medicine|0|0 + extended|global_mmlu_instruct_cs_vie:college_medicine|0|0 + extended|global_mmlu_instruct_unk_vie:college_medicine|0|0 + extended|global_mmlu_instruct_all_ara:college_physics|0|0 + extended|global_mmlu_instruct_ca_ara:college_physics|0|0 + extended|global_mmlu_instruct_cs_ara:college_physics|0|0 + extended|global_mmlu_instruct_unk_ara:college_physics|0|0 + extended|global_mmlu_instruct_all_ben:college_physics|0|0 + extended|global_mmlu_instruct_ca_ben:college_physics|0|0 + extended|global_mmlu_instruct_cs_ben:college_physics|0|0 + extended|global_mmlu_instruct_unk_ben:college_physics|0|0 + extended|global_mmlu_instruct_all_deu:college_physics|0|0 + extended|global_mmlu_instruct_ca_deu:college_physics|0|0 + extended|global_mmlu_instruct_cs_deu:college_physics|0|0 + extended|global_mmlu_instruct_unk_deu:college_physics|0|0 + extended|global_mmlu_instruct_all_spa:college_physics|0|0 + extended|global_mmlu_instruct_ca_spa:college_physics|0|0 + extended|global_mmlu_instruct_cs_spa:college_physics|0|0 + extended|global_mmlu_instruct_unk_spa:college_physics|0|0 + extended|global_mmlu_instruct_all_eng:college_physics|0|0 + extended|global_mmlu_instruct_ca_eng:college_physics|0|0 + extended|global_mmlu_instruct_cs_eng:college_physics|0|0 + extended|global_mmlu_instruct_unk_eng:college_physics|0|0 + extended|global_mmlu_instruct_all_fra:college_physics|0|0 + extended|global_mmlu_instruct_ca_fra:college_physics|0|0 + extended|global_mmlu_instruct_cs_fra:college_physics|0|0 + extended|global_mmlu_instruct_unk_fra:college_physics|0|0 + extended|global_mmlu_instruct_all_hin:college_physics|0|0 + extended|global_mmlu_instruct_ca_hin:college_physics|0|0 + extended|global_mmlu_instruct_cs_hin:college_physics|0|0 + extended|global_mmlu_instruct_unk_hin:college_physics|0|0 + extended|global_mmlu_instruct_all_ind:college_physics|0|0 + extended|global_mmlu_instruct_ca_ind:college_physics|0|0 + extended|global_mmlu_instruct_cs_ind:college_physics|0|0 + extended|global_mmlu_instruct_unk_ind:college_physics|0|0 + extended|global_mmlu_instruct_all_ita:college_physics|0|0 + extended|global_mmlu_instruct_ca_ita:college_physics|0|0 + extended|global_mmlu_instruct_cs_ita:college_physics|0|0 + extended|global_mmlu_instruct_unk_ita:college_physics|0|0 + extended|global_mmlu_instruct_all_jpn:college_physics|0|0 + extended|global_mmlu_instruct_ca_jpn:college_physics|0|0 + extended|global_mmlu_instruct_cs_jpn:college_physics|0|0 + extended|global_mmlu_instruct_unk_jpn:college_physics|0|0 + extended|global_mmlu_instruct_all_nld:college_physics|0|0 + extended|global_mmlu_instruct_ca_nld:college_physics|0|0 + extended|global_mmlu_instruct_cs_nld:college_physics|0|0 + extended|global_mmlu_instruct_unk_nld:college_physics|0|0 + extended|global_mmlu_instruct_all_nor:college_physics|0|0 + extended|global_mmlu_instruct_ca_nor:college_physics|0|0 + extended|global_mmlu_instruct_cs_nor:college_physics|0|0 + extended|global_mmlu_instruct_unk_nor:college_physics|0|0 + extended|global_mmlu_instruct_all_pol:college_physics|0|0 + extended|global_mmlu_instruct_ca_pol:college_physics|0|0 + extended|global_mmlu_instruct_cs_pol:college_physics|0|0 + extended|global_mmlu_instruct_unk_pol:college_physics|0|0 + extended|global_mmlu_instruct_all_por:college_physics|0|0 + extended|global_mmlu_instruct_ca_por:college_physics|0|0 + extended|global_mmlu_instruct_cs_por:college_physics|0|0 + extended|global_mmlu_instruct_unk_por:college_physics|0|0 + extended|global_mmlu_instruct_all_rus:college_physics|0|0 + extended|global_mmlu_instruct_ca_rus:college_physics|0|0 + extended|global_mmlu_instruct_cs_rus:college_physics|0|0 + extended|global_mmlu_instruct_unk_rus:college_physics|0|0 + extended|global_mmlu_instruct_all_srp:college_physics|0|0 + extended|global_mmlu_instruct_ca_srp:college_physics|0|0 + extended|global_mmlu_instruct_cs_srp:college_physics|0|0 + extended|global_mmlu_instruct_unk_srp:college_physics|0|0 + extended|global_mmlu_instruct_all_swe:college_physics|0|0 + extended|global_mmlu_instruct_ca_swe:college_physics|0|0 + extended|global_mmlu_instruct_cs_swe:college_physics|0|0 + extended|global_mmlu_instruct_unk_swe:college_physics|0|0 + extended|global_mmlu_instruct_all_swa:college_physics|0|0 + extended|global_mmlu_instruct_ca_swa:college_physics|0|0 + extended|global_mmlu_instruct_cs_swa:college_physics|0|0 + extended|global_mmlu_instruct_unk_swa:college_physics|0|0 + extended|global_mmlu_instruct_all_tel:college_physics|0|0 + extended|global_mmlu_instruct_ca_tel:college_physics|0|0 + extended|global_mmlu_instruct_cs_tel:college_physics|0|0 + extended|global_mmlu_instruct_unk_tel:college_physics|0|0 + extended|global_mmlu_instruct_all_tha:college_physics|0|0 + extended|global_mmlu_instruct_ca_tha:college_physics|0|0 + extended|global_mmlu_instruct_cs_tha:college_physics|0|0 + extended|global_mmlu_instruct_unk_tha:college_physics|0|0 + extended|global_mmlu_instruct_all_tur:college_physics|0|0 + extended|global_mmlu_instruct_ca_tur:college_physics|0|0 + extended|global_mmlu_instruct_cs_tur:college_physics|0|0 + extended|global_mmlu_instruct_unk_tur:college_physics|0|0 + extended|global_mmlu_instruct_all_ukr:college_physics|0|0 + extended|global_mmlu_instruct_ca_ukr:college_physics|0|0 + extended|global_mmlu_instruct_cs_ukr:college_physics|0|0 + extended|global_mmlu_instruct_unk_ukr:college_physics|0|0 + extended|global_mmlu_instruct_all_urd:college_physics|0|0 + extended|global_mmlu_instruct_ca_urd:college_physics|0|0 + extended|global_mmlu_instruct_cs_urd:college_physics|0|0 + extended|global_mmlu_instruct_unk_urd:college_physics|0|0 + extended|global_mmlu_instruct_all_vie:college_physics|0|0 + extended|global_mmlu_instruct_ca_vie:college_physics|0|0 + extended|global_mmlu_instruct_cs_vie:college_physics|0|0 + extended|global_mmlu_instruct_unk_vie:college_physics|0|0 + extended|global_mmlu_instruct_all_ara:computer_security|0|0 + extended|global_mmlu_instruct_ca_ara:computer_security|0|0 + extended|global_mmlu_instruct_cs_ara:computer_security|0|0 + extended|global_mmlu_instruct_unk_ara:computer_security|0|0 + extended|global_mmlu_instruct_all_ben:computer_security|0|0 + extended|global_mmlu_instruct_ca_ben:computer_security|0|0 + extended|global_mmlu_instruct_cs_ben:computer_security|0|0 + extended|global_mmlu_instruct_unk_ben:computer_security|0|0 + extended|global_mmlu_instruct_all_deu:computer_security|0|0 + extended|global_mmlu_instruct_ca_deu:computer_security|0|0 + extended|global_mmlu_instruct_cs_deu:computer_security|0|0 + extended|global_mmlu_instruct_unk_deu:computer_security|0|0 + extended|global_mmlu_instruct_all_spa:computer_security|0|0 + extended|global_mmlu_instruct_ca_spa:computer_security|0|0 + extended|global_mmlu_instruct_cs_spa:computer_security|0|0 + extended|global_mmlu_instruct_unk_spa:computer_security|0|0 + extended|global_mmlu_instruct_all_eng:computer_security|0|0 + extended|global_mmlu_instruct_ca_eng:computer_security|0|0 + extended|global_mmlu_instruct_cs_eng:computer_security|0|0 + extended|global_mmlu_instruct_unk_eng:computer_security|0|0 + extended|global_mmlu_instruct_all_fra:computer_security|0|0 + extended|global_mmlu_instruct_ca_fra:computer_security|0|0 + extended|global_mmlu_instruct_cs_fra:computer_security|0|0 + extended|global_mmlu_instruct_unk_fra:computer_security|0|0 + extended|global_mmlu_instruct_all_hin:computer_security|0|0 + extended|global_mmlu_instruct_ca_hin:computer_security|0|0 + extended|global_mmlu_instruct_cs_hin:computer_security|0|0 + extended|global_mmlu_instruct_unk_hin:computer_security|0|0 + extended|global_mmlu_instruct_all_ind:computer_security|0|0 + extended|global_mmlu_instruct_ca_ind:computer_security|0|0 + extended|global_mmlu_instruct_cs_ind:computer_security|0|0 + extended|global_mmlu_instruct_unk_ind:computer_security|0|0 + extended|global_mmlu_instruct_all_ita:computer_security|0|0 + extended|global_mmlu_instruct_ca_ita:computer_security|0|0 + extended|global_mmlu_instruct_cs_ita:computer_security|0|0 + extended|global_mmlu_instruct_unk_ita:computer_security|0|0 + extended|global_mmlu_instruct_all_jpn:computer_security|0|0 + extended|global_mmlu_instruct_ca_jpn:computer_security|0|0 + extended|global_mmlu_instruct_cs_jpn:computer_security|0|0 + extended|global_mmlu_instruct_unk_jpn:computer_security|0|0 + extended|global_mmlu_instruct_all_nld:computer_security|0|0 + extended|global_mmlu_instruct_ca_nld:computer_security|0|0 + extended|global_mmlu_instruct_cs_nld:computer_security|0|0 + extended|global_mmlu_instruct_unk_nld:computer_security|0|0 + extended|global_mmlu_instruct_all_nor:computer_security|0|0 + extended|global_mmlu_instruct_ca_nor:computer_security|0|0 + extended|global_mmlu_instruct_cs_nor:computer_security|0|0 + extended|global_mmlu_instruct_unk_nor:computer_security|0|0 + extended|global_mmlu_instruct_all_pol:computer_security|0|0 + extended|global_mmlu_instruct_ca_pol:computer_security|0|0 + extended|global_mmlu_instruct_cs_pol:computer_security|0|0 + extended|global_mmlu_instruct_unk_pol:computer_security|0|0 + extended|global_mmlu_instruct_all_por:computer_security|0|0 + extended|global_mmlu_instruct_ca_por:computer_security|0|0 + extended|global_mmlu_instruct_cs_por:computer_security|0|0 + extended|global_mmlu_instruct_unk_por:computer_security|0|0 + extended|global_mmlu_instruct_all_rus:computer_security|0|0 + extended|global_mmlu_instruct_ca_rus:computer_security|0|0 + extended|global_mmlu_instruct_cs_rus:computer_security|0|0 + extended|global_mmlu_instruct_unk_rus:computer_security|0|0 + extended|global_mmlu_instruct_all_srp:computer_security|0|0 + extended|global_mmlu_instruct_ca_srp:computer_security|0|0 + extended|global_mmlu_instruct_cs_srp:computer_security|0|0 + extended|global_mmlu_instruct_unk_srp:computer_security|0|0 + extended|global_mmlu_instruct_all_swe:computer_security|0|0 + extended|global_mmlu_instruct_ca_swe:computer_security|0|0 + extended|global_mmlu_instruct_cs_swe:computer_security|0|0 + extended|global_mmlu_instruct_unk_swe:computer_security|0|0 + extended|global_mmlu_instruct_all_swa:computer_security|0|0 + extended|global_mmlu_instruct_ca_swa:computer_security|0|0 + extended|global_mmlu_instruct_cs_swa:computer_security|0|0 + extended|global_mmlu_instruct_unk_swa:computer_security|0|0 + extended|global_mmlu_instruct_all_tel:computer_security|0|0 + extended|global_mmlu_instruct_ca_tel:computer_security|0|0 + extended|global_mmlu_instruct_cs_tel:computer_security|0|0 + extended|global_mmlu_instruct_unk_tel:computer_security|0|0 + extended|global_mmlu_instruct_all_tha:computer_security|0|0 + extended|global_mmlu_instruct_ca_tha:computer_security|0|0 + extended|global_mmlu_instruct_cs_tha:computer_security|0|0 + extended|global_mmlu_instruct_unk_tha:computer_security|0|0 + extended|global_mmlu_instruct_all_tur:computer_security|0|0 + extended|global_mmlu_instruct_ca_tur:computer_security|0|0 + extended|global_mmlu_instruct_cs_tur:computer_security|0|0 + extended|global_mmlu_instruct_unk_tur:computer_security|0|0 + extended|global_mmlu_instruct_all_ukr:computer_security|0|0 + extended|global_mmlu_instruct_ca_ukr:computer_security|0|0 + extended|global_mmlu_instruct_cs_ukr:computer_security|0|0 + extended|global_mmlu_instruct_unk_ukr:computer_security|0|0 + extended|global_mmlu_instruct_all_urd:computer_security|0|0 + extended|global_mmlu_instruct_ca_urd:computer_security|0|0 + extended|global_mmlu_instruct_cs_urd:computer_security|0|0 + extended|global_mmlu_instruct_unk_urd:computer_security|0|0 + extended|global_mmlu_instruct_all_vie:computer_security|0|0 + extended|global_mmlu_instruct_ca_vie:computer_security|0|0 + extended|global_mmlu_instruct_cs_vie:computer_security|0|0 + extended|global_mmlu_instruct_unk_vie:computer_security|0|0 + extended|global_mmlu_instruct_all_ara:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_ara:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_ara:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_ara:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_ben:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_ben:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_ben:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_ben:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_deu:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_deu:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_deu:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_deu:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_spa:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_spa:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_spa:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_spa:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_eng:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_eng:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_eng:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_eng:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_fra:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_fra:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_fra:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_fra:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_hin:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_hin:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_hin:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_hin:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_ind:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_ind:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_ind:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_ind:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_ita:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_ita:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_ita:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_ita:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_jpn:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_jpn:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_jpn:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_jpn:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_nld:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_nld:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_nld:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_nld:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_nor:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_nor:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_nor:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_nor:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_pol:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_pol:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_pol:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_pol:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_por:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_por:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_por:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_por:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_rus:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_rus:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_rus:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_rus:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_srp:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_srp:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_srp:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_srp:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_swe:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_swe:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_swe:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_swe:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_swa:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_swa:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_swa:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_swa:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_tel:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_tel:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_tel:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_tel:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_tha:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_tha:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_tha:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_tha:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_tur:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_tur:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_tur:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_tur:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_ukr:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_ukr:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_ukr:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_ukr:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_urd:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_urd:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_urd:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_urd:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_vie:conceptual_physics|0|0 + extended|global_mmlu_instruct_ca_vie:conceptual_physics|0|0 + extended|global_mmlu_instruct_cs_vie:conceptual_physics|0|0 + extended|global_mmlu_instruct_unk_vie:conceptual_physics|0|0 + extended|global_mmlu_instruct_all_ara:econometrics|0|0 + extended|global_mmlu_instruct_ca_ara:econometrics|0|0 + extended|global_mmlu_instruct_cs_ara:econometrics|0|0 + extended|global_mmlu_instruct_unk_ara:econometrics|0|0 + extended|global_mmlu_instruct_all_ben:econometrics|0|0 + extended|global_mmlu_instruct_ca_ben:econometrics|0|0 + extended|global_mmlu_instruct_cs_ben:econometrics|0|0 + extended|global_mmlu_instruct_unk_ben:econometrics|0|0 + extended|global_mmlu_instruct_all_deu:econometrics|0|0 + extended|global_mmlu_instruct_ca_deu:econometrics|0|0 + extended|global_mmlu_instruct_cs_deu:econometrics|0|0 + extended|global_mmlu_instruct_unk_deu:econometrics|0|0 + extended|global_mmlu_instruct_all_spa:econometrics|0|0 + extended|global_mmlu_instruct_ca_spa:econometrics|0|0 + extended|global_mmlu_instruct_cs_spa:econometrics|0|0 + extended|global_mmlu_instruct_unk_spa:econometrics|0|0 + extended|global_mmlu_instruct_all_eng:econometrics|0|0 + extended|global_mmlu_instruct_ca_eng:econometrics|0|0 + extended|global_mmlu_instruct_cs_eng:econometrics|0|0 + extended|global_mmlu_instruct_unk_eng:econometrics|0|0 + extended|global_mmlu_instruct_all_fra:econometrics|0|0 + extended|global_mmlu_instruct_ca_fra:econometrics|0|0 + extended|global_mmlu_instruct_cs_fra:econometrics|0|0 + extended|global_mmlu_instruct_unk_fra:econometrics|0|0 + extended|global_mmlu_instruct_all_hin:econometrics|0|0 + extended|global_mmlu_instruct_ca_hin:econometrics|0|0 + extended|global_mmlu_instruct_cs_hin:econometrics|0|0 + extended|global_mmlu_instruct_unk_hin:econometrics|0|0 + extended|global_mmlu_instruct_all_ind:econometrics|0|0 + extended|global_mmlu_instruct_ca_ind:econometrics|0|0 + extended|global_mmlu_instruct_cs_ind:econometrics|0|0 + extended|global_mmlu_instruct_unk_ind:econometrics|0|0 + extended|global_mmlu_instruct_all_ita:econometrics|0|0 + extended|global_mmlu_instruct_ca_ita:econometrics|0|0 + extended|global_mmlu_instruct_cs_ita:econometrics|0|0 + extended|global_mmlu_instruct_unk_ita:econometrics|0|0 + extended|global_mmlu_instruct_all_jpn:econometrics|0|0 + extended|global_mmlu_instruct_ca_jpn:econometrics|0|0 + extended|global_mmlu_instruct_cs_jpn:econometrics|0|0 + extended|global_mmlu_instruct_unk_jpn:econometrics|0|0 + extended|global_mmlu_instruct_all_nld:econometrics|0|0 + extended|global_mmlu_instruct_ca_nld:econometrics|0|0 + extended|global_mmlu_instruct_cs_nld:econometrics|0|0 + extended|global_mmlu_instruct_unk_nld:econometrics|0|0 + extended|global_mmlu_instruct_all_nor:econometrics|0|0 + extended|global_mmlu_instruct_ca_nor:econometrics|0|0 + extended|global_mmlu_instruct_cs_nor:econometrics|0|0 + extended|global_mmlu_instruct_unk_nor:econometrics|0|0 + extended|global_mmlu_instruct_all_pol:econometrics|0|0 + extended|global_mmlu_instruct_ca_pol:econometrics|0|0 + extended|global_mmlu_instruct_cs_pol:econometrics|0|0 + extended|global_mmlu_instruct_unk_pol:econometrics|0|0 + extended|global_mmlu_instruct_all_por:econometrics|0|0 + extended|global_mmlu_instruct_ca_por:econometrics|0|0 + extended|global_mmlu_instruct_cs_por:econometrics|0|0 + extended|global_mmlu_instruct_unk_por:econometrics|0|0 + extended|global_mmlu_instruct_all_rus:econometrics|0|0 + extended|global_mmlu_instruct_ca_rus:econometrics|0|0 + extended|global_mmlu_instruct_cs_rus:econometrics|0|0 + extended|global_mmlu_instruct_unk_rus:econometrics|0|0 + extended|global_mmlu_instruct_all_srp:econometrics|0|0 + extended|global_mmlu_instruct_ca_srp:econometrics|0|0 + extended|global_mmlu_instruct_cs_srp:econometrics|0|0 + extended|global_mmlu_instruct_unk_srp:econometrics|0|0 + extended|global_mmlu_instruct_all_swe:econometrics|0|0 + extended|global_mmlu_instruct_ca_swe:econometrics|0|0 + extended|global_mmlu_instruct_cs_swe:econometrics|0|0 + extended|global_mmlu_instruct_unk_swe:econometrics|0|0 + extended|global_mmlu_instruct_all_swa:econometrics|0|0 + extended|global_mmlu_instruct_ca_swa:econometrics|0|0 + extended|global_mmlu_instruct_cs_swa:econometrics|0|0 + extended|global_mmlu_instruct_unk_swa:econometrics|0|0 + extended|global_mmlu_instruct_all_tel:econometrics|0|0 + extended|global_mmlu_instruct_ca_tel:econometrics|0|0 + extended|global_mmlu_instruct_cs_tel:econometrics|0|0 + extended|global_mmlu_instruct_unk_tel:econometrics|0|0 + extended|global_mmlu_instruct_all_tha:econometrics|0|0 + extended|global_mmlu_instruct_ca_tha:econometrics|0|0 + extended|global_mmlu_instruct_cs_tha:econometrics|0|0 + extended|global_mmlu_instruct_unk_tha:econometrics|0|0 + extended|global_mmlu_instruct_all_tur:econometrics|0|0 + extended|global_mmlu_instruct_ca_tur:econometrics|0|0 + extended|global_mmlu_instruct_cs_tur:econometrics|0|0 + extended|global_mmlu_instruct_unk_tur:econometrics|0|0 + extended|global_mmlu_instruct_all_ukr:econometrics|0|0 + extended|global_mmlu_instruct_ca_ukr:econometrics|0|0 + extended|global_mmlu_instruct_cs_ukr:econometrics|0|0 + extended|global_mmlu_instruct_unk_ukr:econometrics|0|0 + extended|global_mmlu_instruct_all_urd:econometrics|0|0 + extended|global_mmlu_instruct_ca_urd:econometrics|0|0 + extended|global_mmlu_instruct_cs_urd:econometrics|0|0 + extended|global_mmlu_instruct_unk_urd:econometrics|0|0 + extended|global_mmlu_instruct_all_vie:econometrics|0|0 + extended|global_mmlu_instruct_ca_vie:econometrics|0|0 + extended|global_mmlu_instruct_cs_vie:econometrics|0|0 + extended|global_mmlu_instruct_unk_vie:econometrics|0|0 + extended|global_mmlu_instruct_all_ara:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_ara:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_ara:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_ara:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_ben:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_ben:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_ben:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_ben:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_deu:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_deu:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_deu:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_deu:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_spa:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_spa:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_spa:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_spa:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_eng:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_eng:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_eng:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_eng:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_fra:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_fra:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_fra:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_fra:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_hin:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_hin:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_hin:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_hin:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_ind:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_ind:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_ind:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_ind:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_ita:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_ita:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_ita:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_ita:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_jpn:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_jpn:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_jpn:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_jpn:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_nld:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_nld:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_nld:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_nld:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_nor:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_nor:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_nor:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_nor:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_pol:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_pol:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_pol:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_pol:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_por:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_por:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_por:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_por:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_rus:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_rus:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_rus:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_rus:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_srp:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_srp:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_srp:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_srp:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_swe:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_swe:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_swe:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_swe:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_swa:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_swa:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_swa:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_swa:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_tel:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_tel:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_tel:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_tel:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_tha:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_tha:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_tha:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_tha:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_tur:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_tur:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_tur:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_tur:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_ukr:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_ukr:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_ukr:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_ukr:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_urd:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_urd:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_urd:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_urd:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_vie:electrical_engineering|0|0 + extended|global_mmlu_instruct_ca_vie:electrical_engineering|0|0 + extended|global_mmlu_instruct_cs_vie:electrical_engineering|0|0 + extended|global_mmlu_instruct_unk_vie:electrical_engineering|0|0 + extended|global_mmlu_instruct_all_ara:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_ara:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_ara:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_ara:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_ben:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_ben:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_ben:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_ben:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_deu:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_deu:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_deu:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_deu:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_spa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_spa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_spa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_spa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_eng:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_eng:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_eng:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_eng:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_fra:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_fra:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_fra:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_fra:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_hin:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_hin:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_hin:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_hin:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_ind:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_ind:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_ind:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_ind:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_ita:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_ita:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_ita:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_ita:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_jpn:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_jpn:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_jpn:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_jpn:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_nld:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_nld:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_nld:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_nld:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_nor:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_nor:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_nor:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_nor:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_pol:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_pol:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_pol:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_pol:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_por:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_por:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_por:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_por:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_rus:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_rus:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_rus:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_rus:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_srp:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_srp:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_srp:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_srp:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_swe:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_swe:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_swe:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_swe:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_swa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_swa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_swa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_swa:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_tel:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_tel:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_tel:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_tel:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_tha:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_tha:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_tha:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_tha:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_tur:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_tur:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_tur:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_tur:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_ukr:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_ukr:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_ukr:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_ukr:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_urd:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_urd:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_urd:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_urd:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_vie:elementary_mathematics|0|0 + extended|global_mmlu_instruct_ca_vie:elementary_mathematics|0|0 + extended|global_mmlu_instruct_cs_vie:elementary_mathematics|0|0 + extended|global_mmlu_instruct_unk_vie:elementary_mathematics|0|0 + extended|global_mmlu_instruct_all_ara:formal_logic|0|0 + extended|global_mmlu_instruct_ca_ara:formal_logic|0|0 + extended|global_mmlu_instruct_cs_ara:formal_logic|0|0 + extended|global_mmlu_instruct_unk_ara:formal_logic|0|0 + extended|global_mmlu_instruct_all_ben:formal_logic|0|0 + extended|global_mmlu_instruct_ca_ben:formal_logic|0|0 + extended|global_mmlu_instruct_cs_ben:formal_logic|0|0 + extended|global_mmlu_instruct_unk_ben:formal_logic|0|0 + extended|global_mmlu_instruct_all_deu:formal_logic|0|0 + extended|global_mmlu_instruct_ca_deu:formal_logic|0|0 + extended|global_mmlu_instruct_cs_deu:formal_logic|0|0 + extended|global_mmlu_instruct_unk_deu:formal_logic|0|0 + extended|global_mmlu_instruct_all_spa:formal_logic|0|0 + extended|global_mmlu_instruct_ca_spa:formal_logic|0|0 + extended|global_mmlu_instruct_cs_spa:formal_logic|0|0 + extended|global_mmlu_instruct_unk_spa:formal_logic|0|0 + extended|global_mmlu_instruct_all_eng:formal_logic|0|0 + extended|global_mmlu_instruct_ca_eng:formal_logic|0|0 + extended|global_mmlu_instruct_cs_eng:formal_logic|0|0 + extended|global_mmlu_instruct_unk_eng:formal_logic|0|0 + extended|global_mmlu_instruct_all_fra:formal_logic|0|0 + extended|global_mmlu_instruct_ca_fra:formal_logic|0|0 + extended|global_mmlu_instruct_cs_fra:formal_logic|0|0 + extended|global_mmlu_instruct_unk_fra:formal_logic|0|0 + extended|global_mmlu_instruct_all_hin:formal_logic|0|0 + extended|global_mmlu_instruct_ca_hin:formal_logic|0|0 + extended|global_mmlu_instruct_cs_hin:formal_logic|0|0 + extended|global_mmlu_instruct_unk_hin:formal_logic|0|0 + extended|global_mmlu_instruct_all_ind:formal_logic|0|0 + extended|global_mmlu_instruct_ca_ind:formal_logic|0|0 + extended|global_mmlu_instruct_cs_ind:formal_logic|0|0 + extended|global_mmlu_instruct_unk_ind:formal_logic|0|0 + extended|global_mmlu_instruct_all_ita:formal_logic|0|0 + extended|global_mmlu_instruct_ca_ita:formal_logic|0|0 + extended|global_mmlu_instruct_cs_ita:formal_logic|0|0 + extended|global_mmlu_instruct_unk_ita:formal_logic|0|0 + extended|global_mmlu_instruct_all_jpn:formal_logic|0|0 + extended|global_mmlu_instruct_ca_jpn:formal_logic|0|0 + extended|global_mmlu_instruct_cs_jpn:formal_logic|0|0 + extended|global_mmlu_instruct_unk_jpn:formal_logic|0|0 + extended|global_mmlu_instruct_all_nld:formal_logic|0|0 + extended|global_mmlu_instruct_ca_nld:formal_logic|0|0 + extended|global_mmlu_instruct_cs_nld:formal_logic|0|0 + extended|global_mmlu_instruct_unk_nld:formal_logic|0|0 + extended|global_mmlu_instruct_all_nor:formal_logic|0|0 + extended|global_mmlu_instruct_ca_nor:formal_logic|0|0 + extended|global_mmlu_instruct_cs_nor:formal_logic|0|0 + extended|global_mmlu_instruct_unk_nor:formal_logic|0|0 + extended|global_mmlu_instruct_all_pol:formal_logic|0|0 + extended|global_mmlu_instruct_ca_pol:formal_logic|0|0 + extended|global_mmlu_instruct_cs_pol:formal_logic|0|0 + extended|global_mmlu_instruct_unk_pol:formal_logic|0|0 + extended|global_mmlu_instruct_all_por:formal_logic|0|0 + extended|global_mmlu_instruct_ca_por:formal_logic|0|0 + extended|global_mmlu_instruct_cs_por:formal_logic|0|0 + extended|global_mmlu_instruct_unk_por:formal_logic|0|0 + extended|global_mmlu_instruct_all_rus:formal_logic|0|0 + extended|global_mmlu_instruct_ca_rus:formal_logic|0|0 + extended|global_mmlu_instruct_cs_rus:formal_logic|0|0 + extended|global_mmlu_instruct_unk_rus:formal_logic|0|0 + extended|global_mmlu_instruct_all_srp:formal_logic|0|0 + extended|global_mmlu_instruct_ca_srp:formal_logic|0|0 + extended|global_mmlu_instruct_cs_srp:formal_logic|0|0 + extended|global_mmlu_instruct_unk_srp:formal_logic|0|0 + extended|global_mmlu_instruct_all_swe:formal_logic|0|0 + extended|global_mmlu_instruct_ca_swe:formal_logic|0|0 + extended|global_mmlu_instruct_cs_swe:formal_logic|0|0 + extended|global_mmlu_instruct_unk_swe:formal_logic|0|0 + extended|global_mmlu_instruct_all_swa:formal_logic|0|0 + extended|global_mmlu_instruct_ca_swa:formal_logic|0|0 + extended|global_mmlu_instruct_cs_swa:formal_logic|0|0 + extended|global_mmlu_instruct_unk_swa:formal_logic|0|0 + extended|global_mmlu_instruct_all_tel:formal_logic|0|0 + extended|global_mmlu_instruct_ca_tel:formal_logic|0|0 + extended|global_mmlu_instruct_cs_tel:formal_logic|0|0 + extended|global_mmlu_instruct_unk_tel:formal_logic|0|0 + extended|global_mmlu_instruct_all_tha:formal_logic|0|0 + extended|global_mmlu_instruct_ca_tha:formal_logic|0|0 + extended|global_mmlu_instruct_cs_tha:formal_logic|0|0 + extended|global_mmlu_instruct_unk_tha:formal_logic|0|0 + extended|global_mmlu_instruct_all_tur:formal_logic|0|0 + extended|global_mmlu_instruct_ca_tur:formal_logic|0|0 + extended|global_mmlu_instruct_cs_tur:formal_logic|0|0 + extended|global_mmlu_instruct_unk_tur:formal_logic|0|0 + extended|global_mmlu_instruct_all_ukr:formal_logic|0|0 + extended|global_mmlu_instruct_ca_ukr:formal_logic|0|0 + extended|global_mmlu_instruct_cs_ukr:formal_logic|0|0 + extended|global_mmlu_instruct_unk_ukr:formal_logic|0|0 + extended|global_mmlu_instruct_all_urd:formal_logic|0|0 + extended|global_mmlu_instruct_ca_urd:formal_logic|0|0 + extended|global_mmlu_instruct_cs_urd:formal_logic|0|0 + extended|global_mmlu_instruct_unk_urd:formal_logic|0|0 + extended|global_mmlu_instruct_all_vie:formal_logic|0|0 + extended|global_mmlu_instruct_ca_vie:formal_logic|0|0 + extended|global_mmlu_instruct_cs_vie:formal_logic|0|0 + extended|global_mmlu_instruct_unk_vie:formal_logic|0|0 + extended|global_mmlu_instruct_all_ara:global_facts|0|0 + extended|global_mmlu_instruct_ca_ara:global_facts|0|0 + extended|global_mmlu_instruct_cs_ara:global_facts|0|0 + extended|global_mmlu_instruct_unk_ara:global_facts|0|0 + extended|global_mmlu_instruct_all_ben:global_facts|0|0 + extended|global_mmlu_instruct_ca_ben:global_facts|0|0 + extended|global_mmlu_instruct_cs_ben:global_facts|0|0 + extended|global_mmlu_instruct_unk_ben:global_facts|0|0 + extended|global_mmlu_instruct_all_deu:global_facts|0|0 + extended|global_mmlu_instruct_ca_deu:global_facts|0|0 + extended|global_mmlu_instruct_cs_deu:global_facts|0|0 + extended|global_mmlu_instruct_unk_deu:global_facts|0|0 + extended|global_mmlu_instruct_all_spa:global_facts|0|0 + extended|global_mmlu_instruct_ca_spa:global_facts|0|0 + extended|global_mmlu_instruct_cs_spa:global_facts|0|0 + extended|global_mmlu_instruct_unk_spa:global_facts|0|0 + extended|global_mmlu_instruct_all_eng:global_facts|0|0 + extended|global_mmlu_instruct_ca_eng:global_facts|0|0 + extended|global_mmlu_instruct_cs_eng:global_facts|0|0 + extended|global_mmlu_instruct_unk_eng:global_facts|0|0 + extended|global_mmlu_instruct_all_fra:global_facts|0|0 + extended|global_mmlu_instruct_ca_fra:global_facts|0|0 + extended|global_mmlu_instruct_cs_fra:global_facts|0|0 + extended|global_mmlu_instruct_unk_fra:global_facts|0|0 + extended|global_mmlu_instruct_all_hin:global_facts|0|0 + extended|global_mmlu_instruct_ca_hin:global_facts|0|0 + extended|global_mmlu_instruct_cs_hin:global_facts|0|0 + extended|global_mmlu_instruct_unk_hin:global_facts|0|0 + extended|global_mmlu_instruct_all_ind:global_facts|0|0 + extended|global_mmlu_instruct_ca_ind:global_facts|0|0 + extended|global_mmlu_instruct_cs_ind:global_facts|0|0 + extended|global_mmlu_instruct_unk_ind:global_facts|0|0 + extended|global_mmlu_instruct_all_ita:global_facts|0|0 + extended|global_mmlu_instruct_ca_ita:global_facts|0|0 + extended|global_mmlu_instruct_cs_ita:global_facts|0|0 + extended|global_mmlu_instruct_unk_ita:global_facts|0|0 + extended|global_mmlu_instruct_all_jpn:global_facts|0|0 + extended|global_mmlu_instruct_ca_jpn:global_facts|0|0 + extended|global_mmlu_instruct_cs_jpn:global_facts|0|0 + extended|global_mmlu_instruct_unk_jpn:global_facts|0|0 + extended|global_mmlu_instruct_all_nld:global_facts|0|0 + extended|global_mmlu_instruct_ca_nld:global_facts|0|0 + extended|global_mmlu_instruct_cs_nld:global_facts|0|0 + extended|global_mmlu_instruct_unk_nld:global_facts|0|0 + extended|global_mmlu_instruct_all_nor:global_facts|0|0 + extended|global_mmlu_instruct_ca_nor:global_facts|0|0 + extended|global_mmlu_instruct_cs_nor:global_facts|0|0 + extended|global_mmlu_instruct_unk_nor:global_facts|0|0 + extended|global_mmlu_instruct_all_pol:global_facts|0|0 + extended|global_mmlu_instruct_ca_pol:global_facts|0|0 + extended|global_mmlu_instruct_cs_pol:global_facts|0|0 + extended|global_mmlu_instruct_unk_pol:global_facts|0|0 + extended|global_mmlu_instruct_all_por:global_facts|0|0 + extended|global_mmlu_instruct_ca_por:global_facts|0|0 + extended|global_mmlu_instruct_cs_por:global_facts|0|0 + extended|global_mmlu_instruct_unk_por:global_facts|0|0 + extended|global_mmlu_instruct_all_rus:global_facts|0|0 + extended|global_mmlu_instruct_ca_rus:global_facts|0|0 + extended|global_mmlu_instruct_cs_rus:global_facts|0|0 + extended|global_mmlu_instruct_unk_rus:global_facts|0|0 + extended|global_mmlu_instruct_all_srp:global_facts|0|0 + extended|global_mmlu_instruct_ca_srp:global_facts|0|0 + extended|global_mmlu_instruct_cs_srp:global_facts|0|0 + extended|global_mmlu_instruct_unk_srp:global_facts|0|0 + extended|global_mmlu_instruct_all_swe:global_facts|0|0 + extended|global_mmlu_instruct_ca_swe:global_facts|0|0 + extended|global_mmlu_instruct_cs_swe:global_facts|0|0 + extended|global_mmlu_instruct_unk_swe:global_facts|0|0 + extended|global_mmlu_instruct_all_swa:global_facts|0|0 + extended|global_mmlu_instruct_ca_swa:global_facts|0|0 + extended|global_mmlu_instruct_cs_swa:global_facts|0|0 + extended|global_mmlu_instruct_unk_swa:global_facts|0|0 + extended|global_mmlu_instruct_all_tel:global_facts|0|0 + extended|global_mmlu_instruct_ca_tel:global_facts|0|0 + extended|global_mmlu_instruct_cs_tel:global_facts|0|0 + extended|global_mmlu_instruct_unk_tel:global_facts|0|0 + extended|global_mmlu_instruct_all_tha:global_facts|0|0 + extended|global_mmlu_instruct_ca_tha:global_facts|0|0 + extended|global_mmlu_instruct_cs_tha:global_facts|0|0 + extended|global_mmlu_instruct_unk_tha:global_facts|0|0 + extended|global_mmlu_instruct_all_tur:global_facts|0|0 + extended|global_mmlu_instruct_ca_tur:global_facts|0|0 + extended|global_mmlu_instruct_cs_tur:global_facts|0|0 + extended|global_mmlu_instruct_unk_tur:global_facts|0|0 + extended|global_mmlu_instruct_all_ukr:global_facts|0|0 + extended|global_mmlu_instruct_ca_ukr:global_facts|0|0 + extended|global_mmlu_instruct_cs_ukr:global_facts|0|0 + extended|global_mmlu_instruct_unk_ukr:global_facts|0|0 + extended|global_mmlu_instruct_all_urd:global_facts|0|0 + extended|global_mmlu_instruct_ca_urd:global_facts|0|0 + extended|global_mmlu_instruct_cs_urd:global_facts|0|0 + extended|global_mmlu_instruct_unk_urd:global_facts|0|0 + extended|global_mmlu_instruct_all_vie:global_facts|0|0 + extended|global_mmlu_instruct_ca_vie:global_facts|0|0 + extended|global_mmlu_instruct_cs_vie:global_facts|0|0 + extended|global_mmlu_instruct_unk_vie:global_facts|0|0 + extended|global_mmlu_instruct_all_ara:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_biology|0|0 + extended|global_mmlu_instruct_all_ben:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_biology|0|0 + extended|global_mmlu_instruct_all_deu:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_biology|0|0 + extended|global_mmlu_instruct_all_spa:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_biology|0|0 + extended|global_mmlu_instruct_all_eng:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_biology|0|0 + extended|global_mmlu_instruct_all_fra:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_biology|0|0 + extended|global_mmlu_instruct_all_hin:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_biology|0|0 + extended|global_mmlu_instruct_all_ind:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_biology|0|0 + extended|global_mmlu_instruct_all_ita:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_biology|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_biology|0|0 + extended|global_mmlu_instruct_all_nld:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_biology|0|0 + extended|global_mmlu_instruct_all_nor:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_biology|0|0 + extended|global_mmlu_instruct_all_pol:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_biology|0|0 + extended|global_mmlu_instruct_all_por:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_por:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_por:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_por:high_school_biology|0|0 + extended|global_mmlu_instruct_all_rus:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_biology|0|0 + extended|global_mmlu_instruct_all_srp:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_biology|0|0 + extended|global_mmlu_instruct_all_swe:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_biology|0|0 + extended|global_mmlu_instruct_all_swa:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_biology|0|0 + extended|global_mmlu_instruct_all_tel:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_biology|0|0 + extended|global_mmlu_instruct_all_tha:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_biology|0|0 + extended|global_mmlu_instruct_all_tur:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_biology|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_biology|0|0 + extended|global_mmlu_instruct_all_urd:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_biology|0|0 + extended|global_mmlu_instruct_all_vie:high_school_biology|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_biology|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_biology|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_biology|0|0 + extended|global_mmlu_instruct_all_ara:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_ben:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_deu:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_spa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_eng:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_fra:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_hin:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_ind:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_ita:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_nld:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_nor:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_pol:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_por:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_por:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_por:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_por:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_rus:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_srp:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_swe:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_swa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_tel:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_tha:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_tur:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_urd:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_vie:high_school_chemistry|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_chemistry|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_chemistry|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_chemistry|0|0 + extended|global_mmlu_instruct_all_ara:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_ben:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_deu:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_spa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_eng:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_fra:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_hin:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_ind:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_ita:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_nld:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_nor:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_pol:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_por:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_por:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_por:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_por:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_rus:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_srp:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_swe:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_swa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_tel:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_tha:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_tur:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_urd:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_vie:high_school_computer_science|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_computer_science|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_computer_science|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_computer_science|0|0 + extended|global_mmlu_instruct_all_ara:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_ben:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_deu:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_spa:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_eng:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_fra:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_hin:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_ind:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_ita:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_nld:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_nor:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_pol:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_por:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_por:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_por:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_por:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_rus:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_srp:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_swe:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_swa:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_tel:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_tha:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_tur:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_urd:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_vie:high_school_european_history|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_european_history|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_european_history|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_european_history|0|0 + extended|global_mmlu_instruct_all_ara:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_geography|0|0 + extended|global_mmlu_instruct_all_ben:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_geography|0|0 + extended|global_mmlu_instruct_all_deu:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_geography|0|0 + extended|global_mmlu_instruct_all_spa:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_geography|0|0 + extended|global_mmlu_instruct_all_eng:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_geography|0|0 + extended|global_mmlu_instruct_all_fra:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_geography|0|0 + extended|global_mmlu_instruct_all_hin:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_geography|0|0 + extended|global_mmlu_instruct_all_ind:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_geography|0|0 + extended|global_mmlu_instruct_all_ita:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_geography|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_geography|0|0 + extended|global_mmlu_instruct_all_nld:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_geography|0|0 + extended|global_mmlu_instruct_all_nor:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_geography|0|0 + extended|global_mmlu_instruct_all_pol:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_geography|0|0 + extended|global_mmlu_instruct_all_por:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_por:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_por:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_por:high_school_geography|0|0 + extended|global_mmlu_instruct_all_rus:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_geography|0|0 + extended|global_mmlu_instruct_all_srp:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_geography|0|0 + extended|global_mmlu_instruct_all_swe:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_geography|0|0 + extended|global_mmlu_instruct_all_swa:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_geography|0|0 + extended|global_mmlu_instruct_all_tel:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_geography|0|0 + extended|global_mmlu_instruct_all_tha:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_geography|0|0 + extended|global_mmlu_instruct_all_tur:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_geography|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_geography|0|0 + extended|global_mmlu_instruct_all_urd:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_geography|0|0 + extended|global_mmlu_instruct_all_vie:high_school_geography|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_geography|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_geography|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_geography|0|0 + extended|global_mmlu_instruct_all_ara:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_ben:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_deu:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_spa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_eng:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_fra:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_hin:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_ind:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_ita:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_nld:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_nor:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_pol:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_por:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_por:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_por:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_por:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_rus:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_srp:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_swe:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_swa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_tel:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_tha:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_tur:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_urd:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_vie:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_government_and_politics|0|0 + extended|global_mmlu_instruct_all_ara:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_ben:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_deu:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_spa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_eng:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_fra:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_hin:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_ind:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_ita:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_nld:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_nor:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_pol:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_por:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_por:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_por:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_por:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_rus:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_srp:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_swe:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_swa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_tel:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_tha:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_tur:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_urd:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_vie:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_macroeconomics|0|0 + extended|global_mmlu_instruct_all_ara:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_ben:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_deu:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_spa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_eng:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_fra:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_hin:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_ind:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_ita:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_nld:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_nor:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_pol:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_por:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_por:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_por:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_por:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_rus:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_srp:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_swe:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_swa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_tel:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_tha:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_tur:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_urd:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_vie:high_school_mathematics|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_mathematics|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_mathematics|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_mathematics|0|0 + extended|global_mmlu_instruct_all_ara:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_ben:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_deu:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_spa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_eng:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_fra:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_hin:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_ind:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_ita:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_nld:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_nor:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_pol:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_por:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_por:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_por:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_por:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_rus:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_srp:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_swe:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_swa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_tel:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_tha:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_tur:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_urd:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_vie:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_microeconomics|0|0 + extended|global_mmlu_instruct_all_ara:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_physics|0|0 + extended|global_mmlu_instruct_all_ben:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_physics|0|0 + extended|global_mmlu_instruct_all_deu:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_physics|0|0 + extended|global_mmlu_instruct_all_spa:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_physics|0|0 + extended|global_mmlu_instruct_all_eng:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_physics|0|0 + extended|global_mmlu_instruct_all_fra:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_physics|0|0 + extended|global_mmlu_instruct_all_hin:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_physics|0|0 + extended|global_mmlu_instruct_all_ind:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_physics|0|0 + extended|global_mmlu_instruct_all_ita:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_physics|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_physics|0|0 + extended|global_mmlu_instruct_all_nld:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_physics|0|0 + extended|global_mmlu_instruct_all_nor:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_physics|0|0 + extended|global_mmlu_instruct_all_pol:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_physics|0|0 + extended|global_mmlu_instruct_all_por:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_por:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_por:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_por:high_school_physics|0|0 + extended|global_mmlu_instruct_all_rus:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_physics|0|0 + extended|global_mmlu_instruct_all_srp:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_physics|0|0 + extended|global_mmlu_instruct_all_swe:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_physics|0|0 + extended|global_mmlu_instruct_all_swa:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_physics|0|0 + extended|global_mmlu_instruct_all_tel:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_physics|0|0 + extended|global_mmlu_instruct_all_tha:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_physics|0|0 + extended|global_mmlu_instruct_all_tur:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_physics|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_physics|0|0 + extended|global_mmlu_instruct_all_urd:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_physics|0|0 + extended|global_mmlu_instruct_all_vie:high_school_physics|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_physics|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_physics|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_physics|0|0 + extended|global_mmlu_instruct_all_ara:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_ben:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_deu:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_spa:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_eng:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_fra:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_hin:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_ind:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_ita:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_nld:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_nor:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_pol:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_por:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_por:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_por:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_por:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_rus:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_srp:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_swe:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_swa:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_tel:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_tha:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_tur:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_urd:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_vie:high_school_psychology|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_psychology|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_psychology|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_psychology|0|0 + extended|global_mmlu_instruct_all_ara:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_ben:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_deu:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_spa:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_eng:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_fra:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_hin:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_ind:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_ita:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_nld:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_nor:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_pol:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_por:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_por:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_por:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_por:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_rus:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_srp:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_swe:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_swa:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_tel:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_tha:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_tur:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_urd:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_vie:high_school_statistics|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_statistics|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_statistics|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_statistics|0|0 + extended|global_mmlu_instruct_all_ara:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_ben:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_deu:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_spa:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_eng:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_fra:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_hin:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_ind:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_ita:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_nld:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_nor:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_pol:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_por:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_por:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_por:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_por:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_rus:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_srp:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_swe:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_swa:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_tel:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_tha:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_tur:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_urd:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_vie:high_school_us_history|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_us_history|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_us_history|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_us_history|0|0 + extended|global_mmlu_instruct_all_ara:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_ara:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_ara:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_ara:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_ben:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_ben:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_ben:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_ben:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_deu:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_deu:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_deu:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_deu:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_spa:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_spa:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_spa:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_spa:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_eng:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_eng:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_eng:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_eng:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_fra:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_fra:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_fra:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_fra:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_hin:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_hin:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_hin:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_hin:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_ind:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_ind:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_ind:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_ind:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_ita:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_ita:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_ita:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_ita:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_jpn:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_jpn:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_jpn:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_jpn:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_nld:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_nld:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_nld:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_nld:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_nor:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_nor:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_nor:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_nor:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_pol:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_pol:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_pol:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_pol:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_por:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_por:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_por:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_por:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_rus:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_rus:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_rus:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_rus:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_srp:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_srp:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_srp:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_srp:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_swe:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_swe:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_swe:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_swe:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_swa:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_swa:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_swa:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_swa:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_tel:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_tel:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_tel:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_tel:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_tha:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_tha:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_tha:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_tha:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_tur:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_tur:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_tur:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_tur:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_ukr:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_ukr:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_ukr:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_ukr:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_urd:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_urd:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_urd:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_urd:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_vie:high_school_world_history|0|0 + extended|global_mmlu_instruct_ca_vie:high_school_world_history|0|0 + extended|global_mmlu_instruct_cs_vie:high_school_world_history|0|0 + extended|global_mmlu_instruct_unk_vie:high_school_world_history|0|0 + extended|global_mmlu_instruct_all_ara:human_aging|0|0 + extended|global_mmlu_instruct_ca_ara:human_aging|0|0 + extended|global_mmlu_instruct_cs_ara:human_aging|0|0 + extended|global_mmlu_instruct_unk_ara:human_aging|0|0 + extended|global_mmlu_instruct_all_ben:human_aging|0|0 + extended|global_mmlu_instruct_ca_ben:human_aging|0|0 + extended|global_mmlu_instruct_cs_ben:human_aging|0|0 + extended|global_mmlu_instruct_unk_ben:human_aging|0|0 + extended|global_mmlu_instruct_all_deu:human_aging|0|0 + extended|global_mmlu_instruct_ca_deu:human_aging|0|0 + extended|global_mmlu_instruct_cs_deu:human_aging|0|0 + extended|global_mmlu_instruct_unk_deu:human_aging|0|0 + extended|global_mmlu_instruct_all_spa:human_aging|0|0 + extended|global_mmlu_instruct_ca_spa:human_aging|0|0 + extended|global_mmlu_instruct_cs_spa:human_aging|0|0 + extended|global_mmlu_instruct_unk_spa:human_aging|0|0 + extended|global_mmlu_instruct_all_eng:human_aging|0|0 + extended|global_mmlu_instruct_ca_eng:human_aging|0|0 + extended|global_mmlu_instruct_cs_eng:human_aging|0|0 + extended|global_mmlu_instruct_unk_eng:human_aging|0|0 + extended|global_mmlu_instruct_all_fra:human_aging|0|0 + extended|global_mmlu_instruct_ca_fra:human_aging|0|0 + extended|global_mmlu_instruct_cs_fra:human_aging|0|0 + extended|global_mmlu_instruct_unk_fra:human_aging|0|0 + extended|global_mmlu_instruct_all_hin:human_aging|0|0 + extended|global_mmlu_instruct_ca_hin:human_aging|0|0 + extended|global_mmlu_instruct_cs_hin:human_aging|0|0 + extended|global_mmlu_instruct_unk_hin:human_aging|0|0 + extended|global_mmlu_instruct_all_ind:human_aging|0|0 + extended|global_mmlu_instruct_ca_ind:human_aging|0|0 + extended|global_mmlu_instruct_cs_ind:human_aging|0|0 + extended|global_mmlu_instruct_unk_ind:human_aging|0|0 + extended|global_mmlu_instruct_all_ita:human_aging|0|0 + extended|global_mmlu_instruct_ca_ita:human_aging|0|0 + extended|global_mmlu_instruct_cs_ita:human_aging|0|0 + extended|global_mmlu_instruct_unk_ita:human_aging|0|0 + extended|global_mmlu_instruct_all_jpn:human_aging|0|0 + extended|global_mmlu_instruct_ca_jpn:human_aging|0|0 + extended|global_mmlu_instruct_cs_jpn:human_aging|0|0 + extended|global_mmlu_instruct_unk_jpn:human_aging|0|0 + extended|global_mmlu_instruct_all_nld:human_aging|0|0 + extended|global_mmlu_instruct_ca_nld:human_aging|0|0 + extended|global_mmlu_instruct_cs_nld:human_aging|0|0 + extended|global_mmlu_instruct_unk_nld:human_aging|0|0 + extended|global_mmlu_instruct_all_nor:human_aging|0|0 + extended|global_mmlu_instruct_ca_nor:human_aging|0|0 + extended|global_mmlu_instruct_cs_nor:human_aging|0|0 + extended|global_mmlu_instruct_unk_nor:human_aging|0|0 + extended|global_mmlu_instruct_all_pol:human_aging|0|0 + extended|global_mmlu_instruct_ca_pol:human_aging|0|0 + extended|global_mmlu_instruct_cs_pol:human_aging|0|0 + extended|global_mmlu_instruct_unk_pol:human_aging|0|0 + extended|global_mmlu_instruct_all_por:human_aging|0|0 + extended|global_mmlu_instruct_ca_por:human_aging|0|0 + extended|global_mmlu_instruct_cs_por:human_aging|0|0 + extended|global_mmlu_instruct_unk_por:human_aging|0|0 + extended|global_mmlu_instruct_all_rus:human_aging|0|0 + extended|global_mmlu_instruct_ca_rus:human_aging|0|0 + extended|global_mmlu_instruct_cs_rus:human_aging|0|0 + extended|global_mmlu_instruct_unk_rus:human_aging|0|0 + extended|global_mmlu_instruct_all_srp:human_aging|0|0 + extended|global_mmlu_instruct_ca_srp:human_aging|0|0 + extended|global_mmlu_instruct_cs_srp:human_aging|0|0 + extended|global_mmlu_instruct_unk_srp:human_aging|0|0 + extended|global_mmlu_instruct_all_swe:human_aging|0|0 + extended|global_mmlu_instruct_ca_swe:human_aging|0|0 + extended|global_mmlu_instruct_cs_swe:human_aging|0|0 + extended|global_mmlu_instruct_unk_swe:human_aging|0|0 + extended|global_mmlu_instruct_all_swa:human_aging|0|0 + extended|global_mmlu_instruct_ca_swa:human_aging|0|0 + extended|global_mmlu_instruct_cs_swa:human_aging|0|0 + extended|global_mmlu_instruct_unk_swa:human_aging|0|0 + extended|global_mmlu_instruct_all_tel:human_aging|0|0 + extended|global_mmlu_instruct_ca_tel:human_aging|0|0 + extended|global_mmlu_instruct_cs_tel:human_aging|0|0 + extended|global_mmlu_instruct_unk_tel:human_aging|0|0 + extended|global_mmlu_instruct_all_tha:human_aging|0|0 + extended|global_mmlu_instruct_ca_tha:human_aging|0|0 + extended|global_mmlu_instruct_cs_tha:human_aging|0|0 + extended|global_mmlu_instruct_unk_tha:human_aging|0|0 + extended|global_mmlu_instruct_all_tur:human_aging|0|0 + extended|global_mmlu_instruct_ca_tur:human_aging|0|0 + extended|global_mmlu_instruct_cs_tur:human_aging|0|0 + extended|global_mmlu_instruct_unk_tur:human_aging|0|0 + extended|global_mmlu_instruct_all_ukr:human_aging|0|0 + extended|global_mmlu_instruct_ca_ukr:human_aging|0|0 + extended|global_mmlu_instruct_cs_ukr:human_aging|0|0 + extended|global_mmlu_instruct_unk_ukr:human_aging|0|0 + extended|global_mmlu_instruct_all_urd:human_aging|0|0 + extended|global_mmlu_instruct_ca_urd:human_aging|0|0 + extended|global_mmlu_instruct_cs_urd:human_aging|0|0 + extended|global_mmlu_instruct_unk_urd:human_aging|0|0 + extended|global_mmlu_instruct_all_vie:human_aging|0|0 + extended|global_mmlu_instruct_ca_vie:human_aging|0|0 + extended|global_mmlu_instruct_cs_vie:human_aging|0|0 + extended|global_mmlu_instruct_unk_vie:human_aging|0|0 + extended|global_mmlu_instruct_all_ara:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_ara:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_ara:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_ara:human_sexuality|0|0 + extended|global_mmlu_instruct_all_ben:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_ben:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_ben:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_ben:human_sexuality|0|0 + extended|global_mmlu_instruct_all_deu:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_deu:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_deu:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_deu:human_sexuality|0|0 + extended|global_mmlu_instruct_all_spa:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_spa:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_spa:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_spa:human_sexuality|0|0 + extended|global_mmlu_instruct_all_eng:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_eng:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_eng:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_eng:human_sexuality|0|0 + extended|global_mmlu_instruct_all_fra:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_fra:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_fra:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_fra:human_sexuality|0|0 + extended|global_mmlu_instruct_all_hin:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_hin:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_hin:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_hin:human_sexuality|0|0 + extended|global_mmlu_instruct_all_ind:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_ind:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_ind:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_ind:human_sexuality|0|0 + extended|global_mmlu_instruct_all_ita:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_ita:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_ita:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_ita:human_sexuality|0|0 + extended|global_mmlu_instruct_all_jpn:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_jpn:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_jpn:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_jpn:human_sexuality|0|0 + extended|global_mmlu_instruct_all_nld:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_nld:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_nld:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_nld:human_sexuality|0|0 + extended|global_mmlu_instruct_all_nor:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_nor:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_nor:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_nor:human_sexuality|0|0 + extended|global_mmlu_instruct_all_pol:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_pol:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_pol:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_pol:human_sexuality|0|0 + extended|global_mmlu_instruct_all_por:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_por:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_por:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_por:human_sexuality|0|0 + extended|global_mmlu_instruct_all_rus:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_rus:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_rus:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_rus:human_sexuality|0|0 + extended|global_mmlu_instruct_all_srp:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_srp:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_srp:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_srp:human_sexuality|0|0 + extended|global_mmlu_instruct_all_swe:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_swe:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_swe:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_swe:human_sexuality|0|0 + extended|global_mmlu_instruct_all_swa:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_swa:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_swa:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_swa:human_sexuality|0|0 + extended|global_mmlu_instruct_all_tel:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_tel:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_tel:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_tel:human_sexuality|0|0 + extended|global_mmlu_instruct_all_tha:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_tha:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_tha:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_tha:human_sexuality|0|0 + extended|global_mmlu_instruct_all_tur:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_tur:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_tur:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_tur:human_sexuality|0|0 + extended|global_mmlu_instruct_all_ukr:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_ukr:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_ukr:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_ukr:human_sexuality|0|0 + extended|global_mmlu_instruct_all_urd:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_urd:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_urd:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_urd:human_sexuality|0|0 + extended|global_mmlu_instruct_all_vie:human_sexuality|0|0 + extended|global_mmlu_instruct_ca_vie:human_sexuality|0|0 + extended|global_mmlu_instruct_cs_vie:human_sexuality|0|0 + extended|global_mmlu_instruct_unk_vie:human_sexuality|0|0 + extended|global_mmlu_instruct_all_ara:international_law|0|0 + extended|global_mmlu_instruct_ca_ara:international_law|0|0 + extended|global_mmlu_instruct_cs_ara:international_law|0|0 + extended|global_mmlu_instruct_unk_ara:international_law|0|0 + extended|global_mmlu_instruct_all_ben:international_law|0|0 + extended|global_mmlu_instruct_ca_ben:international_law|0|0 + extended|global_mmlu_instruct_cs_ben:international_law|0|0 + extended|global_mmlu_instruct_unk_ben:international_law|0|0 + extended|global_mmlu_instruct_all_deu:international_law|0|0 + extended|global_mmlu_instruct_ca_deu:international_law|0|0 + extended|global_mmlu_instruct_cs_deu:international_law|0|0 + extended|global_mmlu_instruct_unk_deu:international_law|0|0 + extended|global_mmlu_instruct_all_spa:international_law|0|0 + extended|global_mmlu_instruct_ca_spa:international_law|0|0 + extended|global_mmlu_instruct_cs_spa:international_law|0|0 + extended|global_mmlu_instruct_unk_spa:international_law|0|0 + extended|global_mmlu_instruct_all_eng:international_law|0|0 + extended|global_mmlu_instruct_ca_eng:international_law|0|0 + extended|global_mmlu_instruct_cs_eng:international_law|0|0 + extended|global_mmlu_instruct_unk_eng:international_law|0|0 + extended|global_mmlu_instruct_all_fra:international_law|0|0 + extended|global_mmlu_instruct_ca_fra:international_law|0|0 + extended|global_mmlu_instruct_cs_fra:international_law|0|0 + extended|global_mmlu_instruct_unk_fra:international_law|0|0 + extended|global_mmlu_instruct_all_hin:international_law|0|0 + extended|global_mmlu_instruct_ca_hin:international_law|0|0 + extended|global_mmlu_instruct_cs_hin:international_law|0|0 + extended|global_mmlu_instruct_unk_hin:international_law|0|0 + extended|global_mmlu_instruct_all_ind:international_law|0|0 + extended|global_mmlu_instruct_ca_ind:international_law|0|0 + extended|global_mmlu_instruct_cs_ind:international_law|0|0 + extended|global_mmlu_instruct_unk_ind:international_law|0|0 + extended|global_mmlu_instruct_all_ita:international_law|0|0 + extended|global_mmlu_instruct_ca_ita:international_law|0|0 + extended|global_mmlu_instruct_cs_ita:international_law|0|0 + extended|global_mmlu_instruct_unk_ita:international_law|0|0 + extended|global_mmlu_instruct_all_jpn:international_law|0|0 + extended|global_mmlu_instruct_ca_jpn:international_law|0|0 + extended|global_mmlu_instruct_cs_jpn:international_law|0|0 + extended|global_mmlu_instruct_unk_jpn:international_law|0|0 + extended|global_mmlu_instruct_all_nld:international_law|0|0 + extended|global_mmlu_instruct_ca_nld:international_law|0|0 + extended|global_mmlu_instruct_cs_nld:international_law|0|0 + extended|global_mmlu_instruct_unk_nld:international_law|0|0 + extended|global_mmlu_instruct_all_nor:international_law|0|0 + extended|global_mmlu_instruct_ca_nor:international_law|0|0 + extended|global_mmlu_instruct_cs_nor:international_law|0|0 + extended|global_mmlu_instruct_unk_nor:international_law|0|0 + extended|global_mmlu_instruct_all_pol:international_law|0|0 + extended|global_mmlu_instruct_ca_pol:international_law|0|0 + extended|global_mmlu_instruct_cs_pol:international_law|0|0 + extended|global_mmlu_instruct_unk_pol:international_law|0|0 + extended|global_mmlu_instruct_all_por:international_law|0|0 + extended|global_mmlu_instruct_ca_por:international_law|0|0 + extended|global_mmlu_instruct_cs_por:international_law|0|0 + extended|global_mmlu_instruct_unk_por:international_law|0|0 + extended|global_mmlu_instruct_all_rus:international_law|0|0 + extended|global_mmlu_instruct_ca_rus:international_law|0|0 + extended|global_mmlu_instruct_cs_rus:international_law|0|0 + extended|global_mmlu_instruct_unk_rus:international_law|0|0 + extended|global_mmlu_instruct_all_srp:international_law|0|0 + extended|global_mmlu_instruct_ca_srp:international_law|0|0 + extended|global_mmlu_instruct_cs_srp:international_law|0|0 + extended|global_mmlu_instruct_unk_srp:international_law|0|0 + extended|global_mmlu_instruct_all_swe:international_law|0|0 + extended|global_mmlu_instruct_ca_swe:international_law|0|0 + extended|global_mmlu_instruct_cs_swe:international_law|0|0 + extended|global_mmlu_instruct_unk_swe:international_law|0|0 + extended|global_mmlu_instruct_all_swa:international_law|0|0 + extended|global_mmlu_instruct_ca_swa:international_law|0|0 + extended|global_mmlu_instruct_cs_swa:international_law|0|0 + extended|global_mmlu_instruct_unk_swa:international_law|0|0 + extended|global_mmlu_instruct_all_tel:international_law|0|0 + extended|global_mmlu_instruct_ca_tel:international_law|0|0 + extended|global_mmlu_instruct_cs_tel:international_law|0|0 + extended|global_mmlu_instruct_unk_tel:international_law|0|0 + extended|global_mmlu_instruct_all_tha:international_law|0|0 + extended|global_mmlu_instruct_ca_tha:international_law|0|0 + extended|global_mmlu_instruct_cs_tha:international_law|0|0 + extended|global_mmlu_instruct_unk_tha:international_law|0|0 + extended|global_mmlu_instruct_all_tur:international_law|0|0 + extended|global_mmlu_instruct_ca_tur:international_law|0|0 + extended|global_mmlu_instruct_cs_tur:international_law|0|0 + extended|global_mmlu_instruct_unk_tur:international_law|0|0 + extended|global_mmlu_instruct_all_ukr:international_law|0|0 + extended|global_mmlu_instruct_ca_ukr:international_law|0|0 + extended|global_mmlu_instruct_cs_ukr:international_law|0|0 + extended|global_mmlu_instruct_unk_ukr:international_law|0|0 + extended|global_mmlu_instruct_all_urd:international_law|0|0 + extended|global_mmlu_instruct_ca_urd:international_law|0|0 + extended|global_mmlu_instruct_cs_urd:international_law|0|0 + extended|global_mmlu_instruct_unk_urd:international_law|0|0 + extended|global_mmlu_instruct_all_vie:international_law|0|0 + extended|global_mmlu_instruct_ca_vie:international_law|0|0 + extended|global_mmlu_instruct_cs_vie:international_law|0|0 + extended|global_mmlu_instruct_unk_vie:international_law|0|0 + extended|global_mmlu_instruct_all_ara:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_ara:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_ara:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_ara:jurisprudence|0|0 + extended|global_mmlu_instruct_all_ben:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_ben:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_ben:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_ben:jurisprudence|0|0 + extended|global_mmlu_instruct_all_deu:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_deu:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_deu:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_deu:jurisprudence|0|0 + extended|global_mmlu_instruct_all_spa:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_spa:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_spa:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_spa:jurisprudence|0|0 + extended|global_mmlu_instruct_all_eng:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_eng:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_eng:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_eng:jurisprudence|0|0 + extended|global_mmlu_instruct_all_fra:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_fra:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_fra:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_fra:jurisprudence|0|0 + extended|global_mmlu_instruct_all_hin:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_hin:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_hin:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_hin:jurisprudence|0|0 + extended|global_mmlu_instruct_all_ind:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_ind:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_ind:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_ind:jurisprudence|0|0 + extended|global_mmlu_instruct_all_ita:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_ita:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_ita:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_ita:jurisprudence|0|0 + extended|global_mmlu_instruct_all_jpn:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_jpn:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_jpn:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_jpn:jurisprudence|0|0 + extended|global_mmlu_instruct_all_nld:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_nld:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_nld:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_nld:jurisprudence|0|0 + extended|global_mmlu_instruct_all_nor:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_nor:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_nor:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_nor:jurisprudence|0|0 + extended|global_mmlu_instruct_all_pol:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_pol:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_pol:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_pol:jurisprudence|0|0 + extended|global_mmlu_instruct_all_por:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_por:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_por:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_por:jurisprudence|0|0 + extended|global_mmlu_instruct_all_rus:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_rus:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_rus:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_rus:jurisprudence|0|0 + extended|global_mmlu_instruct_all_srp:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_srp:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_srp:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_srp:jurisprudence|0|0 + extended|global_mmlu_instruct_all_swe:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_swe:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_swe:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_swe:jurisprudence|0|0 + extended|global_mmlu_instruct_all_swa:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_swa:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_swa:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_swa:jurisprudence|0|0 + extended|global_mmlu_instruct_all_tel:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_tel:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_tel:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_tel:jurisprudence|0|0 + extended|global_mmlu_instruct_all_tha:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_tha:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_tha:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_tha:jurisprudence|0|0 + extended|global_mmlu_instruct_all_tur:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_tur:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_tur:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_tur:jurisprudence|0|0 + extended|global_mmlu_instruct_all_ukr:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_ukr:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_ukr:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_ukr:jurisprudence|0|0 + extended|global_mmlu_instruct_all_urd:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_urd:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_urd:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_urd:jurisprudence|0|0 + extended|global_mmlu_instruct_all_vie:jurisprudence|0|0 + extended|global_mmlu_instruct_ca_vie:jurisprudence|0|0 + extended|global_mmlu_instruct_cs_vie:jurisprudence|0|0 + extended|global_mmlu_instruct_unk_vie:jurisprudence|0|0 + extended|global_mmlu_instruct_all_ara:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_ara:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_ara:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_ara:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_ben:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_ben:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_ben:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_ben:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_deu:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_deu:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_deu:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_deu:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_spa:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_spa:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_spa:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_spa:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_eng:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_eng:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_eng:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_eng:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_fra:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_fra:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_fra:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_fra:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_hin:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_hin:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_hin:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_hin:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_ind:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_ind:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_ind:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_ind:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_ita:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_ita:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_ita:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_ita:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_jpn:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_jpn:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_jpn:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_jpn:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_nld:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_nld:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_nld:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_nld:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_nor:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_nor:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_nor:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_nor:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_pol:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_pol:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_pol:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_pol:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_por:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_por:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_por:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_por:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_rus:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_rus:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_rus:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_rus:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_srp:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_srp:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_srp:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_srp:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_swe:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_swe:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_swe:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_swe:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_swa:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_swa:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_swa:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_swa:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_tel:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_tel:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_tel:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_tel:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_tha:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_tha:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_tha:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_tha:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_tur:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_tur:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_tur:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_tur:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_ukr:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_ukr:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_ukr:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_ukr:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_urd:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_urd:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_urd:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_urd:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_vie:logical_fallacies|0|0 + extended|global_mmlu_instruct_ca_vie:logical_fallacies|0|0 + extended|global_mmlu_instruct_cs_vie:logical_fallacies|0|0 + extended|global_mmlu_instruct_unk_vie:logical_fallacies|0|0 + extended|global_mmlu_instruct_all_ara:machine_learning|0|0 + extended|global_mmlu_instruct_ca_ara:machine_learning|0|0 + extended|global_mmlu_instruct_cs_ara:machine_learning|0|0 + extended|global_mmlu_instruct_unk_ara:machine_learning|0|0 + extended|global_mmlu_instruct_all_ben:machine_learning|0|0 + extended|global_mmlu_instruct_ca_ben:machine_learning|0|0 + extended|global_mmlu_instruct_cs_ben:machine_learning|0|0 + extended|global_mmlu_instruct_unk_ben:machine_learning|0|0 + extended|global_mmlu_instruct_all_deu:machine_learning|0|0 + extended|global_mmlu_instruct_ca_deu:machine_learning|0|0 + extended|global_mmlu_instruct_cs_deu:machine_learning|0|0 + extended|global_mmlu_instruct_unk_deu:machine_learning|0|0 + extended|global_mmlu_instruct_all_spa:machine_learning|0|0 + extended|global_mmlu_instruct_ca_spa:machine_learning|0|0 + extended|global_mmlu_instruct_cs_spa:machine_learning|0|0 + extended|global_mmlu_instruct_unk_spa:machine_learning|0|0 + extended|global_mmlu_instruct_all_eng:machine_learning|0|0 + extended|global_mmlu_instruct_ca_eng:machine_learning|0|0 + extended|global_mmlu_instruct_cs_eng:machine_learning|0|0 + extended|global_mmlu_instruct_unk_eng:machine_learning|0|0 + extended|global_mmlu_instruct_all_fra:machine_learning|0|0 + extended|global_mmlu_instruct_ca_fra:machine_learning|0|0 + extended|global_mmlu_instruct_cs_fra:machine_learning|0|0 + extended|global_mmlu_instruct_unk_fra:machine_learning|0|0 + extended|global_mmlu_instruct_all_hin:machine_learning|0|0 + extended|global_mmlu_instruct_ca_hin:machine_learning|0|0 + extended|global_mmlu_instruct_cs_hin:machine_learning|0|0 + extended|global_mmlu_instruct_unk_hin:machine_learning|0|0 + extended|global_mmlu_instruct_all_ind:machine_learning|0|0 + extended|global_mmlu_instruct_ca_ind:machine_learning|0|0 + extended|global_mmlu_instruct_cs_ind:machine_learning|0|0 + extended|global_mmlu_instruct_unk_ind:machine_learning|0|0 + extended|global_mmlu_instruct_all_ita:machine_learning|0|0 + extended|global_mmlu_instruct_ca_ita:machine_learning|0|0 + extended|global_mmlu_instruct_cs_ita:machine_learning|0|0 + extended|global_mmlu_instruct_unk_ita:machine_learning|0|0 + extended|global_mmlu_instruct_all_jpn:machine_learning|0|0 + extended|global_mmlu_instruct_ca_jpn:machine_learning|0|0 + extended|global_mmlu_instruct_cs_jpn:machine_learning|0|0 + extended|global_mmlu_instruct_unk_jpn:machine_learning|0|0 + extended|global_mmlu_instruct_all_nld:machine_learning|0|0 + extended|global_mmlu_instruct_ca_nld:machine_learning|0|0 + extended|global_mmlu_instruct_cs_nld:machine_learning|0|0 + extended|global_mmlu_instruct_unk_nld:machine_learning|0|0 + extended|global_mmlu_instruct_all_nor:machine_learning|0|0 + extended|global_mmlu_instruct_ca_nor:machine_learning|0|0 + extended|global_mmlu_instruct_cs_nor:machine_learning|0|0 + extended|global_mmlu_instruct_unk_nor:machine_learning|0|0 + extended|global_mmlu_instruct_all_pol:machine_learning|0|0 + extended|global_mmlu_instruct_ca_pol:machine_learning|0|0 + extended|global_mmlu_instruct_cs_pol:machine_learning|0|0 + extended|global_mmlu_instruct_unk_pol:machine_learning|0|0 + extended|global_mmlu_instruct_all_por:machine_learning|0|0 + extended|global_mmlu_instruct_ca_por:machine_learning|0|0 + extended|global_mmlu_instruct_cs_por:machine_learning|0|0 + extended|global_mmlu_instruct_unk_por:machine_learning|0|0 + extended|global_mmlu_instruct_all_rus:machine_learning|0|0 + extended|global_mmlu_instruct_ca_rus:machine_learning|0|0 + extended|global_mmlu_instruct_cs_rus:machine_learning|0|0 + extended|global_mmlu_instruct_unk_rus:machine_learning|0|0 + extended|global_mmlu_instruct_all_srp:machine_learning|0|0 + extended|global_mmlu_instruct_ca_srp:machine_learning|0|0 + extended|global_mmlu_instruct_cs_srp:machine_learning|0|0 + extended|global_mmlu_instruct_unk_srp:machine_learning|0|0 + extended|global_mmlu_instruct_all_swe:machine_learning|0|0 + extended|global_mmlu_instruct_ca_swe:machine_learning|0|0 + extended|global_mmlu_instruct_cs_swe:machine_learning|0|0 + extended|global_mmlu_instruct_unk_swe:machine_learning|0|0 + extended|global_mmlu_instruct_all_swa:machine_learning|0|0 + extended|global_mmlu_instruct_ca_swa:machine_learning|0|0 + extended|global_mmlu_instruct_cs_swa:machine_learning|0|0 + extended|global_mmlu_instruct_unk_swa:machine_learning|0|0 + extended|global_mmlu_instruct_all_tel:machine_learning|0|0 + extended|global_mmlu_instruct_ca_tel:machine_learning|0|0 + extended|global_mmlu_instruct_cs_tel:machine_learning|0|0 + extended|global_mmlu_instruct_unk_tel:machine_learning|0|0 + extended|global_mmlu_instruct_all_tha:machine_learning|0|0 + extended|global_mmlu_instruct_ca_tha:machine_learning|0|0 + extended|global_mmlu_instruct_cs_tha:machine_learning|0|0 + extended|global_mmlu_instruct_unk_tha:machine_learning|0|0 + extended|global_mmlu_instruct_all_tur:machine_learning|0|0 + extended|global_mmlu_instruct_ca_tur:machine_learning|0|0 + extended|global_mmlu_instruct_cs_tur:machine_learning|0|0 + extended|global_mmlu_instruct_unk_tur:machine_learning|0|0 + extended|global_mmlu_instruct_all_ukr:machine_learning|0|0 + extended|global_mmlu_instruct_ca_ukr:machine_learning|0|0 + extended|global_mmlu_instruct_cs_ukr:machine_learning|0|0 + extended|global_mmlu_instruct_unk_ukr:machine_learning|0|0 + extended|global_mmlu_instruct_all_urd:machine_learning|0|0 + extended|global_mmlu_instruct_ca_urd:machine_learning|0|0 + extended|global_mmlu_instruct_cs_urd:machine_learning|0|0 + extended|global_mmlu_instruct_unk_urd:machine_learning|0|0 + extended|global_mmlu_instruct_all_vie:machine_learning|0|0 + extended|global_mmlu_instruct_ca_vie:machine_learning|0|0 + extended|global_mmlu_instruct_cs_vie:machine_learning|0|0 + extended|global_mmlu_instruct_unk_vie:machine_learning|0|0 + extended|global_mmlu_instruct_all_ara:management|0|0 + extended|global_mmlu_instruct_ca_ara:management|0|0 + extended|global_mmlu_instruct_cs_ara:management|0|0 + extended|global_mmlu_instruct_unk_ara:management|0|0 + extended|global_mmlu_instruct_all_ben:management|0|0 + extended|global_mmlu_instruct_ca_ben:management|0|0 + extended|global_mmlu_instruct_cs_ben:management|0|0 + extended|global_mmlu_instruct_unk_ben:management|0|0 + extended|global_mmlu_instruct_all_deu:management|0|0 + extended|global_mmlu_instruct_ca_deu:management|0|0 + extended|global_mmlu_instruct_cs_deu:management|0|0 + extended|global_mmlu_instruct_unk_deu:management|0|0 + extended|global_mmlu_instruct_all_spa:management|0|0 + extended|global_mmlu_instruct_ca_spa:management|0|0 + extended|global_mmlu_instruct_cs_spa:management|0|0 + extended|global_mmlu_instruct_unk_spa:management|0|0 + extended|global_mmlu_instruct_all_eng:management|0|0 + extended|global_mmlu_instruct_ca_eng:management|0|0 + extended|global_mmlu_instruct_cs_eng:management|0|0 + extended|global_mmlu_instruct_unk_eng:management|0|0 + extended|global_mmlu_instruct_all_fra:management|0|0 + extended|global_mmlu_instruct_ca_fra:management|0|0 + extended|global_mmlu_instruct_cs_fra:management|0|0 + extended|global_mmlu_instruct_unk_fra:management|0|0 + extended|global_mmlu_instruct_all_hin:management|0|0 + extended|global_mmlu_instruct_ca_hin:management|0|0 + extended|global_mmlu_instruct_cs_hin:management|0|0 + extended|global_mmlu_instruct_unk_hin:management|0|0 + extended|global_mmlu_instruct_all_ind:management|0|0 + extended|global_mmlu_instruct_ca_ind:management|0|0 + extended|global_mmlu_instruct_cs_ind:management|0|0 + extended|global_mmlu_instruct_unk_ind:management|0|0 + extended|global_mmlu_instruct_all_ita:management|0|0 + extended|global_mmlu_instruct_ca_ita:management|0|0 + extended|global_mmlu_instruct_cs_ita:management|0|0 + extended|global_mmlu_instruct_unk_ita:management|0|0 + extended|global_mmlu_instruct_all_jpn:management|0|0 + extended|global_mmlu_instruct_ca_jpn:management|0|0 + extended|global_mmlu_instruct_cs_jpn:management|0|0 + extended|global_mmlu_instruct_unk_jpn:management|0|0 + extended|global_mmlu_instruct_all_nld:management|0|0 + extended|global_mmlu_instruct_ca_nld:management|0|0 + extended|global_mmlu_instruct_cs_nld:management|0|0 + extended|global_mmlu_instruct_unk_nld:management|0|0 + extended|global_mmlu_instruct_all_nor:management|0|0 + extended|global_mmlu_instruct_ca_nor:management|0|0 + extended|global_mmlu_instruct_cs_nor:management|0|0 + extended|global_mmlu_instruct_unk_nor:management|0|0 + extended|global_mmlu_instruct_all_pol:management|0|0 + extended|global_mmlu_instruct_ca_pol:management|0|0 + extended|global_mmlu_instruct_cs_pol:management|0|0 + extended|global_mmlu_instruct_unk_pol:management|0|0 + extended|global_mmlu_instruct_all_por:management|0|0 + extended|global_mmlu_instruct_ca_por:management|0|0 + extended|global_mmlu_instruct_cs_por:management|0|0 + extended|global_mmlu_instruct_unk_por:management|0|0 + extended|global_mmlu_instruct_all_rus:management|0|0 + extended|global_mmlu_instruct_ca_rus:management|0|0 + extended|global_mmlu_instruct_cs_rus:management|0|0 + extended|global_mmlu_instruct_unk_rus:management|0|0 + extended|global_mmlu_instruct_all_srp:management|0|0 + extended|global_mmlu_instruct_ca_srp:management|0|0 + extended|global_mmlu_instruct_cs_srp:management|0|0 + extended|global_mmlu_instruct_unk_srp:management|0|0 + extended|global_mmlu_instruct_all_swe:management|0|0 + extended|global_mmlu_instruct_ca_swe:management|0|0 + extended|global_mmlu_instruct_cs_swe:management|0|0 + extended|global_mmlu_instruct_unk_swe:management|0|0 + extended|global_mmlu_instruct_all_swa:management|0|0 + extended|global_mmlu_instruct_ca_swa:management|0|0 + extended|global_mmlu_instruct_cs_swa:management|0|0 + extended|global_mmlu_instruct_unk_swa:management|0|0 + extended|global_mmlu_instruct_all_tel:management|0|0 + extended|global_mmlu_instruct_ca_tel:management|0|0 + extended|global_mmlu_instruct_cs_tel:management|0|0 + extended|global_mmlu_instruct_unk_tel:management|0|0 + extended|global_mmlu_instruct_all_tha:management|0|0 + extended|global_mmlu_instruct_ca_tha:management|0|0 + extended|global_mmlu_instruct_cs_tha:management|0|0 + extended|global_mmlu_instruct_unk_tha:management|0|0 + extended|global_mmlu_instruct_all_tur:management|0|0 + extended|global_mmlu_instruct_ca_tur:management|0|0 + extended|global_mmlu_instruct_cs_tur:management|0|0 + extended|global_mmlu_instruct_unk_tur:management|0|0 + extended|global_mmlu_instruct_all_ukr:management|0|0 + extended|global_mmlu_instruct_ca_ukr:management|0|0 + extended|global_mmlu_instruct_cs_ukr:management|0|0 + extended|global_mmlu_instruct_unk_ukr:management|0|0 + extended|global_mmlu_instruct_all_urd:management|0|0 + extended|global_mmlu_instruct_ca_urd:management|0|0 + extended|global_mmlu_instruct_cs_urd:management|0|0 + extended|global_mmlu_instruct_unk_urd:management|0|0 + extended|global_mmlu_instruct_all_vie:management|0|0 + extended|global_mmlu_instruct_ca_vie:management|0|0 + extended|global_mmlu_instruct_cs_vie:management|0|0 + extended|global_mmlu_instruct_unk_vie:management|0|0 + extended|global_mmlu_instruct_all_ara:marketing|0|0 + extended|global_mmlu_instruct_ca_ara:marketing|0|0 + extended|global_mmlu_instruct_cs_ara:marketing|0|0 + extended|global_mmlu_instruct_unk_ara:marketing|0|0 + extended|global_mmlu_instruct_all_ben:marketing|0|0 + extended|global_mmlu_instruct_ca_ben:marketing|0|0 + extended|global_mmlu_instruct_cs_ben:marketing|0|0 + extended|global_mmlu_instruct_unk_ben:marketing|0|0 + extended|global_mmlu_instruct_all_deu:marketing|0|0 + extended|global_mmlu_instruct_ca_deu:marketing|0|0 + extended|global_mmlu_instruct_cs_deu:marketing|0|0 + extended|global_mmlu_instruct_unk_deu:marketing|0|0 + extended|global_mmlu_instruct_all_spa:marketing|0|0 + extended|global_mmlu_instruct_ca_spa:marketing|0|0 + extended|global_mmlu_instruct_cs_spa:marketing|0|0 + extended|global_mmlu_instruct_unk_spa:marketing|0|0 + extended|global_mmlu_instruct_all_eng:marketing|0|0 + extended|global_mmlu_instruct_ca_eng:marketing|0|0 + extended|global_mmlu_instruct_cs_eng:marketing|0|0 + extended|global_mmlu_instruct_unk_eng:marketing|0|0 + extended|global_mmlu_instruct_all_fra:marketing|0|0 + extended|global_mmlu_instruct_ca_fra:marketing|0|0 + extended|global_mmlu_instruct_cs_fra:marketing|0|0 + extended|global_mmlu_instruct_unk_fra:marketing|0|0 + extended|global_mmlu_instruct_all_hin:marketing|0|0 + extended|global_mmlu_instruct_ca_hin:marketing|0|0 + extended|global_mmlu_instruct_cs_hin:marketing|0|0 + extended|global_mmlu_instruct_unk_hin:marketing|0|0 + extended|global_mmlu_instruct_all_ind:marketing|0|0 + extended|global_mmlu_instruct_ca_ind:marketing|0|0 + extended|global_mmlu_instruct_cs_ind:marketing|0|0 + extended|global_mmlu_instruct_unk_ind:marketing|0|0 + extended|global_mmlu_instruct_all_ita:marketing|0|0 + extended|global_mmlu_instruct_ca_ita:marketing|0|0 + extended|global_mmlu_instruct_cs_ita:marketing|0|0 + extended|global_mmlu_instruct_unk_ita:marketing|0|0 + extended|global_mmlu_instruct_all_jpn:marketing|0|0 + extended|global_mmlu_instruct_ca_jpn:marketing|0|0 + extended|global_mmlu_instruct_cs_jpn:marketing|0|0 + extended|global_mmlu_instruct_unk_jpn:marketing|0|0 + extended|global_mmlu_instruct_all_nld:marketing|0|0 + extended|global_mmlu_instruct_ca_nld:marketing|0|0 + extended|global_mmlu_instruct_cs_nld:marketing|0|0 + extended|global_mmlu_instruct_unk_nld:marketing|0|0 + extended|global_mmlu_instruct_all_nor:marketing|0|0 + extended|global_mmlu_instruct_ca_nor:marketing|0|0 + extended|global_mmlu_instruct_cs_nor:marketing|0|0 + extended|global_mmlu_instruct_unk_nor:marketing|0|0 + extended|global_mmlu_instruct_all_pol:marketing|0|0 + extended|global_mmlu_instruct_ca_pol:marketing|0|0 + extended|global_mmlu_instruct_cs_pol:marketing|0|0 + extended|global_mmlu_instruct_unk_pol:marketing|0|0 + extended|global_mmlu_instruct_all_por:marketing|0|0 + extended|global_mmlu_instruct_ca_por:marketing|0|0 + extended|global_mmlu_instruct_cs_por:marketing|0|0 + extended|global_mmlu_instruct_unk_por:marketing|0|0 + extended|global_mmlu_instruct_all_rus:marketing|0|0 + extended|global_mmlu_instruct_ca_rus:marketing|0|0 + extended|global_mmlu_instruct_cs_rus:marketing|0|0 + extended|global_mmlu_instruct_unk_rus:marketing|0|0 + extended|global_mmlu_instruct_all_srp:marketing|0|0 + extended|global_mmlu_instruct_ca_srp:marketing|0|0 + extended|global_mmlu_instruct_cs_srp:marketing|0|0 + extended|global_mmlu_instruct_unk_srp:marketing|0|0 + extended|global_mmlu_instruct_all_swe:marketing|0|0 + extended|global_mmlu_instruct_ca_swe:marketing|0|0 + extended|global_mmlu_instruct_cs_swe:marketing|0|0 + extended|global_mmlu_instruct_unk_swe:marketing|0|0 + extended|global_mmlu_instruct_all_swa:marketing|0|0 + extended|global_mmlu_instruct_ca_swa:marketing|0|0 + extended|global_mmlu_instruct_cs_swa:marketing|0|0 + extended|global_mmlu_instruct_unk_swa:marketing|0|0 + extended|global_mmlu_instruct_all_tel:marketing|0|0 + extended|global_mmlu_instruct_ca_tel:marketing|0|0 + extended|global_mmlu_instruct_cs_tel:marketing|0|0 + extended|global_mmlu_instruct_unk_tel:marketing|0|0 + extended|global_mmlu_instruct_all_tha:marketing|0|0 + extended|global_mmlu_instruct_ca_tha:marketing|0|0 + extended|global_mmlu_instruct_cs_tha:marketing|0|0 + extended|global_mmlu_instruct_unk_tha:marketing|0|0 + extended|global_mmlu_instruct_all_tur:marketing|0|0 + extended|global_mmlu_instruct_ca_tur:marketing|0|0 + extended|global_mmlu_instruct_cs_tur:marketing|0|0 + extended|global_mmlu_instruct_unk_tur:marketing|0|0 + extended|global_mmlu_instruct_all_ukr:marketing|0|0 + extended|global_mmlu_instruct_ca_ukr:marketing|0|0 + extended|global_mmlu_instruct_cs_ukr:marketing|0|0 + extended|global_mmlu_instruct_unk_ukr:marketing|0|0 + extended|global_mmlu_instruct_all_urd:marketing|0|0 + extended|global_mmlu_instruct_ca_urd:marketing|0|0 + extended|global_mmlu_instruct_cs_urd:marketing|0|0 + extended|global_mmlu_instruct_unk_urd:marketing|0|0 + extended|global_mmlu_instruct_all_vie:marketing|0|0 + extended|global_mmlu_instruct_ca_vie:marketing|0|0 + extended|global_mmlu_instruct_cs_vie:marketing|0|0 + extended|global_mmlu_instruct_unk_vie:marketing|0|0 + extended|global_mmlu_instruct_all_ara:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_ara:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_ara:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_ara:medical_genetics|0|0 + extended|global_mmlu_instruct_all_ben:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_ben:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_ben:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_ben:medical_genetics|0|0 + extended|global_mmlu_instruct_all_deu:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_deu:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_deu:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_deu:medical_genetics|0|0 + extended|global_mmlu_instruct_all_spa:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_spa:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_spa:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_spa:medical_genetics|0|0 + extended|global_mmlu_instruct_all_eng:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_eng:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_eng:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_eng:medical_genetics|0|0 + extended|global_mmlu_instruct_all_fra:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_fra:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_fra:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_fra:medical_genetics|0|0 + extended|global_mmlu_instruct_all_hin:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_hin:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_hin:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_hin:medical_genetics|0|0 + extended|global_mmlu_instruct_all_ind:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_ind:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_ind:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_ind:medical_genetics|0|0 + extended|global_mmlu_instruct_all_ita:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_ita:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_ita:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_ita:medical_genetics|0|0 + extended|global_mmlu_instruct_all_jpn:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_jpn:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_jpn:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_jpn:medical_genetics|0|0 + extended|global_mmlu_instruct_all_nld:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_nld:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_nld:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_nld:medical_genetics|0|0 + extended|global_mmlu_instruct_all_nor:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_nor:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_nor:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_nor:medical_genetics|0|0 + extended|global_mmlu_instruct_all_pol:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_pol:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_pol:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_pol:medical_genetics|0|0 + extended|global_mmlu_instruct_all_por:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_por:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_por:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_por:medical_genetics|0|0 + extended|global_mmlu_instruct_all_rus:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_rus:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_rus:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_rus:medical_genetics|0|0 + extended|global_mmlu_instruct_all_srp:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_srp:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_srp:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_srp:medical_genetics|0|0 + extended|global_mmlu_instruct_all_swe:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_swe:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_swe:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_swe:medical_genetics|0|0 + extended|global_mmlu_instruct_all_swa:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_swa:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_swa:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_swa:medical_genetics|0|0 + extended|global_mmlu_instruct_all_tel:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_tel:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_tel:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_tel:medical_genetics|0|0 + extended|global_mmlu_instruct_all_tha:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_tha:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_tha:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_tha:medical_genetics|0|0 + extended|global_mmlu_instruct_all_tur:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_tur:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_tur:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_tur:medical_genetics|0|0 + extended|global_mmlu_instruct_all_ukr:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_ukr:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_ukr:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_ukr:medical_genetics|0|0 + extended|global_mmlu_instruct_all_urd:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_urd:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_urd:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_urd:medical_genetics|0|0 + extended|global_mmlu_instruct_all_vie:medical_genetics|0|0 + extended|global_mmlu_instruct_ca_vie:medical_genetics|0|0 + extended|global_mmlu_instruct_cs_vie:medical_genetics|0|0 + extended|global_mmlu_instruct_unk_vie:medical_genetics|0|0 + extended|global_mmlu_instruct_all_ara:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_ara:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_ara:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_ara:miscellaneous|0|0 + extended|global_mmlu_instruct_all_ben:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_ben:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_ben:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_ben:miscellaneous|0|0 + extended|global_mmlu_instruct_all_deu:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_deu:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_deu:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_deu:miscellaneous|0|0 + extended|global_mmlu_instruct_all_spa:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_spa:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_spa:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_spa:miscellaneous|0|0 + extended|global_mmlu_instruct_all_eng:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_eng:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_eng:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_eng:miscellaneous|0|0 + extended|global_mmlu_instruct_all_fra:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_fra:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_fra:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_fra:miscellaneous|0|0 + extended|global_mmlu_instruct_all_hin:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_hin:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_hin:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_hin:miscellaneous|0|0 + extended|global_mmlu_instruct_all_ind:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_ind:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_ind:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_ind:miscellaneous|0|0 + extended|global_mmlu_instruct_all_ita:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_ita:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_ita:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_ita:miscellaneous|0|0 + extended|global_mmlu_instruct_all_jpn:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_jpn:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_jpn:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_jpn:miscellaneous|0|0 + extended|global_mmlu_instruct_all_nld:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_nld:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_nld:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_nld:miscellaneous|0|0 + extended|global_mmlu_instruct_all_nor:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_nor:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_nor:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_nor:miscellaneous|0|0 + extended|global_mmlu_instruct_all_pol:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_pol:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_pol:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_pol:miscellaneous|0|0 + extended|global_mmlu_instruct_all_por:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_por:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_por:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_por:miscellaneous|0|0 + extended|global_mmlu_instruct_all_rus:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_rus:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_rus:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_rus:miscellaneous|0|0 + extended|global_mmlu_instruct_all_srp:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_srp:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_srp:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_srp:miscellaneous|0|0 + extended|global_mmlu_instruct_all_swe:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_swe:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_swe:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_swe:miscellaneous|0|0 + extended|global_mmlu_instruct_all_swa:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_swa:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_swa:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_swa:miscellaneous|0|0 + extended|global_mmlu_instruct_all_tel:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_tel:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_tel:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_tel:miscellaneous|0|0 + extended|global_mmlu_instruct_all_tha:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_tha:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_tha:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_tha:miscellaneous|0|0 + extended|global_mmlu_instruct_all_tur:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_tur:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_tur:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_tur:miscellaneous|0|0 + extended|global_mmlu_instruct_all_ukr:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_ukr:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_ukr:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_ukr:miscellaneous|0|0 + extended|global_mmlu_instruct_all_urd:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_urd:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_urd:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_urd:miscellaneous|0|0 + extended|global_mmlu_instruct_all_vie:miscellaneous|0|0 + extended|global_mmlu_instruct_ca_vie:miscellaneous|0|0 + extended|global_mmlu_instruct_cs_vie:miscellaneous|0|0 + extended|global_mmlu_instruct_unk_vie:miscellaneous|0|0 + extended|global_mmlu_instruct_all_ara:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_ara:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_ara:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_ara:moral_disputes|0|0 + extended|global_mmlu_instruct_all_ben:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_ben:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_ben:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_ben:moral_disputes|0|0 + extended|global_mmlu_instruct_all_deu:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_deu:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_deu:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_deu:moral_disputes|0|0 + extended|global_mmlu_instruct_all_spa:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_spa:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_spa:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_spa:moral_disputes|0|0 + extended|global_mmlu_instruct_all_eng:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_eng:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_eng:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_eng:moral_disputes|0|0 + extended|global_mmlu_instruct_all_fra:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_fra:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_fra:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_fra:moral_disputes|0|0 + extended|global_mmlu_instruct_all_hin:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_hin:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_hin:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_hin:moral_disputes|0|0 + extended|global_mmlu_instruct_all_ind:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_ind:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_ind:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_ind:moral_disputes|0|0 + extended|global_mmlu_instruct_all_ita:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_ita:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_ita:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_ita:moral_disputes|0|0 + extended|global_mmlu_instruct_all_jpn:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_jpn:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_jpn:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_jpn:moral_disputes|0|0 + extended|global_mmlu_instruct_all_nld:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_nld:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_nld:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_nld:moral_disputes|0|0 + extended|global_mmlu_instruct_all_nor:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_nor:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_nor:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_nor:moral_disputes|0|0 + extended|global_mmlu_instruct_all_pol:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_pol:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_pol:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_pol:moral_disputes|0|0 + extended|global_mmlu_instruct_all_por:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_por:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_por:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_por:moral_disputes|0|0 + extended|global_mmlu_instruct_all_rus:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_rus:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_rus:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_rus:moral_disputes|0|0 + extended|global_mmlu_instruct_all_srp:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_srp:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_srp:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_srp:moral_disputes|0|0 + extended|global_mmlu_instruct_all_swe:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_swe:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_swe:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_swe:moral_disputes|0|0 + extended|global_mmlu_instruct_all_swa:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_swa:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_swa:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_swa:moral_disputes|0|0 + extended|global_mmlu_instruct_all_tel:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_tel:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_tel:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_tel:moral_disputes|0|0 + extended|global_mmlu_instruct_all_tha:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_tha:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_tha:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_tha:moral_disputes|0|0 + extended|global_mmlu_instruct_all_tur:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_tur:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_tur:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_tur:moral_disputes|0|0 + extended|global_mmlu_instruct_all_ukr:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_ukr:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_ukr:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_ukr:moral_disputes|0|0 + extended|global_mmlu_instruct_all_urd:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_urd:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_urd:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_urd:moral_disputes|0|0 + extended|global_mmlu_instruct_all_vie:moral_disputes|0|0 + extended|global_mmlu_instruct_ca_vie:moral_disputes|0|0 + extended|global_mmlu_instruct_cs_vie:moral_disputes|0|0 + extended|global_mmlu_instruct_unk_vie:moral_disputes|0|0 + extended|global_mmlu_instruct_all_ara:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_ara:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_ara:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_ara:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_ben:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_ben:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_ben:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_ben:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_deu:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_deu:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_deu:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_deu:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_spa:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_spa:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_spa:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_spa:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_eng:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_eng:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_eng:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_eng:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_fra:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_fra:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_fra:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_fra:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_hin:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_hin:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_hin:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_hin:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_ind:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_ind:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_ind:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_ind:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_ita:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_ita:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_ita:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_ita:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_jpn:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_jpn:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_jpn:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_jpn:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_nld:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_nld:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_nld:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_nld:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_nor:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_nor:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_nor:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_nor:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_pol:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_pol:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_pol:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_pol:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_por:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_por:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_por:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_por:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_rus:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_rus:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_rus:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_rus:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_srp:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_srp:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_srp:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_srp:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_swe:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_swe:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_swe:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_swe:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_swa:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_swa:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_swa:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_swa:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_tel:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_tel:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_tel:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_tel:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_tha:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_tha:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_tha:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_tha:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_tur:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_tur:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_tur:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_tur:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_ukr:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_ukr:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_ukr:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_ukr:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_urd:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_urd:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_urd:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_urd:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_vie:moral_scenarios|0|0 + extended|global_mmlu_instruct_ca_vie:moral_scenarios|0|0 + extended|global_mmlu_instruct_cs_vie:moral_scenarios|0|0 + extended|global_mmlu_instruct_unk_vie:moral_scenarios|0|0 + extended|global_mmlu_instruct_all_ara:nutrition|0|0 + extended|global_mmlu_instruct_ca_ara:nutrition|0|0 + extended|global_mmlu_instruct_cs_ara:nutrition|0|0 + extended|global_mmlu_instruct_unk_ara:nutrition|0|0 + extended|global_mmlu_instruct_all_ben:nutrition|0|0 + extended|global_mmlu_instruct_ca_ben:nutrition|0|0 + extended|global_mmlu_instruct_cs_ben:nutrition|0|0 + extended|global_mmlu_instruct_unk_ben:nutrition|0|0 + extended|global_mmlu_instruct_all_deu:nutrition|0|0 + extended|global_mmlu_instruct_ca_deu:nutrition|0|0 + extended|global_mmlu_instruct_cs_deu:nutrition|0|0 + extended|global_mmlu_instruct_unk_deu:nutrition|0|0 + extended|global_mmlu_instruct_all_spa:nutrition|0|0 + extended|global_mmlu_instruct_ca_spa:nutrition|0|0 + extended|global_mmlu_instruct_cs_spa:nutrition|0|0 + extended|global_mmlu_instruct_unk_spa:nutrition|0|0 + extended|global_mmlu_instruct_all_eng:nutrition|0|0 + extended|global_mmlu_instruct_ca_eng:nutrition|0|0 + extended|global_mmlu_instruct_cs_eng:nutrition|0|0 + extended|global_mmlu_instruct_unk_eng:nutrition|0|0 + extended|global_mmlu_instruct_all_fra:nutrition|0|0 + extended|global_mmlu_instruct_ca_fra:nutrition|0|0 + extended|global_mmlu_instruct_cs_fra:nutrition|0|0 + extended|global_mmlu_instruct_unk_fra:nutrition|0|0 + extended|global_mmlu_instruct_all_hin:nutrition|0|0 + extended|global_mmlu_instruct_ca_hin:nutrition|0|0 + extended|global_mmlu_instruct_cs_hin:nutrition|0|0 + extended|global_mmlu_instruct_unk_hin:nutrition|0|0 + extended|global_mmlu_instruct_all_ind:nutrition|0|0 + extended|global_mmlu_instruct_ca_ind:nutrition|0|0 + extended|global_mmlu_instruct_cs_ind:nutrition|0|0 + extended|global_mmlu_instruct_unk_ind:nutrition|0|0 + extended|global_mmlu_instruct_all_ita:nutrition|0|0 + extended|global_mmlu_instruct_ca_ita:nutrition|0|0 + extended|global_mmlu_instruct_cs_ita:nutrition|0|0 + extended|global_mmlu_instruct_unk_ita:nutrition|0|0 + extended|global_mmlu_instruct_all_jpn:nutrition|0|0 + extended|global_mmlu_instruct_ca_jpn:nutrition|0|0 + extended|global_mmlu_instruct_cs_jpn:nutrition|0|0 + extended|global_mmlu_instruct_unk_jpn:nutrition|0|0 + extended|global_mmlu_instruct_all_nld:nutrition|0|0 + extended|global_mmlu_instruct_ca_nld:nutrition|0|0 + extended|global_mmlu_instruct_cs_nld:nutrition|0|0 + extended|global_mmlu_instruct_unk_nld:nutrition|0|0 + extended|global_mmlu_instruct_all_nor:nutrition|0|0 + extended|global_mmlu_instruct_ca_nor:nutrition|0|0 + extended|global_mmlu_instruct_cs_nor:nutrition|0|0 + extended|global_mmlu_instruct_unk_nor:nutrition|0|0 + extended|global_mmlu_instruct_all_pol:nutrition|0|0 + extended|global_mmlu_instruct_ca_pol:nutrition|0|0 + extended|global_mmlu_instruct_cs_pol:nutrition|0|0 + extended|global_mmlu_instruct_unk_pol:nutrition|0|0 + extended|global_mmlu_instruct_all_por:nutrition|0|0 + extended|global_mmlu_instruct_ca_por:nutrition|0|0 + extended|global_mmlu_instruct_cs_por:nutrition|0|0 + extended|global_mmlu_instruct_unk_por:nutrition|0|0 + extended|global_mmlu_instruct_all_rus:nutrition|0|0 + extended|global_mmlu_instruct_ca_rus:nutrition|0|0 + extended|global_mmlu_instruct_cs_rus:nutrition|0|0 + extended|global_mmlu_instruct_unk_rus:nutrition|0|0 + extended|global_mmlu_instruct_all_srp:nutrition|0|0 + extended|global_mmlu_instruct_ca_srp:nutrition|0|0 + extended|global_mmlu_instruct_cs_srp:nutrition|0|0 + extended|global_mmlu_instruct_unk_srp:nutrition|0|0 + extended|global_mmlu_instruct_all_swe:nutrition|0|0 + extended|global_mmlu_instruct_ca_swe:nutrition|0|0 + extended|global_mmlu_instruct_cs_swe:nutrition|0|0 + extended|global_mmlu_instruct_unk_swe:nutrition|0|0 + extended|global_mmlu_instruct_all_swa:nutrition|0|0 + extended|global_mmlu_instruct_ca_swa:nutrition|0|0 + extended|global_mmlu_instruct_cs_swa:nutrition|0|0 + extended|global_mmlu_instruct_unk_swa:nutrition|0|0 + extended|global_mmlu_instruct_all_tel:nutrition|0|0 + extended|global_mmlu_instruct_ca_tel:nutrition|0|0 + extended|global_mmlu_instruct_cs_tel:nutrition|0|0 + extended|global_mmlu_instruct_unk_tel:nutrition|0|0 + extended|global_mmlu_instruct_all_tha:nutrition|0|0 + extended|global_mmlu_instruct_ca_tha:nutrition|0|0 + extended|global_mmlu_instruct_cs_tha:nutrition|0|0 + extended|global_mmlu_instruct_unk_tha:nutrition|0|0 + extended|global_mmlu_instruct_all_tur:nutrition|0|0 + extended|global_mmlu_instruct_ca_tur:nutrition|0|0 + extended|global_mmlu_instruct_cs_tur:nutrition|0|0 + extended|global_mmlu_instruct_unk_tur:nutrition|0|0 + extended|global_mmlu_instruct_all_ukr:nutrition|0|0 + extended|global_mmlu_instruct_ca_ukr:nutrition|0|0 + extended|global_mmlu_instruct_cs_ukr:nutrition|0|0 + extended|global_mmlu_instruct_unk_ukr:nutrition|0|0 + extended|global_mmlu_instruct_all_urd:nutrition|0|0 + extended|global_mmlu_instruct_ca_urd:nutrition|0|0 + extended|global_mmlu_instruct_cs_urd:nutrition|0|0 + extended|global_mmlu_instruct_unk_urd:nutrition|0|0 + extended|global_mmlu_instruct_all_vie:nutrition|0|0 + extended|global_mmlu_instruct_ca_vie:nutrition|0|0 + extended|global_mmlu_instruct_cs_vie:nutrition|0|0 + extended|global_mmlu_instruct_unk_vie:nutrition|0|0 + extended|global_mmlu_instruct_all_ara:philosophy|0|0 + extended|global_mmlu_instruct_ca_ara:philosophy|0|0 + extended|global_mmlu_instruct_cs_ara:philosophy|0|0 + extended|global_mmlu_instruct_unk_ara:philosophy|0|0 + extended|global_mmlu_instruct_all_ben:philosophy|0|0 + extended|global_mmlu_instruct_ca_ben:philosophy|0|0 + extended|global_mmlu_instruct_cs_ben:philosophy|0|0 + extended|global_mmlu_instruct_unk_ben:philosophy|0|0 + extended|global_mmlu_instruct_all_deu:philosophy|0|0 + extended|global_mmlu_instruct_ca_deu:philosophy|0|0 + extended|global_mmlu_instruct_cs_deu:philosophy|0|0 + extended|global_mmlu_instruct_unk_deu:philosophy|0|0 + extended|global_mmlu_instruct_all_spa:philosophy|0|0 + extended|global_mmlu_instruct_ca_spa:philosophy|0|0 + extended|global_mmlu_instruct_cs_spa:philosophy|0|0 + extended|global_mmlu_instruct_unk_spa:philosophy|0|0 + extended|global_mmlu_instruct_all_eng:philosophy|0|0 + extended|global_mmlu_instruct_ca_eng:philosophy|0|0 + extended|global_mmlu_instruct_cs_eng:philosophy|0|0 + extended|global_mmlu_instruct_unk_eng:philosophy|0|0 + extended|global_mmlu_instruct_all_fra:philosophy|0|0 + extended|global_mmlu_instruct_ca_fra:philosophy|0|0 + extended|global_mmlu_instruct_cs_fra:philosophy|0|0 + extended|global_mmlu_instruct_unk_fra:philosophy|0|0 + extended|global_mmlu_instruct_all_hin:philosophy|0|0 + extended|global_mmlu_instruct_ca_hin:philosophy|0|0 + extended|global_mmlu_instruct_cs_hin:philosophy|0|0 + extended|global_mmlu_instruct_unk_hin:philosophy|0|0 + extended|global_mmlu_instruct_all_ind:philosophy|0|0 + extended|global_mmlu_instruct_ca_ind:philosophy|0|0 + extended|global_mmlu_instruct_cs_ind:philosophy|0|0 + extended|global_mmlu_instruct_unk_ind:philosophy|0|0 + extended|global_mmlu_instruct_all_ita:philosophy|0|0 + extended|global_mmlu_instruct_ca_ita:philosophy|0|0 + extended|global_mmlu_instruct_cs_ita:philosophy|0|0 + extended|global_mmlu_instruct_unk_ita:philosophy|0|0 + extended|global_mmlu_instruct_all_jpn:philosophy|0|0 + extended|global_mmlu_instruct_ca_jpn:philosophy|0|0 + extended|global_mmlu_instruct_cs_jpn:philosophy|0|0 + extended|global_mmlu_instruct_unk_jpn:philosophy|0|0 + extended|global_mmlu_instruct_all_nld:philosophy|0|0 + extended|global_mmlu_instruct_ca_nld:philosophy|0|0 + extended|global_mmlu_instruct_cs_nld:philosophy|0|0 + extended|global_mmlu_instruct_unk_nld:philosophy|0|0 + extended|global_mmlu_instruct_all_nor:philosophy|0|0 + extended|global_mmlu_instruct_ca_nor:philosophy|0|0 + extended|global_mmlu_instruct_cs_nor:philosophy|0|0 + extended|global_mmlu_instruct_unk_nor:philosophy|0|0 + extended|global_mmlu_instruct_all_pol:philosophy|0|0 + extended|global_mmlu_instruct_ca_pol:philosophy|0|0 + extended|global_mmlu_instruct_cs_pol:philosophy|0|0 + extended|global_mmlu_instruct_unk_pol:philosophy|0|0 + extended|global_mmlu_instruct_all_por:philosophy|0|0 + extended|global_mmlu_instruct_ca_por:philosophy|0|0 + extended|global_mmlu_instruct_cs_por:philosophy|0|0 + extended|global_mmlu_instruct_unk_por:philosophy|0|0 + extended|global_mmlu_instruct_all_rus:philosophy|0|0 + extended|global_mmlu_instruct_ca_rus:philosophy|0|0 + extended|global_mmlu_instruct_cs_rus:philosophy|0|0 + extended|global_mmlu_instruct_unk_rus:philosophy|0|0 + extended|global_mmlu_instruct_all_srp:philosophy|0|0 + extended|global_mmlu_instruct_ca_srp:philosophy|0|0 + extended|global_mmlu_instruct_cs_srp:philosophy|0|0 + extended|global_mmlu_instruct_unk_srp:philosophy|0|0 + extended|global_mmlu_instruct_all_swe:philosophy|0|0 + extended|global_mmlu_instruct_ca_swe:philosophy|0|0 + extended|global_mmlu_instruct_cs_swe:philosophy|0|0 + extended|global_mmlu_instruct_unk_swe:philosophy|0|0 + extended|global_mmlu_instruct_all_swa:philosophy|0|0 + extended|global_mmlu_instruct_ca_swa:philosophy|0|0 + extended|global_mmlu_instruct_cs_swa:philosophy|0|0 + extended|global_mmlu_instruct_unk_swa:philosophy|0|0 + extended|global_mmlu_instruct_all_tel:philosophy|0|0 + extended|global_mmlu_instruct_ca_tel:philosophy|0|0 + extended|global_mmlu_instruct_cs_tel:philosophy|0|0 + extended|global_mmlu_instruct_unk_tel:philosophy|0|0 + extended|global_mmlu_instruct_all_tha:philosophy|0|0 + extended|global_mmlu_instruct_ca_tha:philosophy|0|0 + extended|global_mmlu_instruct_cs_tha:philosophy|0|0 + extended|global_mmlu_instruct_unk_tha:philosophy|0|0 + extended|global_mmlu_instruct_all_tur:philosophy|0|0 + extended|global_mmlu_instruct_ca_tur:philosophy|0|0 + extended|global_mmlu_instruct_cs_tur:philosophy|0|0 + extended|global_mmlu_instruct_unk_tur:philosophy|0|0 + extended|global_mmlu_instruct_all_ukr:philosophy|0|0 + extended|global_mmlu_instruct_ca_ukr:philosophy|0|0 + extended|global_mmlu_instruct_cs_ukr:philosophy|0|0 + extended|global_mmlu_instruct_unk_ukr:philosophy|0|0 + extended|global_mmlu_instruct_all_urd:philosophy|0|0 + extended|global_mmlu_instruct_ca_urd:philosophy|0|0 + extended|global_mmlu_instruct_cs_urd:philosophy|0|0 + extended|global_mmlu_instruct_unk_urd:philosophy|0|0 + extended|global_mmlu_instruct_all_vie:philosophy|0|0 + extended|global_mmlu_instruct_ca_vie:philosophy|0|0 + extended|global_mmlu_instruct_cs_vie:philosophy|0|0 + extended|global_mmlu_instruct_unk_vie:philosophy|0|0 + extended|global_mmlu_instruct_all_ara:prehistory|0|0 + extended|global_mmlu_instruct_ca_ara:prehistory|0|0 + extended|global_mmlu_instruct_cs_ara:prehistory|0|0 + extended|global_mmlu_instruct_unk_ara:prehistory|0|0 + extended|global_mmlu_instruct_all_ben:prehistory|0|0 + extended|global_mmlu_instruct_ca_ben:prehistory|0|0 + extended|global_mmlu_instruct_cs_ben:prehistory|0|0 + extended|global_mmlu_instruct_unk_ben:prehistory|0|0 + extended|global_mmlu_instruct_all_deu:prehistory|0|0 + extended|global_mmlu_instruct_ca_deu:prehistory|0|0 + extended|global_mmlu_instruct_cs_deu:prehistory|0|0 + extended|global_mmlu_instruct_unk_deu:prehistory|0|0 + extended|global_mmlu_instruct_all_spa:prehistory|0|0 + extended|global_mmlu_instruct_ca_spa:prehistory|0|0 + extended|global_mmlu_instruct_cs_spa:prehistory|0|0 + extended|global_mmlu_instruct_unk_spa:prehistory|0|0 + extended|global_mmlu_instruct_all_eng:prehistory|0|0 + extended|global_mmlu_instruct_ca_eng:prehistory|0|0 + extended|global_mmlu_instruct_cs_eng:prehistory|0|0 + extended|global_mmlu_instruct_unk_eng:prehistory|0|0 + extended|global_mmlu_instruct_all_fra:prehistory|0|0 + extended|global_mmlu_instruct_ca_fra:prehistory|0|0 + extended|global_mmlu_instruct_cs_fra:prehistory|0|0 + extended|global_mmlu_instruct_unk_fra:prehistory|0|0 + extended|global_mmlu_instruct_all_hin:prehistory|0|0 + extended|global_mmlu_instruct_ca_hin:prehistory|0|0 + extended|global_mmlu_instruct_cs_hin:prehistory|0|0 + extended|global_mmlu_instruct_unk_hin:prehistory|0|0 + extended|global_mmlu_instruct_all_ind:prehistory|0|0 + extended|global_mmlu_instruct_ca_ind:prehistory|0|0 + extended|global_mmlu_instruct_cs_ind:prehistory|0|0 + extended|global_mmlu_instruct_unk_ind:prehistory|0|0 + extended|global_mmlu_instruct_all_ita:prehistory|0|0 + extended|global_mmlu_instruct_ca_ita:prehistory|0|0 + extended|global_mmlu_instruct_cs_ita:prehistory|0|0 + extended|global_mmlu_instruct_unk_ita:prehistory|0|0 + extended|global_mmlu_instruct_all_jpn:prehistory|0|0 + extended|global_mmlu_instruct_ca_jpn:prehistory|0|0 + extended|global_mmlu_instruct_cs_jpn:prehistory|0|0 + extended|global_mmlu_instruct_unk_jpn:prehistory|0|0 + extended|global_mmlu_instruct_all_nld:prehistory|0|0 + extended|global_mmlu_instruct_ca_nld:prehistory|0|0 + extended|global_mmlu_instruct_cs_nld:prehistory|0|0 + extended|global_mmlu_instruct_unk_nld:prehistory|0|0 + extended|global_mmlu_instruct_all_nor:prehistory|0|0 + extended|global_mmlu_instruct_ca_nor:prehistory|0|0 + extended|global_mmlu_instruct_cs_nor:prehistory|0|0 + extended|global_mmlu_instruct_unk_nor:prehistory|0|0 + extended|global_mmlu_instruct_all_pol:prehistory|0|0 + extended|global_mmlu_instruct_ca_pol:prehistory|0|0 + extended|global_mmlu_instruct_cs_pol:prehistory|0|0 + extended|global_mmlu_instruct_unk_pol:prehistory|0|0 + extended|global_mmlu_instruct_all_por:prehistory|0|0 + extended|global_mmlu_instruct_ca_por:prehistory|0|0 + extended|global_mmlu_instruct_cs_por:prehistory|0|0 + extended|global_mmlu_instruct_unk_por:prehistory|0|0 + extended|global_mmlu_instruct_all_rus:prehistory|0|0 + extended|global_mmlu_instruct_ca_rus:prehistory|0|0 + extended|global_mmlu_instruct_cs_rus:prehistory|0|0 + extended|global_mmlu_instruct_unk_rus:prehistory|0|0 + extended|global_mmlu_instruct_all_srp:prehistory|0|0 + extended|global_mmlu_instruct_ca_srp:prehistory|0|0 + extended|global_mmlu_instruct_cs_srp:prehistory|0|0 + extended|global_mmlu_instruct_unk_srp:prehistory|0|0 + extended|global_mmlu_instruct_all_swe:prehistory|0|0 + extended|global_mmlu_instruct_ca_swe:prehistory|0|0 + extended|global_mmlu_instruct_cs_swe:prehistory|0|0 + extended|global_mmlu_instruct_unk_swe:prehistory|0|0 + extended|global_mmlu_instruct_all_swa:prehistory|0|0 + extended|global_mmlu_instruct_ca_swa:prehistory|0|0 + extended|global_mmlu_instruct_cs_swa:prehistory|0|0 + extended|global_mmlu_instruct_unk_swa:prehistory|0|0 + extended|global_mmlu_instruct_all_tel:prehistory|0|0 + extended|global_mmlu_instruct_ca_tel:prehistory|0|0 + extended|global_mmlu_instruct_cs_tel:prehistory|0|0 + extended|global_mmlu_instruct_unk_tel:prehistory|0|0 + extended|global_mmlu_instruct_all_tha:prehistory|0|0 + extended|global_mmlu_instruct_ca_tha:prehistory|0|0 + extended|global_mmlu_instruct_cs_tha:prehistory|0|0 + extended|global_mmlu_instruct_unk_tha:prehistory|0|0 + extended|global_mmlu_instruct_all_tur:prehistory|0|0 + extended|global_mmlu_instruct_ca_tur:prehistory|0|0 + extended|global_mmlu_instruct_cs_tur:prehistory|0|0 + extended|global_mmlu_instruct_unk_tur:prehistory|0|0 + extended|global_mmlu_instruct_all_ukr:prehistory|0|0 + extended|global_mmlu_instruct_ca_ukr:prehistory|0|0 + extended|global_mmlu_instruct_cs_ukr:prehistory|0|0 + extended|global_mmlu_instruct_unk_ukr:prehistory|0|0 + extended|global_mmlu_instruct_all_urd:prehistory|0|0 + extended|global_mmlu_instruct_ca_urd:prehistory|0|0 + extended|global_mmlu_instruct_cs_urd:prehistory|0|0 + extended|global_mmlu_instruct_unk_urd:prehistory|0|0 + extended|global_mmlu_instruct_all_vie:prehistory|0|0 + extended|global_mmlu_instruct_ca_vie:prehistory|0|0 + extended|global_mmlu_instruct_cs_vie:prehistory|0|0 + extended|global_mmlu_instruct_unk_vie:prehistory|0|0 + extended|global_mmlu_instruct_all_ara:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_ara:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_ara:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_ara:professional_accounting|0|0 + extended|global_mmlu_instruct_all_ben:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_ben:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_ben:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_ben:professional_accounting|0|0 + extended|global_mmlu_instruct_all_deu:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_deu:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_deu:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_deu:professional_accounting|0|0 + extended|global_mmlu_instruct_all_spa:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_spa:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_spa:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_spa:professional_accounting|0|0 + extended|global_mmlu_instruct_all_eng:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_eng:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_eng:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_eng:professional_accounting|0|0 + extended|global_mmlu_instruct_all_fra:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_fra:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_fra:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_fra:professional_accounting|0|0 + extended|global_mmlu_instruct_all_hin:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_hin:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_hin:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_hin:professional_accounting|0|0 + extended|global_mmlu_instruct_all_ind:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_ind:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_ind:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_ind:professional_accounting|0|0 + extended|global_mmlu_instruct_all_ita:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_ita:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_ita:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_ita:professional_accounting|0|0 + extended|global_mmlu_instruct_all_jpn:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_jpn:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_jpn:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_jpn:professional_accounting|0|0 + extended|global_mmlu_instruct_all_nld:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_nld:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_nld:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_nld:professional_accounting|0|0 + extended|global_mmlu_instruct_all_nor:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_nor:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_nor:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_nor:professional_accounting|0|0 + extended|global_mmlu_instruct_all_pol:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_pol:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_pol:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_pol:professional_accounting|0|0 + extended|global_mmlu_instruct_all_por:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_por:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_por:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_por:professional_accounting|0|0 + extended|global_mmlu_instruct_all_rus:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_rus:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_rus:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_rus:professional_accounting|0|0 + extended|global_mmlu_instruct_all_srp:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_srp:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_srp:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_srp:professional_accounting|0|0 + extended|global_mmlu_instruct_all_swe:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_swe:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_swe:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_swe:professional_accounting|0|0 + extended|global_mmlu_instruct_all_swa:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_swa:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_swa:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_swa:professional_accounting|0|0 + extended|global_mmlu_instruct_all_tel:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_tel:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_tel:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_tel:professional_accounting|0|0 + extended|global_mmlu_instruct_all_tha:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_tha:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_tha:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_tha:professional_accounting|0|0 + extended|global_mmlu_instruct_all_tur:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_tur:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_tur:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_tur:professional_accounting|0|0 + extended|global_mmlu_instruct_all_ukr:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_ukr:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_ukr:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_ukr:professional_accounting|0|0 + extended|global_mmlu_instruct_all_urd:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_urd:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_urd:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_urd:professional_accounting|0|0 + extended|global_mmlu_instruct_all_vie:professional_accounting|0|0 + extended|global_mmlu_instruct_ca_vie:professional_accounting|0|0 + extended|global_mmlu_instruct_cs_vie:professional_accounting|0|0 + extended|global_mmlu_instruct_unk_vie:professional_accounting|0|0 + extended|global_mmlu_instruct_all_ara:professional_law|0|0 + extended|global_mmlu_instruct_ca_ara:professional_law|0|0 + extended|global_mmlu_instruct_cs_ara:professional_law|0|0 + extended|global_mmlu_instruct_unk_ara:professional_law|0|0 + extended|global_mmlu_instruct_all_ben:professional_law|0|0 + extended|global_mmlu_instruct_ca_ben:professional_law|0|0 + extended|global_mmlu_instruct_cs_ben:professional_law|0|0 + extended|global_mmlu_instruct_unk_ben:professional_law|0|0 + extended|global_mmlu_instruct_all_deu:professional_law|0|0 + extended|global_mmlu_instruct_ca_deu:professional_law|0|0 + extended|global_mmlu_instruct_cs_deu:professional_law|0|0 + extended|global_mmlu_instruct_unk_deu:professional_law|0|0 + extended|global_mmlu_instruct_all_spa:professional_law|0|0 + extended|global_mmlu_instruct_ca_spa:professional_law|0|0 + extended|global_mmlu_instruct_cs_spa:professional_law|0|0 + extended|global_mmlu_instruct_unk_spa:professional_law|0|0 + extended|global_mmlu_instruct_all_eng:professional_law|0|0 + extended|global_mmlu_instruct_ca_eng:professional_law|0|0 + extended|global_mmlu_instruct_cs_eng:professional_law|0|0 + extended|global_mmlu_instruct_unk_eng:professional_law|0|0 + extended|global_mmlu_instruct_all_fra:professional_law|0|0 + extended|global_mmlu_instruct_ca_fra:professional_law|0|0 + extended|global_mmlu_instruct_cs_fra:professional_law|0|0 + extended|global_mmlu_instruct_unk_fra:professional_law|0|0 + extended|global_mmlu_instruct_all_hin:professional_law|0|0 + extended|global_mmlu_instruct_ca_hin:professional_law|0|0 + extended|global_mmlu_instruct_cs_hin:professional_law|0|0 + extended|global_mmlu_instruct_unk_hin:professional_law|0|0 + extended|global_mmlu_instruct_all_ind:professional_law|0|0 + extended|global_mmlu_instruct_ca_ind:professional_law|0|0 + extended|global_mmlu_instruct_cs_ind:professional_law|0|0 + extended|global_mmlu_instruct_unk_ind:professional_law|0|0 + extended|global_mmlu_instruct_all_ita:professional_law|0|0 + extended|global_mmlu_instruct_ca_ita:professional_law|0|0 + extended|global_mmlu_instruct_cs_ita:professional_law|0|0 + extended|global_mmlu_instruct_unk_ita:professional_law|0|0 + extended|global_mmlu_instruct_all_jpn:professional_law|0|0 + extended|global_mmlu_instruct_ca_jpn:professional_law|0|0 + extended|global_mmlu_instruct_cs_jpn:professional_law|0|0 + extended|global_mmlu_instruct_unk_jpn:professional_law|0|0 + extended|global_mmlu_instruct_all_nld:professional_law|0|0 + extended|global_mmlu_instruct_ca_nld:professional_law|0|0 + extended|global_mmlu_instruct_cs_nld:professional_law|0|0 + extended|global_mmlu_instruct_unk_nld:professional_law|0|0 + extended|global_mmlu_instruct_all_nor:professional_law|0|0 + extended|global_mmlu_instruct_ca_nor:professional_law|0|0 + extended|global_mmlu_instruct_cs_nor:professional_law|0|0 + extended|global_mmlu_instruct_unk_nor:professional_law|0|0 + extended|global_mmlu_instruct_all_pol:professional_law|0|0 + extended|global_mmlu_instruct_ca_pol:professional_law|0|0 + extended|global_mmlu_instruct_cs_pol:professional_law|0|0 + extended|global_mmlu_instruct_unk_pol:professional_law|0|0 + extended|global_mmlu_instruct_all_por:professional_law|0|0 + extended|global_mmlu_instruct_ca_por:professional_law|0|0 + extended|global_mmlu_instruct_cs_por:professional_law|0|0 + extended|global_mmlu_instruct_unk_por:professional_law|0|0 + extended|global_mmlu_instruct_all_rus:professional_law|0|0 + extended|global_mmlu_instruct_ca_rus:professional_law|0|0 + extended|global_mmlu_instruct_cs_rus:professional_law|0|0 + extended|global_mmlu_instruct_unk_rus:professional_law|0|0 + extended|global_mmlu_instruct_all_srp:professional_law|0|0 + extended|global_mmlu_instruct_ca_srp:professional_law|0|0 + extended|global_mmlu_instruct_cs_srp:professional_law|0|0 + extended|global_mmlu_instruct_unk_srp:professional_law|0|0 + extended|global_mmlu_instruct_all_swe:professional_law|0|0 + extended|global_mmlu_instruct_ca_swe:professional_law|0|0 + extended|global_mmlu_instruct_cs_swe:professional_law|0|0 + extended|global_mmlu_instruct_unk_swe:professional_law|0|0 + extended|global_mmlu_instruct_all_swa:professional_law|0|0 + extended|global_mmlu_instruct_ca_swa:professional_law|0|0 + extended|global_mmlu_instruct_cs_swa:professional_law|0|0 + extended|global_mmlu_instruct_unk_swa:professional_law|0|0 + extended|global_mmlu_instruct_all_tel:professional_law|0|0 + extended|global_mmlu_instruct_ca_tel:professional_law|0|0 + extended|global_mmlu_instruct_cs_tel:professional_law|0|0 + extended|global_mmlu_instruct_unk_tel:professional_law|0|0 + extended|global_mmlu_instruct_all_tha:professional_law|0|0 + extended|global_mmlu_instruct_ca_tha:professional_law|0|0 + extended|global_mmlu_instruct_cs_tha:professional_law|0|0 + extended|global_mmlu_instruct_unk_tha:professional_law|0|0 + extended|global_mmlu_instruct_all_tur:professional_law|0|0 + extended|global_mmlu_instruct_ca_tur:professional_law|0|0 + extended|global_mmlu_instruct_cs_tur:professional_law|0|0 + extended|global_mmlu_instruct_unk_tur:professional_law|0|0 + extended|global_mmlu_instruct_all_ukr:professional_law|0|0 + extended|global_mmlu_instruct_ca_ukr:professional_law|0|0 + extended|global_mmlu_instruct_cs_ukr:professional_law|0|0 + extended|global_mmlu_instruct_unk_ukr:professional_law|0|0 + extended|global_mmlu_instruct_all_urd:professional_law|0|0 + extended|global_mmlu_instruct_ca_urd:professional_law|0|0 + extended|global_mmlu_instruct_cs_urd:professional_law|0|0 + extended|global_mmlu_instruct_unk_urd:professional_law|0|0 + extended|global_mmlu_instruct_all_vie:professional_law|0|0 + extended|global_mmlu_instruct_ca_vie:professional_law|0|0 + extended|global_mmlu_instruct_cs_vie:professional_law|0|0 + extended|global_mmlu_instruct_unk_vie:professional_law|0|0 + extended|global_mmlu_instruct_all_ara:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_ara:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_ara:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_ara:professional_medicine|0|0 + extended|global_mmlu_instruct_all_ben:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_ben:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_ben:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_ben:professional_medicine|0|0 + extended|global_mmlu_instruct_all_deu:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_deu:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_deu:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_deu:professional_medicine|0|0 + extended|global_mmlu_instruct_all_spa:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_spa:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_spa:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_spa:professional_medicine|0|0 + extended|global_mmlu_instruct_all_eng:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_eng:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_eng:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_eng:professional_medicine|0|0 + extended|global_mmlu_instruct_all_fra:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_fra:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_fra:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_fra:professional_medicine|0|0 + extended|global_mmlu_instruct_all_hin:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_hin:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_hin:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_hin:professional_medicine|0|0 + extended|global_mmlu_instruct_all_ind:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_ind:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_ind:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_ind:professional_medicine|0|0 + extended|global_mmlu_instruct_all_ita:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_ita:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_ita:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_ita:professional_medicine|0|0 + extended|global_mmlu_instruct_all_jpn:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_jpn:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_jpn:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_jpn:professional_medicine|0|0 + extended|global_mmlu_instruct_all_nld:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_nld:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_nld:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_nld:professional_medicine|0|0 + extended|global_mmlu_instruct_all_nor:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_nor:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_nor:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_nor:professional_medicine|0|0 + extended|global_mmlu_instruct_all_pol:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_pol:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_pol:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_pol:professional_medicine|0|0 + extended|global_mmlu_instruct_all_por:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_por:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_por:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_por:professional_medicine|0|0 + extended|global_mmlu_instruct_all_rus:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_rus:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_rus:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_rus:professional_medicine|0|0 + extended|global_mmlu_instruct_all_srp:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_srp:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_srp:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_srp:professional_medicine|0|0 + extended|global_mmlu_instruct_all_swe:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_swe:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_swe:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_swe:professional_medicine|0|0 + extended|global_mmlu_instruct_all_swa:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_swa:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_swa:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_swa:professional_medicine|0|0 + extended|global_mmlu_instruct_all_tel:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_tel:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_tel:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_tel:professional_medicine|0|0 + extended|global_mmlu_instruct_all_tha:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_tha:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_tha:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_tha:professional_medicine|0|0 + extended|global_mmlu_instruct_all_tur:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_tur:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_tur:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_tur:professional_medicine|0|0 + extended|global_mmlu_instruct_all_ukr:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_ukr:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_ukr:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_ukr:professional_medicine|0|0 + extended|global_mmlu_instruct_all_urd:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_urd:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_urd:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_urd:professional_medicine|0|0 + extended|global_mmlu_instruct_all_vie:professional_medicine|0|0 + extended|global_mmlu_instruct_ca_vie:professional_medicine|0|0 + extended|global_mmlu_instruct_cs_vie:professional_medicine|0|0 + extended|global_mmlu_instruct_unk_vie:professional_medicine|0|0 + extended|global_mmlu_instruct_all_ara:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_ara:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_ara:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_ara:professional_psychology|0|0 + extended|global_mmlu_instruct_all_ben:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_ben:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_ben:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_ben:professional_psychology|0|0 + extended|global_mmlu_instruct_all_deu:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_deu:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_deu:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_deu:professional_psychology|0|0 + extended|global_mmlu_instruct_all_spa:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_spa:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_spa:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_spa:professional_psychology|0|0 + extended|global_mmlu_instruct_all_eng:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_eng:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_eng:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_eng:professional_psychology|0|0 + extended|global_mmlu_instruct_all_fra:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_fra:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_fra:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_fra:professional_psychology|0|0 + extended|global_mmlu_instruct_all_hin:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_hin:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_hin:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_hin:professional_psychology|0|0 + extended|global_mmlu_instruct_all_ind:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_ind:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_ind:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_ind:professional_psychology|0|0 + extended|global_mmlu_instruct_all_ita:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_ita:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_ita:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_ita:professional_psychology|0|0 + extended|global_mmlu_instruct_all_jpn:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_jpn:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_jpn:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_jpn:professional_psychology|0|0 + extended|global_mmlu_instruct_all_nld:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_nld:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_nld:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_nld:professional_psychology|0|0 + extended|global_mmlu_instruct_all_nor:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_nor:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_nor:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_nor:professional_psychology|0|0 + extended|global_mmlu_instruct_all_pol:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_pol:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_pol:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_pol:professional_psychology|0|0 + extended|global_mmlu_instruct_all_por:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_por:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_por:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_por:professional_psychology|0|0 + extended|global_mmlu_instruct_all_rus:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_rus:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_rus:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_rus:professional_psychology|0|0 + extended|global_mmlu_instruct_all_srp:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_srp:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_srp:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_srp:professional_psychology|0|0 + extended|global_mmlu_instruct_all_swe:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_swe:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_swe:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_swe:professional_psychology|0|0 + extended|global_mmlu_instruct_all_swa:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_swa:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_swa:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_swa:professional_psychology|0|0 + extended|global_mmlu_instruct_all_tel:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_tel:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_tel:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_tel:professional_psychology|0|0 + extended|global_mmlu_instruct_all_tha:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_tha:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_tha:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_tha:professional_psychology|0|0 + extended|global_mmlu_instruct_all_tur:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_tur:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_tur:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_tur:professional_psychology|0|0 + extended|global_mmlu_instruct_all_ukr:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_ukr:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_ukr:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_ukr:professional_psychology|0|0 + extended|global_mmlu_instruct_all_urd:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_urd:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_urd:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_urd:professional_psychology|0|0 + extended|global_mmlu_instruct_all_vie:professional_psychology|0|0 + extended|global_mmlu_instruct_ca_vie:professional_psychology|0|0 + extended|global_mmlu_instruct_cs_vie:professional_psychology|0|0 + extended|global_mmlu_instruct_unk_vie:professional_psychology|0|0 + extended|global_mmlu_instruct_all_ara:public_relations|0|0 + extended|global_mmlu_instruct_ca_ara:public_relations|0|0 + extended|global_mmlu_instruct_cs_ara:public_relations|0|0 + extended|global_mmlu_instruct_unk_ara:public_relations|0|0 + extended|global_mmlu_instruct_all_ben:public_relations|0|0 + extended|global_mmlu_instruct_ca_ben:public_relations|0|0 + extended|global_mmlu_instruct_cs_ben:public_relations|0|0 + extended|global_mmlu_instruct_unk_ben:public_relations|0|0 + extended|global_mmlu_instruct_all_deu:public_relations|0|0 + extended|global_mmlu_instruct_ca_deu:public_relations|0|0 + extended|global_mmlu_instruct_cs_deu:public_relations|0|0 + extended|global_mmlu_instruct_unk_deu:public_relations|0|0 + extended|global_mmlu_instruct_all_spa:public_relations|0|0 + extended|global_mmlu_instruct_ca_spa:public_relations|0|0 + extended|global_mmlu_instruct_cs_spa:public_relations|0|0 + extended|global_mmlu_instruct_unk_spa:public_relations|0|0 + extended|global_mmlu_instruct_all_eng:public_relations|0|0 + extended|global_mmlu_instruct_ca_eng:public_relations|0|0 + extended|global_mmlu_instruct_cs_eng:public_relations|0|0 + extended|global_mmlu_instruct_unk_eng:public_relations|0|0 + extended|global_mmlu_instruct_all_fra:public_relations|0|0 + extended|global_mmlu_instruct_ca_fra:public_relations|0|0 + extended|global_mmlu_instruct_cs_fra:public_relations|0|0 + extended|global_mmlu_instruct_unk_fra:public_relations|0|0 + extended|global_mmlu_instruct_all_hin:public_relations|0|0 + extended|global_mmlu_instruct_ca_hin:public_relations|0|0 + extended|global_mmlu_instruct_cs_hin:public_relations|0|0 + extended|global_mmlu_instruct_unk_hin:public_relations|0|0 + extended|global_mmlu_instruct_all_ind:public_relations|0|0 + extended|global_mmlu_instruct_ca_ind:public_relations|0|0 + extended|global_mmlu_instruct_cs_ind:public_relations|0|0 + extended|global_mmlu_instruct_unk_ind:public_relations|0|0 + extended|global_mmlu_instruct_all_ita:public_relations|0|0 + extended|global_mmlu_instruct_ca_ita:public_relations|0|0 + extended|global_mmlu_instruct_cs_ita:public_relations|0|0 + extended|global_mmlu_instruct_unk_ita:public_relations|0|0 + extended|global_mmlu_instruct_all_jpn:public_relations|0|0 + extended|global_mmlu_instruct_ca_jpn:public_relations|0|0 + extended|global_mmlu_instruct_cs_jpn:public_relations|0|0 + extended|global_mmlu_instruct_unk_jpn:public_relations|0|0 + extended|global_mmlu_instruct_all_nld:public_relations|0|0 + extended|global_mmlu_instruct_ca_nld:public_relations|0|0 + extended|global_mmlu_instruct_cs_nld:public_relations|0|0 + extended|global_mmlu_instruct_unk_nld:public_relations|0|0 + extended|global_mmlu_instruct_all_nor:public_relations|0|0 + extended|global_mmlu_instruct_ca_nor:public_relations|0|0 + extended|global_mmlu_instruct_cs_nor:public_relations|0|0 + extended|global_mmlu_instruct_unk_nor:public_relations|0|0 + extended|global_mmlu_instruct_all_pol:public_relations|0|0 + extended|global_mmlu_instruct_ca_pol:public_relations|0|0 + extended|global_mmlu_instruct_cs_pol:public_relations|0|0 + extended|global_mmlu_instruct_unk_pol:public_relations|0|0 + extended|global_mmlu_instruct_all_por:public_relations|0|0 + extended|global_mmlu_instruct_ca_por:public_relations|0|0 + extended|global_mmlu_instruct_cs_por:public_relations|0|0 + extended|global_mmlu_instruct_unk_por:public_relations|0|0 + extended|global_mmlu_instruct_all_rus:public_relations|0|0 + extended|global_mmlu_instruct_ca_rus:public_relations|0|0 + extended|global_mmlu_instruct_cs_rus:public_relations|0|0 + extended|global_mmlu_instruct_unk_rus:public_relations|0|0 + extended|global_mmlu_instruct_all_srp:public_relations|0|0 + extended|global_mmlu_instruct_ca_srp:public_relations|0|0 + extended|global_mmlu_instruct_cs_srp:public_relations|0|0 + extended|global_mmlu_instruct_unk_srp:public_relations|0|0 + extended|global_mmlu_instruct_all_swe:public_relations|0|0 + extended|global_mmlu_instruct_ca_swe:public_relations|0|0 + extended|global_mmlu_instruct_cs_swe:public_relations|0|0 + extended|global_mmlu_instruct_unk_swe:public_relations|0|0 + extended|global_mmlu_instruct_all_swa:public_relations|0|0 + extended|global_mmlu_instruct_ca_swa:public_relations|0|0 + extended|global_mmlu_instruct_cs_swa:public_relations|0|0 + extended|global_mmlu_instruct_unk_swa:public_relations|0|0 + extended|global_mmlu_instruct_all_tel:public_relations|0|0 + extended|global_mmlu_instruct_ca_tel:public_relations|0|0 + extended|global_mmlu_instruct_cs_tel:public_relations|0|0 + extended|global_mmlu_instruct_unk_tel:public_relations|0|0 + extended|global_mmlu_instruct_all_tha:public_relations|0|0 + extended|global_mmlu_instruct_ca_tha:public_relations|0|0 + extended|global_mmlu_instruct_cs_tha:public_relations|0|0 + extended|global_mmlu_instruct_unk_tha:public_relations|0|0 + extended|global_mmlu_instruct_all_tur:public_relations|0|0 + extended|global_mmlu_instruct_ca_tur:public_relations|0|0 + extended|global_mmlu_instruct_cs_tur:public_relations|0|0 + extended|global_mmlu_instruct_unk_tur:public_relations|0|0 + extended|global_mmlu_instruct_all_ukr:public_relations|0|0 + extended|global_mmlu_instruct_ca_ukr:public_relations|0|0 + extended|global_mmlu_instruct_cs_ukr:public_relations|0|0 + extended|global_mmlu_instruct_unk_ukr:public_relations|0|0 + extended|global_mmlu_instruct_all_urd:public_relations|0|0 + extended|global_mmlu_instruct_ca_urd:public_relations|0|0 + extended|global_mmlu_instruct_cs_urd:public_relations|0|0 + extended|global_mmlu_instruct_unk_urd:public_relations|0|0 + extended|global_mmlu_instruct_all_vie:public_relations|0|0 + extended|global_mmlu_instruct_ca_vie:public_relations|0|0 + extended|global_mmlu_instruct_cs_vie:public_relations|0|0 + extended|global_mmlu_instruct_unk_vie:public_relations|0|0 + extended|global_mmlu_instruct_all_ara:security_studies|0|0 + extended|global_mmlu_instruct_ca_ara:security_studies|0|0 + extended|global_mmlu_instruct_cs_ara:security_studies|0|0 + extended|global_mmlu_instruct_unk_ara:security_studies|0|0 + extended|global_mmlu_instruct_all_ben:security_studies|0|0 + extended|global_mmlu_instruct_ca_ben:security_studies|0|0 + extended|global_mmlu_instruct_cs_ben:security_studies|0|0 + extended|global_mmlu_instruct_unk_ben:security_studies|0|0 + extended|global_mmlu_instruct_all_deu:security_studies|0|0 + extended|global_mmlu_instruct_ca_deu:security_studies|0|0 + extended|global_mmlu_instruct_cs_deu:security_studies|0|0 + extended|global_mmlu_instruct_unk_deu:security_studies|0|0 + extended|global_mmlu_instruct_all_spa:security_studies|0|0 + extended|global_mmlu_instruct_ca_spa:security_studies|0|0 + extended|global_mmlu_instruct_cs_spa:security_studies|0|0 + extended|global_mmlu_instruct_unk_spa:security_studies|0|0 + extended|global_mmlu_instruct_all_eng:security_studies|0|0 + extended|global_mmlu_instruct_ca_eng:security_studies|0|0 + extended|global_mmlu_instruct_cs_eng:security_studies|0|0 + extended|global_mmlu_instruct_unk_eng:security_studies|0|0 + extended|global_mmlu_instruct_all_fra:security_studies|0|0 + extended|global_mmlu_instruct_ca_fra:security_studies|0|0 + extended|global_mmlu_instruct_cs_fra:security_studies|0|0 + extended|global_mmlu_instruct_unk_fra:security_studies|0|0 + extended|global_mmlu_instruct_all_hin:security_studies|0|0 + extended|global_mmlu_instruct_ca_hin:security_studies|0|0 + extended|global_mmlu_instruct_cs_hin:security_studies|0|0 + extended|global_mmlu_instruct_unk_hin:security_studies|0|0 + extended|global_mmlu_instruct_all_ind:security_studies|0|0 + extended|global_mmlu_instruct_ca_ind:security_studies|0|0 + extended|global_mmlu_instruct_cs_ind:security_studies|0|0 + extended|global_mmlu_instruct_unk_ind:security_studies|0|0 + extended|global_mmlu_instruct_all_ita:security_studies|0|0 + extended|global_mmlu_instruct_ca_ita:security_studies|0|0 + extended|global_mmlu_instruct_cs_ita:security_studies|0|0 + extended|global_mmlu_instruct_unk_ita:security_studies|0|0 + extended|global_mmlu_instruct_all_jpn:security_studies|0|0 + extended|global_mmlu_instruct_ca_jpn:security_studies|0|0 + extended|global_mmlu_instruct_cs_jpn:security_studies|0|0 + extended|global_mmlu_instruct_unk_jpn:security_studies|0|0 + extended|global_mmlu_instruct_all_nld:security_studies|0|0 + extended|global_mmlu_instruct_ca_nld:security_studies|0|0 + extended|global_mmlu_instruct_cs_nld:security_studies|0|0 + extended|global_mmlu_instruct_unk_nld:security_studies|0|0 + extended|global_mmlu_instruct_all_nor:security_studies|0|0 + extended|global_mmlu_instruct_ca_nor:security_studies|0|0 + extended|global_mmlu_instruct_cs_nor:security_studies|0|0 + extended|global_mmlu_instruct_unk_nor:security_studies|0|0 + extended|global_mmlu_instruct_all_pol:security_studies|0|0 + extended|global_mmlu_instruct_ca_pol:security_studies|0|0 + extended|global_mmlu_instruct_cs_pol:security_studies|0|0 + extended|global_mmlu_instruct_unk_pol:security_studies|0|0 + extended|global_mmlu_instruct_all_por:security_studies|0|0 + extended|global_mmlu_instruct_ca_por:security_studies|0|0 + extended|global_mmlu_instruct_cs_por:security_studies|0|0 + extended|global_mmlu_instruct_unk_por:security_studies|0|0 + extended|global_mmlu_instruct_all_rus:security_studies|0|0 + extended|global_mmlu_instruct_ca_rus:security_studies|0|0 + extended|global_mmlu_instruct_cs_rus:security_studies|0|0 + extended|global_mmlu_instruct_unk_rus:security_studies|0|0 + extended|global_mmlu_instruct_all_srp:security_studies|0|0 + extended|global_mmlu_instruct_ca_srp:security_studies|0|0 + extended|global_mmlu_instruct_cs_srp:security_studies|0|0 + extended|global_mmlu_instruct_unk_srp:security_studies|0|0 + extended|global_mmlu_instruct_all_swe:security_studies|0|0 + extended|global_mmlu_instruct_ca_swe:security_studies|0|0 + extended|global_mmlu_instruct_cs_swe:security_studies|0|0 + extended|global_mmlu_instruct_unk_swe:security_studies|0|0 + extended|global_mmlu_instruct_all_swa:security_studies|0|0 + extended|global_mmlu_instruct_ca_swa:security_studies|0|0 + extended|global_mmlu_instruct_cs_swa:security_studies|0|0 + extended|global_mmlu_instruct_unk_swa:security_studies|0|0 + extended|global_mmlu_instruct_all_tel:security_studies|0|0 + extended|global_mmlu_instruct_ca_tel:security_studies|0|0 + extended|global_mmlu_instruct_cs_tel:security_studies|0|0 + extended|global_mmlu_instruct_unk_tel:security_studies|0|0 + extended|global_mmlu_instruct_all_tha:security_studies|0|0 + extended|global_mmlu_instruct_ca_tha:security_studies|0|0 + extended|global_mmlu_instruct_cs_tha:security_studies|0|0 + extended|global_mmlu_instruct_unk_tha:security_studies|0|0 + extended|global_mmlu_instruct_all_tur:security_studies|0|0 + extended|global_mmlu_instruct_ca_tur:security_studies|0|0 + extended|global_mmlu_instruct_cs_tur:security_studies|0|0 + extended|global_mmlu_instruct_unk_tur:security_studies|0|0 + extended|global_mmlu_instruct_all_ukr:security_studies|0|0 + extended|global_mmlu_instruct_ca_ukr:security_studies|0|0 + extended|global_mmlu_instruct_cs_ukr:security_studies|0|0 + extended|global_mmlu_instruct_unk_ukr:security_studies|0|0 + extended|global_mmlu_instruct_all_urd:security_studies|0|0 + extended|global_mmlu_instruct_ca_urd:security_studies|0|0 + extended|global_mmlu_instruct_cs_urd:security_studies|0|0 + extended|global_mmlu_instruct_unk_urd:security_studies|0|0 + extended|global_mmlu_instruct_all_vie:security_studies|0|0 + extended|global_mmlu_instruct_ca_vie:security_studies|0|0 + extended|global_mmlu_instruct_cs_vie:security_studies|0|0 + extended|global_mmlu_instruct_unk_vie:security_studies|0|0 + extended|global_mmlu_instruct_all_ara:sociology|0|0 + extended|global_mmlu_instruct_ca_ara:sociology|0|0 + extended|global_mmlu_instruct_cs_ara:sociology|0|0 + extended|global_mmlu_instruct_unk_ara:sociology|0|0 + extended|global_mmlu_instruct_all_ben:sociology|0|0 + extended|global_mmlu_instruct_ca_ben:sociology|0|0 + extended|global_mmlu_instruct_cs_ben:sociology|0|0 + extended|global_mmlu_instruct_unk_ben:sociology|0|0 + extended|global_mmlu_instruct_all_deu:sociology|0|0 + extended|global_mmlu_instruct_ca_deu:sociology|0|0 + extended|global_mmlu_instruct_cs_deu:sociology|0|0 + extended|global_mmlu_instruct_unk_deu:sociology|0|0 + extended|global_mmlu_instruct_all_spa:sociology|0|0 + extended|global_mmlu_instruct_ca_spa:sociology|0|0 + extended|global_mmlu_instruct_cs_spa:sociology|0|0 + extended|global_mmlu_instruct_unk_spa:sociology|0|0 + extended|global_mmlu_instruct_all_eng:sociology|0|0 + extended|global_mmlu_instruct_ca_eng:sociology|0|0 + extended|global_mmlu_instruct_cs_eng:sociology|0|0 + extended|global_mmlu_instruct_unk_eng:sociology|0|0 + extended|global_mmlu_instruct_all_fra:sociology|0|0 + extended|global_mmlu_instruct_ca_fra:sociology|0|0 + extended|global_mmlu_instruct_cs_fra:sociology|0|0 + extended|global_mmlu_instruct_unk_fra:sociology|0|0 + extended|global_mmlu_instruct_all_hin:sociology|0|0 + extended|global_mmlu_instruct_ca_hin:sociology|0|0 + extended|global_mmlu_instruct_cs_hin:sociology|0|0 + extended|global_mmlu_instruct_unk_hin:sociology|0|0 + extended|global_mmlu_instruct_all_ind:sociology|0|0 + extended|global_mmlu_instruct_ca_ind:sociology|0|0 + extended|global_mmlu_instruct_cs_ind:sociology|0|0 + extended|global_mmlu_instruct_unk_ind:sociology|0|0 + extended|global_mmlu_instruct_all_ita:sociology|0|0 + extended|global_mmlu_instruct_ca_ita:sociology|0|0 + extended|global_mmlu_instruct_cs_ita:sociology|0|0 + extended|global_mmlu_instruct_unk_ita:sociology|0|0 + extended|global_mmlu_instruct_all_jpn:sociology|0|0 + extended|global_mmlu_instruct_ca_jpn:sociology|0|0 + extended|global_mmlu_instruct_cs_jpn:sociology|0|0 + extended|global_mmlu_instruct_unk_jpn:sociology|0|0 + extended|global_mmlu_instruct_all_nld:sociology|0|0 + extended|global_mmlu_instruct_ca_nld:sociology|0|0 + extended|global_mmlu_instruct_cs_nld:sociology|0|0 + extended|global_mmlu_instruct_unk_nld:sociology|0|0 + extended|global_mmlu_instruct_all_nor:sociology|0|0 + extended|global_mmlu_instruct_ca_nor:sociology|0|0 + extended|global_mmlu_instruct_cs_nor:sociology|0|0 + extended|global_mmlu_instruct_unk_nor:sociology|0|0 + extended|global_mmlu_instruct_all_pol:sociology|0|0 + extended|global_mmlu_instruct_ca_pol:sociology|0|0 + extended|global_mmlu_instruct_cs_pol:sociology|0|0 + extended|global_mmlu_instruct_unk_pol:sociology|0|0 + extended|global_mmlu_instruct_all_por:sociology|0|0 + extended|global_mmlu_instruct_ca_por:sociology|0|0 + extended|global_mmlu_instruct_cs_por:sociology|0|0 + extended|global_mmlu_instruct_unk_por:sociology|0|0 + extended|global_mmlu_instruct_all_rus:sociology|0|0 + extended|global_mmlu_instruct_ca_rus:sociology|0|0 + extended|global_mmlu_instruct_cs_rus:sociology|0|0 + extended|global_mmlu_instruct_unk_rus:sociology|0|0 + extended|global_mmlu_instruct_all_srp:sociology|0|0 + extended|global_mmlu_instruct_ca_srp:sociology|0|0 + extended|global_mmlu_instruct_cs_srp:sociology|0|0 + extended|global_mmlu_instruct_unk_srp:sociology|0|0 + extended|global_mmlu_instruct_all_swe:sociology|0|0 + extended|global_mmlu_instruct_ca_swe:sociology|0|0 + extended|global_mmlu_instruct_cs_swe:sociology|0|0 + extended|global_mmlu_instruct_unk_swe:sociology|0|0 + extended|global_mmlu_instruct_all_swa:sociology|0|0 + extended|global_mmlu_instruct_ca_swa:sociology|0|0 + extended|global_mmlu_instruct_cs_swa:sociology|0|0 + extended|global_mmlu_instruct_unk_swa:sociology|0|0 + extended|global_mmlu_instruct_all_tel:sociology|0|0 + extended|global_mmlu_instruct_ca_tel:sociology|0|0 + extended|global_mmlu_instruct_cs_tel:sociology|0|0 + extended|global_mmlu_instruct_unk_tel:sociology|0|0 + extended|global_mmlu_instruct_all_tha:sociology|0|0 + extended|global_mmlu_instruct_ca_tha:sociology|0|0 + extended|global_mmlu_instruct_cs_tha:sociology|0|0 + extended|global_mmlu_instruct_unk_tha:sociology|0|0 + extended|global_mmlu_instruct_all_tur:sociology|0|0 + extended|global_mmlu_instruct_ca_tur:sociology|0|0 + extended|global_mmlu_instruct_cs_tur:sociology|0|0 + extended|global_mmlu_instruct_unk_tur:sociology|0|0 + extended|global_mmlu_instruct_all_ukr:sociology|0|0 + extended|global_mmlu_instruct_ca_ukr:sociology|0|0 + extended|global_mmlu_instruct_cs_ukr:sociology|0|0 + extended|global_mmlu_instruct_unk_ukr:sociology|0|0 + extended|global_mmlu_instruct_all_urd:sociology|0|0 + extended|global_mmlu_instruct_ca_urd:sociology|0|0 + extended|global_mmlu_instruct_cs_urd:sociology|0|0 + extended|global_mmlu_instruct_unk_urd:sociology|0|0 + extended|global_mmlu_instruct_all_vie:sociology|0|0 + extended|global_mmlu_instruct_ca_vie:sociology|0|0 + extended|global_mmlu_instruct_cs_vie:sociology|0|0 + extended|global_mmlu_instruct_unk_vie:sociology|0|0 + extended|global_mmlu_instruct_all_ara:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_ara:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_ara:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_ara:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_ben:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_ben:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_ben:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_ben:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_deu:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_deu:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_deu:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_deu:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_spa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_spa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_spa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_spa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_eng:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_eng:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_eng:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_eng:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_fra:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_fra:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_fra:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_fra:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_hin:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_hin:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_hin:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_hin:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_ind:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_ind:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_ind:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_ind:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_ita:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_ita:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_ita:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_ita:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_jpn:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_jpn:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_jpn:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_jpn:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_nld:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_nld:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_nld:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_nld:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_nor:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_nor:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_nor:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_nor:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_pol:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_pol:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_pol:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_pol:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_por:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_por:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_por:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_por:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_rus:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_rus:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_rus:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_rus:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_srp:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_srp:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_srp:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_srp:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_swe:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_swe:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_swe:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_swe:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_swa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_swa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_swa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_swa:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_tel:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_tel:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_tel:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_tel:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_tha:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_tha:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_tha:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_tha:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_tur:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_tur:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_tur:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_tur:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_ukr:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_ukr:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_ukr:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_ukr:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_urd:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_urd:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_urd:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_urd:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_vie:us_foreign_policy|0|0 + extended|global_mmlu_instruct_ca_vie:us_foreign_policy|0|0 + extended|global_mmlu_instruct_cs_vie:us_foreign_policy|0|0 + extended|global_mmlu_instruct_unk_vie:us_foreign_policy|0|0 + extended|global_mmlu_instruct_all_ara:virology|0|0 + extended|global_mmlu_instruct_ca_ara:virology|0|0 + extended|global_mmlu_instruct_cs_ara:virology|0|0 + extended|global_mmlu_instruct_unk_ara:virology|0|0 + extended|global_mmlu_instruct_all_ben:virology|0|0 + extended|global_mmlu_instruct_ca_ben:virology|0|0 + extended|global_mmlu_instruct_cs_ben:virology|0|0 + extended|global_mmlu_instruct_unk_ben:virology|0|0 + extended|global_mmlu_instruct_all_deu:virology|0|0 + extended|global_mmlu_instruct_ca_deu:virology|0|0 + extended|global_mmlu_instruct_cs_deu:virology|0|0 + extended|global_mmlu_instruct_unk_deu:virology|0|0 + extended|global_mmlu_instruct_all_spa:virology|0|0 + extended|global_mmlu_instruct_ca_spa:virology|0|0 + extended|global_mmlu_instruct_cs_spa:virology|0|0 + extended|global_mmlu_instruct_unk_spa:virology|0|0 + extended|global_mmlu_instruct_all_eng:virology|0|0 + extended|global_mmlu_instruct_ca_eng:virology|0|0 + extended|global_mmlu_instruct_cs_eng:virology|0|0 + extended|global_mmlu_instruct_unk_eng:virology|0|0 + extended|global_mmlu_instruct_all_fra:virology|0|0 + extended|global_mmlu_instruct_ca_fra:virology|0|0 + extended|global_mmlu_instruct_cs_fra:virology|0|0 + extended|global_mmlu_instruct_unk_fra:virology|0|0 + extended|global_mmlu_instruct_all_hin:virology|0|0 + extended|global_mmlu_instruct_ca_hin:virology|0|0 + extended|global_mmlu_instruct_cs_hin:virology|0|0 + extended|global_mmlu_instruct_unk_hin:virology|0|0 + extended|global_mmlu_instruct_all_ind:virology|0|0 + extended|global_mmlu_instruct_ca_ind:virology|0|0 + extended|global_mmlu_instruct_cs_ind:virology|0|0 + extended|global_mmlu_instruct_unk_ind:virology|0|0 + extended|global_mmlu_instruct_all_ita:virology|0|0 + extended|global_mmlu_instruct_ca_ita:virology|0|0 + extended|global_mmlu_instruct_cs_ita:virology|0|0 + extended|global_mmlu_instruct_unk_ita:virology|0|0 + extended|global_mmlu_instruct_all_jpn:virology|0|0 + extended|global_mmlu_instruct_ca_jpn:virology|0|0 + extended|global_mmlu_instruct_cs_jpn:virology|0|0 + extended|global_mmlu_instruct_unk_jpn:virology|0|0 + extended|global_mmlu_instruct_all_nld:virology|0|0 + extended|global_mmlu_instruct_ca_nld:virology|0|0 + extended|global_mmlu_instruct_cs_nld:virology|0|0 + extended|global_mmlu_instruct_unk_nld:virology|0|0 + extended|global_mmlu_instruct_all_nor:virology|0|0 + extended|global_mmlu_instruct_ca_nor:virology|0|0 + extended|global_mmlu_instruct_cs_nor:virology|0|0 + extended|global_mmlu_instruct_unk_nor:virology|0|0 + extended|global_mmlu_instruct_all_pol:virology|0|0 + extended|global_mmlu_instruct_ca_pol:virology|0|0 + extended|global_mmlu_instruct_cs_pol:virology|0|0 + extended|global_mmlu_instruct_unk_pol:virology|0|0 + extended|global_mmlu_instruct_all_por:virology|0|0 + extended|global_mmlu_instruct_ca_por:virology|0|0 + extended|global_mmlu_instruct_cs_por:virology|0|0 + extended|global_mmlu_instruct_unk_por:virology|0|0 + extended|global_mmlu_instruct_all_rus:virology|0|0 + extended|global_mmlu_instruct_ca_rus:virology|0|0 + extended|global_mmlu_instruct_cs_rus:virology|0|0 + extended|global_mmlu_instruct_unk_rus:virology|0|0 + extended|global_mmlu_instruct_all_srp:virology|0|0 + extended|global_mmlu_instruct_ca_srp:virology|0|0 + extended|global_mmlu_instruct_cs_srp:virology|0|0 + extended|global_mmlu_instruct_unk_srp:virology|0|0 + extended|global_mmlu_instruct_all_swe:virology|0|0 + extended|global_mmlu_instruct_ca_swe:virology|0|0 + extended|global_mmlu_instruct_cs_swe:virology|0|0 + extended|global_mmlu_instruct_unk_swe:virology|0|0 + extended|global_mmlu_instruct_all_swa:virology|0|0 + extended|global_mmlu_instruct_ca_swa:virology|0|0 + extended|global_mmlu_instruct_cs_swa:virology|0|0 + extended|global_mmlu_instruct_unk_swa:virology|0|0 + extended|global_mmlu_instruct_all_tel:virology|0|0 + extended|global_mmlu_instruct_ca_tel:virology|0|0 + extended|global_mmlu_instruct_cs_tel:virology|0|0 + extended|global_mmlu_instruct_unk_tel:virology|0|0 + extended|global_mmlu_instruct_all_tha:virology|0|0 + extended|global_mmlu_instruct_ca_tha:virology|0|0 + extended|global_mmlu_instruct_cs_tha:virology|0|0 + extended|global_mmlu_instruct_unk_tha:virology|0|0 + extended|global_mmlu_instruct_all_tur:virology|0|0 + extended|global_mmlu_instruct_ca_tur:virology|0|0 + extended|global_mmlu_instruct_cs_tur:virology|0|0 + extended|global_mmlu_instruct_unk_tur:virology|0|0 + extended|global_mmlu_instruct_all_ukr:virology|0|0 + extended|global_mmlu_instruct_ca_ukr:virology|0|0 + extended|global_mmlu_instruct_cs_ukr:virology|0|0 + extended|global_mmlu_instruct_unk_ukr:virology|0|0 + extended|global_mmlu_instruct_all_urd:virology|0|0 + extended|global_mmlu_instruct_ca_urd:virology|0|0 + extended|global_mmlu_instruct_cs_urd:virology|0|0 + extended|global_mmlu_instruct_unk_urd:virology|0|0 + extended|global_mmlu_instruct_all_vie:virology|0|0 + extended|global_mmlu_instruct_ca_vie:virology|0|0 + extended|global_mmlu_instruct_cs_vie:virology|0|0 + extended|global_mmlu_instruct_unk_vie:virology|0|0 + extended|global_mmlu_instruct_all_ara:world_religions|0|0 + extended|global_mmlu_instruct_ca_ara:world_religions|0|0 + extended|global_mmlu_instruct_cs_ara:world_religions|0|0 + extended|global_mmlu_instruct_unk_ara:world_religions|0|0 + extended|global_mmlu_instruct_all_ben:world_religions|0|0 + extended|global_mmlu_instruct_ca_ben:world_religions|0|0 + extended|global_mmlu_instruct_cs_ben:world_religions|0|0 + extended|global_mmlu_instruct_unk_ben:world_religions|0|0 + extended|global_mmlu_instruct_all_deu:world_religions|0|0 + extended|global_mmlu_instruct_ca_deu:world_religions|0|0 + extended|global_mmlu_instruct_cs_deu:world_religions|0|0 + extended|global_mmlu_instruct_unk_deu:world_religions|0|0 + extended|global_mmlu_instruct_all_spa:world_religions|0|0 + extended|global_mmlu_instruct_ca_spa:world_religions|0|0 + extended|global_mmlu_instruct_cs_spa:world_religions|0|0 + extended|global_mmlu_instruct_unk_spa:world_religions|0|0 + extended|global_mmlu_instruct_all_eng:world_religions|0|0 + extended|global_mmlu_instruct_ca_eng:world_religions|0|0 + extended|global_mmlu_instruct_cs_eng:world_religions|0|0 + extended|global_mmlu_instruct_unk_eng:world_religions|0|0 + extended|global_mmlu_instruct_all_fra:world_religions|0|0 + extended|global_mmlu_instruct_ca_fra:world_religions|0|0 + extended|global_mmlu_instruct_cs_fra:world_religions|0|0 + extended|global_mmlu_instruct_unk_fra:world_religions|0|0 + extended|global_mmlu_instruct_all_hin:world_religions|0|0 + extended|global_mmlu_instruct_ca_hin:world_religions|0|0 + extended|global_mmlu_instruct_cs_hin:world_religions|0|0 + extended|global_mmlu_instruct_unk_hin:world_religions|0|0 + extended|global_mmlu_instruct_all_ind:world_religions|0|0 + extended|global_mmlu_instruct_ca_ind:world_religions|0|0 + extended|global_mmlu_instruct_cs_ind:world_religions|0|0 + extended|global_mmlu_instruct_unk_ind:world_religions|0|0 + extended|global_mmlu_instruct_all_ita:world_religions|0|0 + extended|global_mmlu_instruct_ca_ita:world_religions|0|0 + extended|global_mmlu_instruct_cs_ita:world_religions|0|0 + extended|global_mmlu_instruct_unk_ita:world_religions|0|0 + extended|global_mmlu_instruct_all_jpn:world_religions|0|0 + extended|global_mmlu_instruct_ca_jpn:world_religions|0|0 + extended|global_mmlu_instruct_cs_jpn:world_religions|0|0 + extended|global_mmlu_instruct_unk_jpn:world_religions|0|0 + extended|global_mmlu_instruct_all_nld:world_religions|0|0 + extended|global_mmlu_instruct_ca_nld:world_religions|0|0 + extended|global_mmlu_instruct_cs_nld:world_religions|0|0 + extended|global_mmlu_instruct_unk_nld:world_religions|0|0 + extended|global_mmlu_instruct_all_nor:world_religions|0|0 + extended|global_mmlu_instruct_ca_nor:world_religions|0|0 + extended|global_mmlu_instruct_cs_nor:world_religions|0|0 + extended|global_mmlu_instruct_unk_nor:world_religions|0|0 + extended|global_mmlu_instruct_all_pol:world_religions|0|0 + extended|global_mmlu_instruct_ca_pol:world_religions|0|0 + extended|global_mmlu_instruct_cs_pol:world_religions|0|0 + extended|global_mmlu_instruct_unk_pol:world_religions|0|0 + extended|global_mmlu_instruct_all_por:world_religions|0|0 + extended|global_mmlu_instruct_ca_por:world_religions|0|0 + extended|global_mmlu_instruct_cs_por:world_religions|0|0 + extended|global_mmlu_instruct_unk_por:world_religions|0|0 + extended|global_mmlu_instruct_all_rus:world_religions|0|0 + extended|global_mmlu_instruct_ca_rus:world_religions|0|0 + extended|global_mmlu_instruct_cs_rus:world_religions|0|0 + extended|global_mmlu_instruct_unk_rus:world_religions|0|0 + extended|global_mmlu_instruct_all_srp:world_religions|0|0 + extended|global_mmlu_instruct_ca_srp:world_religions|0|0 + extended|global_mmlu_instruct_cs_srp:world_religions|0|0 + extended|global_mmlu_instruct_unk_srp:world_religions|0|0 + extended|global_mmlu_instruct_all_swe:world_religions|0|0 + extended|global_mmlu_instruct_ca_swe:world_religions|0|0 + extended|global_mmlu_instruct_cs_swe:world_religions|0|0 + extended|global_mmlu_instruct_unk_swe:world_religions|0|0 + extended|global_mmlu_instruct_all_swa:world_religions|0|0 + extended|global_mmlu_instruct_ca_swa:world_religions|0|0 + extended|global_mmlu_instruct_cs_swa:world_religions|0|0 + extended|global_mmlu_instruct_unk_swa:world_religions|0|0 + extended|global_mmlu_instruct_all_tel:world_religions|0|0 + extended|global_mmlu_instruct_ca_tel:world_religions|0|0 + extended|global_mmlu_instruct_cs_tel:world_religions|0|0 + extended|global_mmlu_instruct_unk_tel:world_religions|0|0 + extended|global_mmlu_instruct_all_tha:world_religions|0|0 + extended|global_mmlu_instruct_ca_tha:world_religions|0|0 + extended|global_mmlu_instruct_cs_tha:world_religions|0|0 + extended|global_mmlu_instruct_unk_tha:world_religions|0|0 + extended|global_mmlu_instruct_all_tur:world_religions|0|0 + extended|global_mmlu_instruct_ca_tur:world_religions|0|0 + extended|global_mmlu_instruct_cs_tur:world_religions|0|0 + extended|global_mmlu_instruct_unk_tur:world_religions|0|0 + extended|global_mmlu_instruct_all_ukr:world_religions|0|0 + extended|global_mmlu_instruct_ca_ukr:world_religions|0|0 + extended|global_mmlu_instruct_cs_ukr:world_religions|0|0 + extended|global_mmlu_instruct_unk_ukr:world_religions|0|0 + extended|global_mmlu_instruct_all_urd:world_religions|0|0 + extended|global_mmlu_instruct_ca_urd:world_religions|0|0 + extended|global_mmlu_instruct_cs_urd:world_religions|0|0 + extended|global_mmlu_instruct_unk_urd:world_religions|0|0 + extended|global_mmlu_instruct_all_vie:world_religions|0|0 + extended|global_mmlu_instruct_ca_vie:world_religions|0|0 + extended|global_mmlu_instruct_cs_vie:world_religions|0|0 + extended|global_mmlu_instruct_unk_vie:world_religions|0|0 diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py index 2dcb15b85..adbdf05be 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -265,9 +265,11 @@ def prompt(self, line, task_name: str = None): ) for subset in MMLU_SUBSETS for language in [ + Language.ARABIC, + Language.BENGALI, Language.GERMAN, - Language.ENGLISH, Language.SPANISH, + Language.ENGLISH, Language.FRENCH, # Language.HEBREW, Language.HINDI, From a549107ceb59eebb62b7177808bb07d89e10a450 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Wed, 25 Jun 2025 16:08:21 +0000 Subject: [PATCH 06/15] focused data on actually existing languages --- examples/tasks/instruct_multilingual.txt | 5524 +---------------- src/lighteval/tasks/extended/misc/instruct.py | 154 +- 2 files changed, 93 insertions(+), 5585 deletions(-) diff --git a/examples/tasks/instruct_multilingual.txt b/examples/tasks/instruct_multilingual.txt index cd65763b6..96f60e42d 100644 --- a/examples/tasks/instruct_multilingual.txt +++ b/examples/tasks/instruct_multilingual.txt @@ -1,5477 +1,47 @@ -extended|'belebele_instruct_deu_Latn|0|0 - extended|belebele_instruct_fra_Latn|0|0 - extended|belebele_instruct_ita_Latn|0|0 - extended|belebele_instruct_por_Latn|0|0 - extended|belebele_instruct_spa_Latn|0|0 - extended|global_mmlu_instruct_all_ara:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_ara:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_ara:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_ara:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_ben:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_ben:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_ben:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_ben:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_deu:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_deu:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_deu:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_deu:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_spa:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_spa:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_spa:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_spa:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_eng:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_eng:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_eng:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_eng:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_fra:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_fra:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_fra:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_fra:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_hin:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_hin:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_hin:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_hin:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_ind:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_ind:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_ind:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_ind:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_ita:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_ita:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_ita:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_ita:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_jpn:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_jpn:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_jpn:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_jpn:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_nld:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_nld:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_nld:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_nld:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_nor:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_nor:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_nor:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_nor:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_pol:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_pol:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_pol:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_pol:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_por:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_por:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_por:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_por:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_rus:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_rus:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_rus:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_rus:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_srp:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_srp:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_srp:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_srp:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_swe:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_swe:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_swe:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_swe:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_swa:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_swa:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_swa:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_swa:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_tel:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_tel:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_tel:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_tel:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_tha:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_tha:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_tha:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_tha:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_tur:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_tur:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_tur:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_tur:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_ukr:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_ukr:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_ukr:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_ukr:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_urd:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_urd:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_urd:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_urd:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_vie:abstract_algebra|0|0 - extended|global_mmlu_instruct_ca_vie:abstract_algebra|0|0 - extended|global_mmlu_instruct_cs_vie:abstract_algebra|0|0 - extended|global_mmlu_instruct_unk_vie:abstract_algebra|0|0 - extended|global_mmlu_instruct_all_ara:anatomy|0|0 - extended|global_mmlu_instruct_ca_ara:anatomy|0|0 - extended|global_mmlu_instruct_cs_ara:anatomy|0|0 - extended|global_mmlu_instruct_unk_ara:anatomy|0|0 - extended|global_mmlu_instruct_all_ben:anatomy|0|0 - extended|global_mmlu_instruct_ca_ben:anatomy|0|0 - extended|global_mmlu_instruct_cs_ben:anatomy|0|0 - extended|global_mmlu_instruct_unk_ben:anatomy|0|0 - extended|global_mmlu_instruct_all_deu:anatomy|0|0 - extended|global_mmlu_instruct_ca_deu:anatomy|0|0 - extended|global_mmlu_instruct_cs_deu:anatomy|0|0 - extended|global_mmlu_instruct_unk_deu:anatomy|0|0 - extended|global_mmlu_instruct_all_spa:anatomy|0|0 - extended|global_mmlu_instruct_ca_spa:anatomy|0|0 - extended|global_mmlu_instruct_cs_spa:anatomy|0|0 - extended|global_mmlu_instruct_unk_spa:anatomy|0|0 - extended|global_mmlu_instruct_all_eng:anatomy|0|0 - extended|global_mmlu_instruct_ca_eng:anatomy|0|0 - extended|global_mmlu_instruct_cs_eng:anatomy|0|0 - extended|global_mmlu_instruct_unk_eng:anatomy|0|0 - extended|global_mmlu_instruct_all_fra:anatomy|0|0 - extended|global_mmlu_instruct_ca_fra:anatomy|0|0 - extended|global_mmlu_instruct_cs_fra:anatomy|0|0 - extended|global_mmlu_instruct_unk_fra:anatomy|0|0 - extended|global_mmlu_instruct_all_hin:anatomy|0|0 - extended|global_mmlu_instruct_ca_hin:anatomy|0|0 - extended|global_mmlu_instruct_cs_hin:anatomy|0|0 - extended|global_mmlu_instruct_unk_hin:anatomy|0|0 - extended|global_mmlu_instruct_all_ind:anatomy|0|0 - extended|global_mmlu_instruct_ca_ind:anatomy|0|0 - extended|global_mmlu_instruct_cs_ind:anatomy|0|0 - extended|global_mmlu_instruct_unk_ind:anatomy|0|0 - extended|global_mmlu_instruct_all_ita:anatomy|0|0 - extended|global_mmlu_instruct_ca_ita:anatomy|0|0 - extended|global_mmlu_instruct_cs_ita:anatomy|0|0 - extended|global_mmlu_instruct_unk_ita:anatomy|0|0 - extended|global_mmlu_instruct_all_jpn:anatomy|0|0 - extended|global_mmlu_instruct_ca_jpn:anatomy|0|0 - extended|global_mmlu_instruct_cs_jpn:anatomy|0|0 - extended|global_mmlu_instruct_unk_jpn:anatomy|0|0 - extended|global_mmlu_instruct_all_nld:anatomy|0|0 - extended|global_mmlu_instruct_ca_nld:anatomy|0|0 - extended|global_mmlu_instruct_cs_nld:anatomy|0|0 - extended|global_mmlu_instruct_unk_nld:anatomy|0|0 - extended|global_mmlu_instruct_all_nor:anatomy|0|0 - extended|global_mmlu_instruct_ca_nor:anatomy|0|0 - extended|global_mmlu_instruct_cs_nor:anatomy|0|0 - extended|global_mmlu_instruct_unk_nor:anatomy|0|0 - extended|global_mmlu_instruct_all_pol:anatomy|0|0 - extended|global_mmlu_instruct_ca_pol:anatomy|0|0 - extended|global_mmlu_instruct_cs_pol:anatomy|0|0 - extended|global_mmlu_instruct_unk_pol:anatomy|0|0 - extended|global_mmlu_instruct_all_por:anatomy|0|0 - extended|global_mmlu_instruct_ca_por:anatomy|0|0 - extended|global_mmlu_instruct_cs_por:anatomy|0|0 - extended|global_mmlu_instruct_unk_por:anatomy|0|0 - extended|global_mmlu_instruct_all_rus:anatomy|0|0 - extended|global_mmlu_instruct_ca_rus:anatomy|0|0 - extended|global_mmlu_instruct_cs_rus:anatomy|0|0 - extended|global_mmlu_instruct_unk_rus:anatomy|0|0 - extended|global_mmlu_instruct_all_srp:anatomy|0|0 - extended|global_mmlu_instruct_ca_srp:anatomy|0|0 - extended|global_mmlu_instruct_cs_srp:anatomy|0|0 - extended|global_mmlu_instruct_unk_srp:anatomy|0|0 - extended|global_mmlu_instruct_all_swe:anatomy|0|0 - extended|global_mmlu_instruct_ca_swe:anatomy|0|0 - extended|global_mmlu_instruct_cs_swe:anatomy|0|0 - extended|global_mmlu_instruct_unk_swe:anatomy|0|0 - extended|global_mmlu_instruct_all_swa:anatomy|0|0 - extended|global_mmlu_instruct_ca_swa:anatomy|0|0 - extended|global_mmlu_instruct_cs_swa:anatomy|0|0 - extended|global_mmlu_instruct_unk_swa:anatomy|0|0 - extended|global_mmlu_instruct_all_tel:anatomy|0|0 - extended|global_mmlu_instruct_ca_tel:anatomy|0|0 - extended|global_mmlu_instruct_cs_tel:anatomy|0|0 - extended|global_mmlu_instruct_unk_tel:anatomy|0|0 - extended|global_mmlu_instruct_all_tha:anatomy|0|0 - extended|global_mmlu_instruct_ca_tha:anatomy|0|0 - extended|global_mmlu_instruct_cs_tha:anatomy|0|0 - extended|global_mmlu_instruct_unk_tha:anatomy|0|0 - extended|global_mmlu_instruct_all_tur:anatomy|0|0 - extended|global_mmlu_instruct_ca_tur:anatomy|0|0 - extended|global_mmlu_instruct_cs_tur:anatomy|0|0 - extended|global_mmlu_instruct_unk_tur:anatomy|0|0 - extended|global_mmlu_instruct_all_ukr:anatomy|0|0 - extended|global_mmlu_instruct_ca_ukr:anatomy|0|0 - extended|global_mmlu_instruct_cs_ukr:anatomy|0|0 - extended|global_mmlu_instruct_unk_ukr:anatomy|0|0 - extended|global_mmlu_instruct_all_urd:anatomy|0|0 - extended|global_mmlu_instruct_ca_urd:anatomy|0|0 - extended|global_mmlu_instruct_cs_urd:anatomy|0|0 - extended|global_mmlu_instruct_unk_urd:anatomy|0|0 - extended|global_mmlu_instruct_all_vie:anatomy|0|0 - extended|global_mmlu_instruct_ca_vie:anatomy|0|0 - extended|global_mmlu_instruct_cs_vie:anatomy|0|0 - extended|global_mmlu_instruct_unk_vie:anatomy|0|0 - extended|global_mmlu_instruct_all_ara:astronomy|0|0 - extended|global_mmlu_instruct_ca_ara:astronomy|0|0 - extended|global_mmlu_instruct_cs_ara:astronomy|0|0 - extended|global_mmlu_instruct_unk_ara:astronomy|0|0 - extended|global_mmlu_instruct_all_ben:astronomy|0|0 - extended|global_mmlu_instruct_ca_ben:astronomy|0|0 - extended|global_mmlu_instruct_cs_ben:astronomy|0|0 - extended|global_mmlu_instruct_unk_ben:astronomy|0|0 - extended|global_mmlu_instruct_all_deu:astronomy|0|0 - extended|global_mmlu_instruct_ca_deu:astronomy|0|0 - extended|global_mmlu_instruct_cs_deu:astronomy|0|0 - extended|global_mmlu_instruct_unk_deu:astronomy|0|0 - extended|global_mmlu_instruct_all_spa:astronomy|0|0 - extended|global_mmlu_instruct_ca_spa:astronomy|0|0 - extended|global_mmlu_instruct_cs_spa:astronomy|0|0 - extended|global_mmlu_instruct_unk_spa:astronomy|0|0 - extended|global_mmlu_instruct_all_eng:astronomy|0|0 - extended|global_mmlu_instruct_ca_eng:astronomy|0|0 - extended|global_mmlu_instruct_cs_eng:astronomy|0|0 - extended|global_mmlu_instruct_unk_eng:astronomy|0|0 - extended|global_mmlu_instruct_all_fra:astronomy|0|0 - extended|global_mmlu_instruct_ca_fra:astronomy|0|0 - extended|global_mmlu_instruct_cs_fra:astronomy|0|0 - extended|global_mmlu_instruct_unk_fra:astronomy|0|0 - extended|global_mmlu_instruct_all_hin:astronomy|0|0 - extended|global_mmlu_instruct_ca_hin:astronomy|0|0 - extended|global_mmlu_instruct_cs_hin:astronomy|0|0 - extended|global_mmlu_instruct_unk_hin:astronomy|0|0 - extended|global_mmlu_instruct_all_ind:astronomy|0|0 - extended|global_mmlu_instruct_ca_ind:astronomy|0|0 - extended|global_mmlu_instruct_cs_ind:astronomy|0|0 - extended|global_mmlu_instruct_unk_ind:astronomy|0|0 - extended|global_mmlu_instruct_all_ita:astronomy|0|0 - extended|global_mmlu_instruct_ca_ita:astronomy|0|0 - extended|global_mmlu_instruct_cs_ita:astronomy|0|0 - extended|global_mmlu_instruct_unk_ita:astronomy|0|0 - extended|global_mmlu_instruct_all_jpn:astronomy|0|0 - extended|global_mmlu_instruct_ca_jpn:astronomy|0|0 - extended|global_mmlu_instruct_cs_jpn:astronomy|0|0 - extended|global_mmlu_instruct_unk_jpn:astronomy|0|0 - extended|global_mmlu_instruct_all_nld:astronomy|0|0 - extended|global_mmlu_instruct_ca_nld:astronomy|0|0 - extended|global_mmlu_instruct_cs_nld:astronomy|0|0 - extended|global_mmlu_instruct_unk_nld:astronomy|0|0 - extended|global_mmlu_instruct_all_nor:astronomy|0|0 - extended|global_mmlu_instruct_ca_nor:astronomy|0|0 - extended|global_mmlu_instruct_cs_nor:astronomy|0|0 - extended|global_mmlu_instruct_unk_nor:astronomy|0|0 - extended|global_mmlu_instruct_all_pol:astronomy|0|0 - extended|global_mmlu_instruct_ca_pol:astronomy|0|0 - extended|global_mmlu_instruct_cs_pol:astronomy|0|0 - extended|global_mmlu_instruct_unk_pol:astronomy|0|0 - extended|global_mmlu_instruct_all_por:astronomy|0|0 - extended|global_mmlu_instruct_ca_por:astronomy|0|0 - extended|global_mmlu_instruct_cs_por:astronomy|0|0 - extended|global_mmlu_instruct_unk_por:astronomy|0|0 - extended|global_mmlu_instruct_all_rus:astronomy|0|0 - extended|global_mmlu_instruct_ca_rus:astronomy|0|0 - extended|global_mmlu_instruct_cs_rus:astronomy|0|0 - extended|global_mmlu_instruct_unk_rus:astronomy|0|0 - extended|global_mmlu_instruct_all_srp:astronomy|0|0 - extended|global_mmlu_instruct_ca_srp:astronomy|0|0 - extended|global_mmlu_instruct_cs_srp:astronomy|0|0 - extended|global_mmlu_instruct_unk_srp:astronomy|0|0 - extended|global_mmlu_instruct_all_swe:astronomy|0|0 - extended|global_mmlu_instruct_ca_swe:astronomy|0|0 - extended|global_mmlu_instruct_cs_swe:astronomy|0|0 - extended|global_mmlu_instruct_unk_swe:astronomy|0|0 - extended|global_mmlu_instruct_all_swa:astronomy|0|0 - extended|global_mmlu_instruct_ca_swa:astronomy|0|0 - extended|global_mmlu_instruct_cs_swa:astronomy|0|0 - extended|global_mmlu_instruct_unk_swa:astronomy|0|0 - extended|global_mmlu_instruct_all_tel:astronomy|0|0 - extended|global_mmlu_instruct_ca_tel:astronomy|0|0 - extended|global_mmlu_instruct_cs_tel:astronomy|0|0 - extended|global_mmlu_instruct_unk_tel:astronomy|0|0 - extended|global_mmlu_instruct_all_tha:astronomy|0|0 - extended|global_mmlu_instruct_ca_tha:astronomy|0|0 - extended|global_mmlu_instruct_cs_tha:astronomy|0|0 - extended|global_mmlu_instruct_unk_tha:astronomy|0|0 - extended|global_mmlu_instruct_all_tur:astronomy|0|0 - extended|global_mmlu_instruct_ca_tur:astronomy|0|0 - extended|global_mmlu_instruct_cs_tur:astronomy|0|0 - extended|global_mmlu_instruct_unk_tur:astronomy|0|0 - extended|global_mmlu_instruct_all_ukr:astronomy|0|0 - extended|global_mmlu_instruct_ca_ukr:astronomy|0|0 - extended|global_mmlu_instruct_cs_ukr:astronomy|0|0 - extended|global_mmlu_instruct_unk_ukr:astronomy|0|0 - extended|global_mmlu_instruct_all_urd:astronomy|0|0 - extended|global_mmlu_instruct_ca_urd:astronomy|0|0 - extended|global_mmlu_instruct_cs_urd:astronomy|0|0 - extended|global_mmlu_instruct_unk_urd:astronomy|0|0 - extended|global_mmlu_instruct_all_vie:astronomy|0|0 - extended|global_mmlu_instruct_ca_vie:astronomy|0|0 - extended|global_mmlu_instruct_cs_vie:astronomy|0|0 - extended|global_mmlu_instruct_unk_vie:astronomy|0|0 - extended|global_mmlu_instruct_all_ara:business_ethics|0|0 - extended|global_mmlu_instruct_ca_ara:business_ethics|0|0 - extended|global_mmlu_instruct_cs_ara:business_ethics|0|0 - extended|global_mmlu_instruct_unk_ara:business_ethics|0|0 - extended|global_mmlu_instruct_all_ben:business_ethics|0|0 - extended|global_mmlu_instruct_ca_ben:business_ethics|0|0 - extended|global_mmlu_instruct_cs_ben:business_ethics|0|0 - extended|global_mmlu_instruct_unk_ben:business_ethics|0|0 - extended|global_mmlu_instruct_all_deu:business_ethics|0|0 - extended|global_mmlu_instruct_ca_deu:business_ethics|0|0 - extended|global_mmlu_instruct_cs_deu:business_ethics|0|0 - extended|global_mmlu_instruct_unk_deu:business_ethics|0|0 - extended|global_mmlu_instruct_all_spa:business_ethics|0|0 - extended|global_mmlu_instruct_ca_spa:business_ethics|0|0 - extended|global_mmlu_instruct_cs_spa:business_ethics|0|0 - extended|global_mmlu_instruct_unk_spa:business_ethics|0|0 - extended|global_mmlu_instruct_all_eng:business_ethics|0|0 - extended|global_mmlu_instruct_ca_eng:business_ethics|0|0 - extended|global_mmlu_instruct_cs_eng:business_ethics|0|0 - extended|global_mmlu_instruct_unk_eng:business_ethics|0|0 - extended|global_mmlu_instruct_all_fra:business_ethics|0|0 - extended|global_mmlu_instruct_ca_fra:business_ethics|0|0 - extended|global_mmlu_instruct_cs_fra:business_ethics|0|0 - extended|global_mmlu_instruct_unk_fra:business_ethics|0|0 - extended|global_mmlu_instruct_all_hin:business_ethics|0|0 - extended|global_mmlu_instruct_ca_hin:business_ethics|0|0 - extended|global_mmlu_instruct_cs_hin:business_ethics|0|0 - extended|global_mmlu_instruct_unk_hin:business_ethics|0|0 - extended|global_mmlu_instruct_all_ind:business_ethics|0|0 - extended|global_mmlu_instruct_ca_ind:business_ethics|0|0 - extended|global_mmlu_instruct_cs_ind:business_ethics|0|0 - extended|global_mmlu_instruct_unk_ind:business_ethics|0|0 - extended|global_mmlu_instruct_all_ita:business_ethics|0|0 - extended|global_mmlu_instruct_ca_ita:business_ethics|0|0 - extended|global_mmlu_instruct_cs_ita:business_ethics|0|0 - extended|global_mmlu_instruct_unk_ita:business_ethics|0|0 - extended|global_mmlu_instruct_all_jpn:business_ethics|0|0 - extended|global_mmlu_instruct_ca_jpn:business_ethics|0|0 - extended|global_mmlu_instruct_cs_jpn:business_ethics|0|0 - extended|global_mmlu_instruct_unk_jpn:business_ethics|0|0 - extended|global_mmlu_instruct_all_nld:business_ethics|0|0 - extended|global_mmlu_instruct_ca_nld:business_ethics|0|0 - extended|global_mmlu_instruct_cs_nld:business_ethics|0|0 - extended|global_mmlu_instruct_unk_nld:business_ethics|0|0 - extended|global_mmlu_instruct_all_nor:business_ethics|0|0 - extended|global_mmlu_instruct_ca_nor:business_ethics|0|0 - extended|global_mmlu_instruct_cs_nor:business_ethics|0|0 - extended|global_mmlu_instruct_unk_nor:business_ethics|0|0 - extended|global_mmlu_instruct_all_pol:business_ethics|0|0 - extended|global_mmlu_instruct_ca_pol:business_ethics|0|0 - extended|global_mmlu_instruct_cs_pol:business_ethics|0|0 - extended|global_mmlu_instruct_unk_pol:business_ethics|0|0 - extended|global_mmlu_instruct_all_por:business_ethics|0|0 - extended|global_mmlu_instruct_ca_por:business_ethics|0|0 - extended|global_mmlu_instruct_cs_por:business_ethics|0|0 - extended|global_mmlu_instruct_unk_por:business_ethics|0|0 - extended|global_mmlu_instruct_all_rus:business_ethics|0|0 - extended|global_mmlu_instruct_ca_rus:business_ethics|0|0 - extended|global_mmlu_instruct_cs_rus:business_ethics|0|0 - extended|global_mmlu_instruct_unk_rus:business_ethics|0|0 - extended|global_mmlu_instruct_all_srp:business_ethics|0|0 - extended|global_mmlu_instruct_ca_srp:business_ethics|0|0 - extended|global_mmlu_instruct_cs_srp:business_ethics|0|0 - extended|global_mmlu_instruct_unk_srp:business_ethics|0|0 - extended|global_mmlu_instruct_all_swe:business_ethics|0|0 - extended|global_mmlu_instruct_ca_swe:business_ethics|0|0 - extended|global_mmlu_instruct_cs_swe:business_ethics|0|0 - extended|global_mmlu_instruct_unk_swe:business_ethics|0|0 - extended|global_mmlu_instruct_all_swa:business_ethics|0|0 - extended|global_mmlu_instruct_ca_swa:business_ethics|0|0 - extended|global_mmlu_instruct_cs_swa:business_ethics|0|0 - extended|global_mmlu_instruct_unk_swa:business_ethics|0|0 - extended|global_mmlu_instruct_all_tel:business_ethics|0|0 - extended|global_mmlu_instruct_ca_tel:business_ethics|0|0 - extended|global_mmlu_instruct_cs_tel:business_ethics|0|0 - extended|global_mmlu_instruct_unk_tel:business_ethics|0|0 - extended|global_mmlu_instruct_all_tha:business_ethics|0|0 - extended|global_mmlu_instruct_ca_tha:business_ethics|0|0 - extended|global_mmlu_instruct_cs_tha:business_ethics|0|0 - extended|global_mmlu_instruct_unk_tha:business_ethics|0|0 - extended|global_mmlu_instruct_all_tur:business_ethics|0|0 - extended|global_mmlu_instruct_ca_tur:business_ethics|0|0 - extended|global_mmlu_instruct_cs_tur:business_ethics|0|0 - extended|global_mmlu_instruct_unk_tur:business_ethics|0|0 - extended|global_mmlu_instruct_all_ukr:business_ethics|0|0 - extended|global_mmlu_instruct_ca_ukr:business_ethics|0|0 - extended|global_mmlu_instruct_cs_ukr:business_ethics|0|0 - extended|global_mmlu_instruct_unk_ukr:business_ethics|0|0 - extended|global_mmlu_instruct_all_urd:business_ethics|0|0 - extended|global_mmlu_instruct_ca_urd:business_ethics|0|0 - extended|global_mmlu_instruct_cs_urd:business_ethics|0|0 - extended|global_mmlu_instruct_unk_urd:business_ethics|0|0 - extended|global_mmlu_instruct_all_vie:business_ethics|0|0 - extended|global_mmlu_instruct_ca_vie:business_ethics|0|0 - extended|global_mmlu_instruct_cs_vie:business_ethics|0|0 - extended|global_mmlu_instruct_unk_vie:business_ethics|0|0 - extended|global_mmlu_instruct_all_ara:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_ara:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_ara:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_ara:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_ben:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_ben:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_ben:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_ben:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_deu:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_deu:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_deu:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_deu:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_spa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_spa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_spa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_spa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_eng:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_eng:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_eng:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_eng:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_fra:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_fra:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_fra:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_fra:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_hin:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_hin:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_hin:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_hin:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_ind:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_ind:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_ind:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_ind:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_ita:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_ita:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_ita:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_ita:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_jpn:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_jpn:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_jpn:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_jpn:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_nld:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_nld:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_nld:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_nld:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_nor:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_nor:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_nor:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_nor:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_pol:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_pol:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_pol:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_pol:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_por:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_por:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_por:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_por:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_rus:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_rus:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_rus:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_rus:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_srp:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_srp:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_srp:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_srp:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_swe:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_swe:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_swe:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_swe:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_swa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_swa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_swa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_swa:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_tel:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_tel:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_tel:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_tel:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_tha:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_tha:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_tha:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_tha:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_tur:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_tur:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_tur:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_tur:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_ukr:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_ukr:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_ukr:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_ukr:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_urd:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_urd:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_urd:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_urd:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_vie:clinical_knowledge|0|0 - extended|global_mmlu_instruct_ca_vie:clinical_knowledge|0|0 - extended|global_mmlu_instruct_cs_vie:clinical_knowledge|0|0 - extended|global_mmlu_instruct_unk_vie:clinical_knowledge|0|0 - extended|global_mmlu_instruct_all_ara:college_biology|0|0 - extended|global_mmlu_instruct_ca_ara:college_biology|0|0 - extended|global_mmlu_instruct_cs_ara:college_biology|0|0 - extended|global_mmlu_instruct_unk_ara:college_biology|0|0 - extended|global_mmlu_instruct_all_ben:college_biology|0|0 - extended|global_mmlu_instruct_ca_ben:college_biology|0|0 - extended|global_mmlu_instruct_cs_ben:college_biology|0|0 - extended|global_mmlu_instruct_unk_ben:college_biology|0|0 - extended|global_mmlu_instruct_all_deu:college_biology|0|0 - extended|global_mmlu_instruct_ca_deu:college_biology|0|0 - extended|global_mmlu_instruct_cs_deu:college_biology|0|0 - extended|global_mmlu_instruct_unk_deu:college_biology|0|0 - extended|global_mmlu_instruct_all_spa:college_biology|0|0 - extended|global_mmlu_instruct_ca_spa:college_biology|0|0 - extended|global_mmlu_instruct_cs_spa:college_biology|0|0 - extended|global_mmlu_instruct_unk_spa:college_biology|0|0 - extended|global_mmlu_instruct_all_eng:college_biology|0|0 - extended|global_mmlu_instruct_ca_eng:college_biology|0|0 - extended|global_mmlu_instruct_cs_eng:college_biology|0|0 - extended|global_mmlu_instruct_unk_eng:college_biology|0|0 - extended|global_mmlu_instruct_all_fra:college_biology|0|0 - extended|global_mmlu_instruct_ca_fra:college_biology|0|0 - extended|global_mmlu_instruct_cs_fra:college_biology|0|0 - extended|global_mmlu_instruct_unk_fra:college_biology|0|0 - extended|global_mmlu_instruct_all_hin:college_biology|0|0 - extended|global_mmlu_instruct_ca_hin:college_biology|0|0 - extended|global_mmlu_instruct_cs_hin:college_biology|0|0 - extended|global_mmlu_instruct_unk_hin:college_biology|0|0 - extended|global_mmlu_instruct_all_ind:college_biology|0|0 - extended|global_mmlu_instruct_ca_ind:college_biology|0|0 - extended|global_mmlu_instruct_cs_ind:college_biology|0|0 - extended|global_mmlu_instruct_unk_ind:college_biology|0|0 - extended|global_mmlu_instruct_all_ita:college_biology|0|0 - extended|global_mmlu_instruct_ca_ita:college_biology|0|0 - extended|global_mmlu_instruct_cs_ita:college_biology|0|0 - extended|global_mmlu_instruct_unk_ita:college_biology|0|0 - extended|global_mmlu_instruct_all_jpn:college_biology|0|0 - extended|global_mmlu_instruct_ca_jpn:college_biology|0|0 - extended|global_mmlu_instruct_cs_jpn:college_biology|0|0 - extended|global_mmlu_instruct_unk_jpn:college_biology|0|0 - extended|global_mmlu_instruct_all_nld:college_biology|0|0 - extended|global_mmlu_instruct_ca_nld:college_biology|0|0 - extended|global_mmlu_instruct_cs_nld:college_biology|0|0 - extended|global_mmlu_instruct_unk_nld:college_biology|0|0 - extended|global_mmlu_instruct_all_nor:college_biology|0|0 - extended|global_mmlu_instruct_ca_nor:college_biology|0|0 - extended|global_mmlu_instruct_cs_nor:college_biology|0|0 - extended|global_mmlu_instruct_unk_nor:college_biology|0|0 - extended|global_mmlu_instruct_all_pol:college_biology|0|0 - extended|global_mmlu_instruct_ca_pol:college_biology|0|0 - extended|global_mmlu_instruct_cs_pol:college_biology|0|0 - extended|global_mmlu_instruct_unk_pol:college_biology|0|0 - extended|global_mmlu_instruct_all_por:college_biology|0|0 - extended|global_mmlu_instruct_ca_por:college_biology|0|0 - extended|global_mmlu_instruct_cs_por:college_biology|0|0 - extended|global_mmlu_instruct_unk_por:college_biology|0|0 - extended|global_mmlu_instruct_all_rus:college_biology|0|0 - extended|global_mmlu_instruct_ca_rus:college_biology|0|0 - extended|global_mmlu_instruct_cs_rus:college_biology|0|0 - extended|global_mmlu_instruct_unk_rus:college_biology|0|0 - extended|global_mmlu_instruct_all_srp:college_biology|0|0 - extended|global_mmlu_instruct_ca_srp:college_biology|0|0 - extended|global_mmlu_instruct_cs_srp:college_biology|0|0 - extended|global_mmlu_instruct_unk_srp:college_biology|0|0 - extended|global_mmlu_instruct_all_swe:college_biology|0|0 - extended|global_mmlu_instruct_ca_swe:college_biology|0|0 - extended|global_mmlu_instruct_cs_swe:college_biology|0|0 - extended|global_mmlu_instruct_unk_swe:college_biology|0|0 - extended|global_mmlu_instruct_all_swa:college_biology|0|0 - extended|global_mmlu_instruct_ca_swa:college_biology|0|0 - extended|global_mmlu_instruct_cs_swa:college_biology|0|0 - extended|global_mmlu_instruct_unk_swa:college_biology|0|0 - extended|global_mmlu_instruct_all_tel:college_biology|0|0 - extended|global_mmlu_instruct_ca_tel:college_biology|0|0 - extended|global_mmlu_instruct_cs_tel:college_biology|0|0 - extended|global_mmlu_instruct_unk_tel:college_biology|0|0 - extended|global_mmlu_instruct_all_tha:college_biology|0|0 - extended|global_mmlu_instruct_ca_tha:college_biology|0|0 - extended|global_mmlu_instruct_cs_tha:college_biology|0|0 - extended|global_mmlu_instruct_unk_tha:college_biology|0|0 - extended|global_mmlu_instruct_all_tur:college_biology|0|0 - extended|global_mmlu_instruct_ca_tur:college_biology|0|0 - extended|global_mmlu_instruct_cs_tur:college_biology|0|0 - extended|global_mmlu_instruct_unk_tur:college_biology|0|0 - extended|global_mmlu_instruct_all_ukr:college_biology|0|0 - extended|global_mmlu_instruct_ca_ukr:college_biology|0|0 - extended|global_mmlu_instruct_cs_ukr:college_biology|0|0 - extended|global_mmlu_instruct_unk_ukr:college_biology|0|0 - extended|global_mmlu_instruct_all_urd:college_biology|0|0 - extended|global_mmlu_instruct_ca_urd:college_biology|0|0 - extended|global_mmlu_instruct_cs_urd:college_biology|0|0 - extended|global_mmlu_instruct_unk_urd:college_biology|0|0 - extended|global_mmlu_instruct_all_vie:college_biology|0|0 - extended|global_mmlu_instruct_ca_vie:college_biology|0|0 - extended|global_mmlu_instruct_cs_vie:college_biology|0|0 - extended|global_mmlu_instruct_unk_vie:college_biology|0|0 - extended|global_mmlu_instruct_all_ara:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_ara:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_ara:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_ara:college_chemistry|0|0 - extended|global_mmlu_instruct_all_ben:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_ben:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_ben:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_ben:college_chemistry|0|0 - extended|global_mmlu_instruct_all_deu:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_deu:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_deu:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_deu:college_chemistry|0|0 - extended|global_mmlu_instruct_all_spa:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_spa:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_spa:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_spa:college_chemistry|0|0 - extended|global_mmlu_instruct_all_eng:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_eng:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_eng:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_eng:college_chemistry|0|0 - extended|global_mmlu_instruct_all_fra:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_fra:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_fra:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_fra:college_chemistry|0|0 - extended|global_mmlu_instruct_all_hin:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_hin:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_hin:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_hin:college_chemistry|0|0 - extended|global_mmlu_instruct_all_ind:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_ind:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_ind:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_ind:college_chemistry|0|0 - extended|global_mmlu_instruct_all_ita:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_ita:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_ita:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_ita:college_chemistry|0|0 - extended|global_mmlu_instruct_all_jpn:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_jpn:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_jpn:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_jpn:college_chemistry|0|0 - extended|global_mmlu_instruct_all_nld:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_nld:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_nld:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_nld:college_chemistry|0|0 - extended|global_mmlu_instruct_all_nor:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_nor:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_nor:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_nor:college_chemistry|0|0 - extended|global_mmlu_instruct_all_pol:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_pol:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_pol:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_pol:college_chemistry|0|0 - extended|global_mmlu_instruct_all_por:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_por:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_por:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_por:college_chemistry|0|0 - extended|global_mmlu_instruct_all_rus:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_rus:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_rus:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_rus:college_chemistry|0|0 - extended|global_mmlu_instruct_all_srp:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_srp:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_srp:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_srp:college_chemistry|0|0 - extended|global_mmlu_instruct_all_swe:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_swe:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_swe:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_swe:college_chemistry|0|0 - extended|global_mmlu_instruct_all_swa:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_swa:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_swa:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_swa:college_chemistry|0|0 - extended|global_mmlu_instruct_all_tel:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_tel:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_tel:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_tel:college_chemistry|0|0 - extended|global_mmlu_instruct_all_tha:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_tha:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_tha:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_tha:college_chemistry|0|0 - extended|global_mmlu_instruct_all_tur:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_tur:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_tur:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_tur:college_chemistry|0|0 - extended|global_mmlu_instruct_all_ukr:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_ukr:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_ukr:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_ukr:college_chemistry|0|0 - extended|global_mmlu_instruct_all_urd:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_urd:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_urd:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_urd:college_chemistry|0|0 - extended|global_mmlu_instruct_all_vie:college_chemistry|0|0 - extended|global_mmlu_instruct_ca_vie:college_chemistry|0|0 - extended|global_mmlu_instruct_cs_vie:college_chemistry|0|0 - extended|global_mmlu_instruct_unk_vie:college_chemistry|0|0 - extended|global_mmlu_instruct_all_ara:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_ara:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_ara:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_ara:college_computer_science|0|0 - extended|global_mmlu_instruct_all_ben:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_ben:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_ben:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_ben:college_computer_science|0|0 - extended|global_mmlu_instruct_all_deu:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_deu:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_deu:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_deu:college_computer_science|0|0 - extended|global_mmlu_instruct_all_spa:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_spa:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_spa:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_spa:college_computer_science|0|0 - extended|global_mmlu_instruct_all_eng:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_eng:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_eng:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_eng:college_computer_science|0|0 - extended|global_mmlu_instruct_all_fra:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_fra:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_fra:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_fra:college_computer_science|0|0 - extended|global_mmlu_instruct_all_hin:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_hin:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_hin:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_hin:college_computer_science|0|0 - extended|global_mmlu_instruct_all_ind:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_ind:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_ind:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_ind:college_computer_science|0|0 - extended|global_mmlu_instruct_all_ita:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_ita:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_ita:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_ita:college_computer_science|0|0 - extended|global_mmlu_instruct_all_jpn:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_jpn:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_jpn:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_jpn:college_computer_science|0|0 - extended|global_mmlu_instruct_all_nld:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_nld:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_nld:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_nld:college_computer_science|0|0 - extended|global_mmlu_instruct_all_nor:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_nor:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_nor:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_nor:college_computer_science|0|0 - extended|global_mmlu_instruct_all_pol:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_pol:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_pol:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_pol:college_computer_science|0|0 - extended|global_mmlu_instruct_all_por:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_por:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_por:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_por:college_computer_science|0|0 - extended|global_mmlu_instruct_all_rus:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_rus:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_rus:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_rus:college_computer_science|0|0 - extended|global_mmlu_instruct_all_srp:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_srp:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_srp:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_srp:college_computer_science|0|0 - extended|global_mmlu_instruct_all_swe:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_swe:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_swe:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_swe:college_computer_science|0|0 - extended|global_mmlu_instruct_all_swa:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_swa:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_swa:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_swa:college_computer_science|0|0 - extended|global_mmlu_instruct_all_tel:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_tel:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_tel:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_tel:college_computer_science|0|0 - extended|global_mmlu_instruct_all_tha:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_tha:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_tha:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_tha:college_computer_science|0|0 - extended|global_mmlu_instruct_all_tur:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_tur:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_tur:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_tur:college_computer_science|0|0 - extended|global_mmlu_instruct_all_ukr:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_ukr:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_ukr:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_ukr:college_computer_science|0|0 - extended|global_mmlu_instruct_all_urd:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_urd:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_urd:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_urd:college_computer_science|0|0 - extended|global_mmlu_instruct_all_vie:college_computer_science|0|0 - extended|global_mmlu_instruct_ca_vie:college_computer_science|0|0 - extended|global_mmlu_instruct_cs_vie:college_computer_science|0|0 - extended|global_mmlu_instruct_unk_vie:college_computer_science|0|0 - extended|global_mmlu_instruct_all_ara:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_ara:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_ara:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_ara:college_mathematics|0|0 - extended|global_mmlu_instruct_all_ben:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_ben:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_ben:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_ben:college_mathematics|0|0 - extended|global_mmlu_instruct_all_deu:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_deu:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_deu:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_deu:college_mathematics|0|0 - extended|global_mmlu_instruct_all_spa:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_spa:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_spa:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_spa:college_mathematics|0|0 - extended|global_mmlu_instruct_all_eng:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_eng:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_eng:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_eng:college_mathematics|0|0 - extended|global_mmlu_instruct_all_fra:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_fra:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_fra:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_fra:college_mathematics|0|0 - extended|global_mmlu_instruct_all_hin:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_hin:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_hin:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_hin:college_mathematics|0|0 - extended|global_mmlu_instruct_all_ind:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_ind:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_ind:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_ind:college_mathematics|0|0 - extended|global_mmlu_instruct_all_ita:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_ita:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_ita:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_ita:college_mathematics|0|0 - extended|global_mmlu_instruct_all_jpn:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_jpn:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_jpn:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_jpn:college_mathematics|0|0 - extended|global_mmlu_instruct_all_nld:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_nld:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_nld:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_nld:college_mathematics|0|0 - extended|global_mmlu_instruct_all_nor:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_nor:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_nor:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_nor:college_mathematics|0|0 - extended|global_mmlu_instruct_all_pol:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_pol:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_pol:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_pol:college_mathematics|0|0 - extended|global_mmlu_instruct_all_por:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_por:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_por:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_por:college_mathematics|0|0 - extended|global_mmlu_instruct_all_rus:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_rus:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_rus:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_rus:college_mathematics|0|0 - extended|global_mmlu_instruct_all_srp:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_srp:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_srp:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_srp:college_mathematics|0|0 - extended|global_mmlu_instruct_all_swe:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_swe:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_swe:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_swe:college_mathematics|0|0 - extended|global_mmlu_instruct_all_swa:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_swa:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_swa:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_swa:college_mathematics|0|0 - extended|global_mmlu_instruct_all_tel:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_tel:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_tel:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_tel:college_mathematics|0|0 - extended|global_mmlu_instruct_all_tha:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_tha:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_tha:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_tha:college_mathematics|0|0 - extended|global_mmlu_instruct_all_tur:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_tur:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_tur:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_tur:college_mathematics|0|0 - extended|global_mmlu_instruct_all_ukr:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_ukr:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_ukr:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_ukr:college_mathematics|0|0 - extended|global_mmlu_instruct_all_urd:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_urd:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_urd:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_urd:college_mathematics|0|0 - extended|global_mmlu_instruct_all_vie:college_mathematics|0|0 - extended|global_mmlu_instruct_ca_vie:college_mathematics|0|0 - extended|global_mmlu_instruct_cs_vie:college_mathematics|0|0 - extended|global_mmlu_instruct_unk_vie:college_mathematics|0|0 - extended|global_mmlu_instruct_all_ara:college_medicine|0|0 - extended|global_mmlu_instruct_ca_ara:college_medicine|0|0 - extended|global_mmlu_instruct_cs_ara:college_medicine|0|0 - extended|global_mmlu_instruct_unk_ara:college_medicine|0|0 - extended|global_mmlu_instruct_all_ben:college_medicine|0|0 - extended|global_mmlu_instruct_ca_ben:college_medicine|0|0 - extended|global_mmlu_instruct_cs_ben:college_medicine|0|0 - extended|global_mmlu_instruct_unk_ben:college_medicine|0|0 - extended|global_mmlu_instruct_all_deu:college_medicine|0|0 - extended|global_mmlu_instruct_ca_deu:college_medicine|0|0 - extended|global_mmlu_instruct_cs_deu:college_medicine|0|0 - extended|global_mmlu_instruct_unk_deu:college_medicine|0|0 - extended|global_mmlu_instruct_all_spa:college_medicine|0|0 - extended|global_mmlu_instruct_ca_spa:college_medicine|0|0 - extended|global_mmlu_instruct_cs_spa:college_medicine|0|0 - extended|global_mmlu_instruct_unk_spa:college_medicine|0|0 - extended|global_mmlu_instruct_all_eng:college_medicine|0|0 - extended|global_mmlu_instruct_ca_eng:college_medicine|0|0 - extended|global_mmlu_instruct_cs_eng:college_medicine|0|0 - extended|global_mmlu_instruct_unk_eng:college_medicine|0|0 - extended|global_mmlu_instruct_all_fra:college_medicine|0|0 - extended|global_mmlu_instruct_ca_fra:college_medicine|0|0 - extended|global_mmlu_instruct_cs_fra:college_medicine|0|0 - extended|global_mmlu_instruct_unk_fra:college_medicine|0|0 - extended|global_mmlu_instruct_all_hin:college_medicine|0|0 - extended|global_mmlu_instruct_ca_hin:college_medicine|0|0 - extended|global_mmlu_instruct_cs_hin:college_medicine|0|0 - extended|global_mmlu_instruct_unk_hin:college_medicine|0|0 - extended|global_mmlu_instruct_all_ind:college_medicine|0|0 - extended|global_mmlu_instruct_ca_ind:college_medicine|0|0 - extended|global_mmlu_instruct_cs_ind:college_medicine|0|0 - extended|global_mmlu_instruct_unk_ind:college_medicine|0|0 - extended|global_mmlu_instruct_all_ita:college_medicine|0|0 - extended|global_mmlu_instruct_ca_ita:college_medicine|0|0 - extended|global_mmlu_instruct_cs_ita:college_medicine|0|0 - extended|global_mmlu_instruct_unk_ita:college_medicine|0|0 - extended|global_mmlu_instruct_all_jpn:college_medicine|0|0 - extended|global_mmlu_instruct_ca_jpn:college_medicine|0|0 - extended|global_mmlu_instruct_cs_jpn:college_medicine|0|0 - extended|global_mmlu_instruct_unk_jpn:college_medicine|0|0 - extended|global_mmlu_instruct_all_nld:college_medicine|0|0 - extended|global_mmlu_instruct_ca_nld:college_medicine|0|0 - extended|global_mmlu_instruct_cs_nld:college_medicine|0|0 - extended|global_mmlu_instruct_unk_nld:college_medicine|0|0 - extended|global_mmlu_instruct_all_nor:college_medicine|0|0 - extended|global_mmlu_instruct_ca_nor:college_medicine|0|0 - extended|global_mmlu_instruct_cs_nor:college_medicine|0|0 - extended|global_mmlu_instruct_unk_nor:college_medicine|0|0 - extended|global_mmlu_instruct_all_pol:college_medicine|0|0 - extended|global_mmlu_instruct_ca_pol:college_medicine|0|0 - extended|global_mmlu_instruct_cs_pol:college_medicine|0|0 - extended|global_mmlu_instruct_unk_pol:college_medicine|0|0 - extended|global_mmlu_instruct_all_por:college_medicine|0|0 - extended|global_mmlu_instruct_ca_por:college_medicine|0|0 - extended|global_mmlu_instruct_cs_por:college_medicine|0|0 - extended|global_mmlu_instruct_unk_por:college_medicine|0|0 - extended|global_mmlu_instruct_all_rus:college_medicine|0|0 - extended|global_mmlu_instruct_ca_rus:college_medicine|0|0 - extended|global_mmlu_instruct_cs_rus:college_medicine|0|0 - extended|global_mmlu_instruct_unk_rus:college_medicine|0|0 - extended|global_mmlu_instruct_all_srp:college_medicine|0|0 - extended|global_mmlu_instruct_ca_srp:college_medicine|0|0 - extended|global_mmlu_instruct_cs_srp:college_medicine|0|0 - extended|global_mmlu_instruct_unk_srp:college_medicine|0|0 - extended|global_mmlu_instruct_all_swe:college_medicine|0|0 - extended|global_mmlu_instruct_ca_swe:college_medicine|0|0 - extended|global_mmlu_instruct_cs_swe:college_medicine|0|0 - extended|global_mmlu_instruct_unk_swe:college_medicine|0|0 - extended|global_mmlu_instruct_all_swa:college_medicine|0|0 - extended|global_mmlu_instruct_ca_swa:college_medicine|0|0 - extended|global_mmlu_instruct_cs_swa:college_medicine|0|0 - extended|global_mmlu_instruct_unk_swa:college_medicine|0|0 - extended|global_mmlu_instruct_all_tel:college_medicine|0|0 - extended|global_mmlu_instruct_ca_tel:college_medicine|0|0 - extended|global_mmlu_instruct_cs_tel:college_medicine|0|0 - extended|global_mmlu_instruct_unk_tel:college_medicine|0|0 - extended|global_mmlu_instruct_all_tha:college_medicine|0|0 - extended|global_mmlu_instruct_ca_tha:college_medicine|0|0 - extended|global_mmlu_instruct_cs_tha:college_medicine|0|0 - extended|global_mmlu_instruct_unk_tha:college_medicine|0|0 - extended|global_mmlu_instruct_all_tur:college_medicine|0|0 - extended|global_mmlu_instruct_ca_tur:college_medicine|0|0 - extended|global_mmlu_instruct_cs_tur:college_medicine|0|0 - extended|global_mmlu_instruct_unk_tur:college_medicine|0|0 - extended|global_mmlu_instruct_all_ukr:college_medicine|0|0 - extended|global_mmlu_instruct_ca_ukr:college_medicine|0|0 - extended|global_mmlu_instruct_cs_ukr:college_medicine|0|0 - extended|global_mmlu_instruct_unk_ukr:college_medicine|0|0 - extended|global_mmlu_instruct_all_urd:college_medicine|0|0 - extended|global_mmlu_instruct_ca_urd:college_medicine|0|0 - extended|global_mmlu_instruct_cs_urd:college_medicine|0|0 - extended|global_mmlu_instruct_unk_urd:college_medicine|0|0 - extended|global_mmlu_instruct_all_vie:college_medicine|0|0 - extended|global_mmlu_instruct_ca_vie:college_medicine|0|0 - extended|global_mmlu_instruct_cs_vie:college_medicine|0|0 - extended|global_mmlu_instruct_unk_vie:college_medicine|0|0 - extended|global_mmlu_instruct_all_ara:college_physics|0|0 - extended|global_mmlu_instruct_ca_ara:college_physics|0|0 - extended|global_mmlu_instruct_cs_ara:college_physics|0|0 - extended|global_mmlu_instruct_unk_ara:college_physics|0|0 - extended|global_mmlu_instruct_all_ben:college_physics|0|0 - extended|global_mmlu_instruct_ca_ben:college_physics|0|0 - extended|global_mmlu_instruct_cs_ben:college_physics|0|0 - extended|global_mmlu_instruct_unk_ben:college_physics|0|0 - extended|global_mmlu_instruct_all_deu:college_physics|0|0 - extended|global_mmlu_instruct_ca_deu:college_physics|0|0 - extended|global_mmlu_instruct_cs_deu:college_physics|0|0 - extended|global_mmlu_instruct_unk_deu:college_physics|0|0 - extended|global_mmlu_instruct_all_spa:college_physics|0|0 - extended|global_mmlu_instruct_ca_spa:college_physics|0|0 - extended|global_mmlu_instruct_cs_spa:college_physics|0|0 - extended|global_mmlu_instruct_unk_spa:college_physics|0|0 - extended|global_mmlu_instruct_all_eng:college_physics|0|0 - extended|global_mmlu_instruct_ca_eng:college_physics|0|0 - extended|global_mmlu_instruct_cs_eng:college_physics|0|0 - extended|global_mmlu_instruct_unk_eng:college_physics|0|0 - extended|global_mmlu_instruct_all_fra:college_physics|0|0 - extended|global_mmlu_instruct_ca_fra:college_physics|0|0 - extended|global_mmlu_instruct_cs_fra:college_physics|0|0 - extended|global_mmlu_instruct_unk_fra:college_physics|0|0 - extended|global_mmlu_instruct_all_hin:college_physics|0|0 - extended|global_mmlu_instruct_ca_hin:college_physics|0|0 - extended|global_mmlu_instruct_cs_hin:college_physics|0|0 - extended|global_mmlu_instruct_unk_hin:college_physics|0|0 - extended|global_mmlu_instruct_all_ind:college_physics|0|0 - extended|global_mmlu_instruct_ca_ind:college_physics|0|0 - extended|global_mmlu_instruct_cs_ind:college_physics|0|0 - extended|global_mmlu_instruct_unk_ind:college_physics|0|0 - extended|global_mmlu_instruct_all_ita:college_physics|0|0 - extended|global_mmlu_instruct_ca_ita:college_physics|0|0 - extended|global_mmlu_instruct_cs_ita:college_physics|0|0 - extended|global_mmlu_instruct_unk_ita:college_physics|0|0 - extended|global_mmlu_instruct_all_jpn:college_physics|0|0 - extended|global_mmlu_instruct_ca_jpn:college_physics|0|0 - extended|global_mmlu_instruct_cs_jpn:college_physics|0|0 - extended|global_mmlu_instruct_unk_jpn:college_physics|0|0 - extended|global_mmlu_instruct_all_nld:college_physics|0|0 - extended|global_mmlu_instruct_ca_nld:college_physics|0|0 - extended|global_mmlu_instruct_cs_nld:college_physics|0|0 - extended|global_mmlu_instruct_unk_nld:college_physics|0|0 - extended|global_mmlu_instruct_all_nor:college_physics|0|0 - extended|global_mmlu_instruct_ca_nor:college_physics|0|0 - extended|global_mmlu_instruct_cs_nor:college_physics|0|0 - extended|global_mmlu_instruct_unk_nor:college_physics|0|0 - extended|global_mmlu_instruct_all_pol:college_physics|0|0 - extended|global_mmlu_instruct_ca_pol:college_physics|0|0 - extended|global_mmlu_instruct_cs_pol:college_physics|0|0 - extended|global_mmlu_instruct_unk_pol:college_physics|0|0 - extended|global_mmlu_instruct_all_por:college_physics|0|0 - extended|global_mmlu_instruct_ca_por:college_physics|0|0 - extended|global_mmlu_instruct_cs_por:college_physics|0|0 - extended|global_mmlu_instruct_unk_por:college_physics|0|0 - extended|global_mmlu_instruct_all_rus:college_physics|0|0 - extended|global_mmlu_instruct_ca_rus:college_physics|0|0 - extended|global_mmlu_instruct_cs_rus:college_physics|0|0 - extended|global_mmlu_instruct_unk_rus:college_physics|0|0 - extended|global_mmlu_instruct_all_srp:college_physics|0|0 - extended|global_mmlu_instruct_ca_srp:college_physics|0|0 - extended|global_mmlu_instruct_cs_srp:college_physics|0|0 - extended|global_mmlu_instruct_unk_srp:college_physics|0|0 - extended|global_mmlu_instruct_all_swe:college_physics|0|0 - extended|global_mmlu_instruct_ca_swe:college_physics|0|0 - extended|global_mmlu_instruct_cs_swe:college_physics|0|0 - extended|global_mmlu_instruct_unk_swe:college_physics|0|0 - extended|global_mmlu_instruct_all_swa:college_physics|0|0 - extended|global_mmlu_instruct_ca_swa:college_physics|0|0 - extended|global_mmlu_instruct_cs_swa:college_physics|0|0 - extended|global_mmlu_instruct_unk_swa:college_physics|0|0 - extended|global_mmlu_instruct_all_tel:college_physics|0|0 - extended|global_mmlu_instruct_ca_tel:college_physics|0|0 - extended|global_mmlu_instruct_cs_tel:college_physics|0|0 - extended|global_mmlu_instruct_unk_tel:college_physics|0|0 - extended|global_mmlu_instruct_all_tha:college_physics|0|0 - extended|global_mmlu_instruct_ca_tha:college_physics|0|0 - extended|global_mmlu_instruct_cs_tha:college_physics|0|0 - extended|global_mmlu_instruct_unk_tha:college_physics|0|0 - extended|global_mmlu_instruct_all_tur:college_physics|0|0 - extended|global_mmlu_instruct_ca_tur:college_physics|0|0 - extended|global_mmlu_instruct_cs_tur:college_physics|0|0 - extended|global_mmlu_instruct_unk_tur:college_physics|0|0 - extended|global_mmlu_instruct_all_ukr:college_physics|0|0 - extended|global_mmlu_instruct_ca_ukr:college_physics|0|0 - extended|global_mmlu_instruct_cs_ukr:college_physics|0|0 - extended|global_mmlu_instruct_unk_ukr:college_physics|0|0 - extended|global_mmlu_instruct_all_urd:college_physics|0|0 - extended|global_mmlu_instruct_ca_urd:college_physics|0|0 - extended|global_mmlu_instruct_cs_urd:college_physics|0|0 - extended|global_mmlu_instruct_unk_urd:college_physics|0|0 - extended|global_mmlu_instruct_all_vie:college_physics|0|0 - extended|global_mmlu_instruct_ca_vie:college_physics|0|0 - extended|global_mmlu_instruct_cs_vie:college_physics|0|0 - extended|global_mmlu_instruct_unk_vie:college_physics|0|0 - extended|global_mmlu_instruct_all_ara:computer_security|0|0 - extended|global_mmlu_instruct_ca_ara:computer_security|0|0 - extended|global_mmlu_instruct_cs_ara:computer_security|0|0 - extended|global_mmlu_instruct_unk_ara:computer_security|0|0 - extended|global_mmlu_instruct_all_ben:computer_security|0|0 - extended|global_mmlu_instruct_ca_ben:computer_security|0|0 - extended|global_mmlu_instruct_cs_ben:computer_security|0|0 - extended|global_mmlu_instruct_unk_ben:computer_security|0|0 - extended|global_mmlu_instruct_all_deu:computer_security|0|0 - extended|global_mmlu_instruct_ca_deu:computer_security|0|0 - extended|global_mmlu_instruct_cs_deu:computer_security|0|0 - extended|global_mmlu_instruct_unk_deu:computer_security|0|0 - extended|global_mmlu_instruct_all_spa:computer_security|0|0 - extended|global_mmlu_instruct_ca_spa:computer_security|0|0 - extended|global_mmlu_instruct_cs_spa:computer_security|0|0 - extended|global_mmlu_instruct_unk_spa:computer_security|0|0 - extended|global_mmlu_instruct_all_eng:computer_security|0|0 - extended|global_mmlu_instruct_ca_eng:computer_security|0|0 - extended|global_mmlu_instruct_cs_eng:computer_security|0|0 - extended|global_mmlu_instruct_unk_eng:computer_security|0|0 - extended|global_mmlu_instruct_all_fra:computer_security|0|0 - extended|global_mmlu_instruct_ca_fra:computer_security|0|0 - extended|global_mmlu_instruct_cs_fra:computer_security|0|0 - extended|global_mmlu_instruct_unk_fra:computer_security|0|0 - extended|global_mmlu_instruct_all_hin:computer_security|0|0 - extended|global_mmlu_instruct_ca_hin:computer_security|0|0 - extended|global_mmlu_instruct_cs_hin:computer_security|0|0 - extended|global_mmlu_instruct_unk_hin:computer_security|0|0 - extended|global_mmlu_instruct_all_ind:computer_security|0|0 - extended|global_mmlu_instruct_ca_ind:computer_security|0|0 - extended|global_mmlu_instruct_cs_ind:computer_security|0|0 - extended|global_mmlu_instruct_unk_ind:computer_security|0|0 - extended|global_mmlu_instruct_all_ita:computer_security|0|0 - extended|global_mmlu_instruct_ca_ita:computer_security|0|0 - extended|global_mmlu_instruct_cs_ita:computer_security|0|0 - extended|global_mmlu_instruct_unk_ita:computer_security|0|0 - extended|global_mmlu_instruct_all_jpn:computer_security|0|0 - extended|global_mmlu_instruct_ca_jpn:computer_security|0|0 - extended|global_mmlu_instruct_cs_jpn:computer_security|0|0 - extended|global_mmlu_instruct_unk_jpn:computer_security|0|0 - extended|global_mmlu_instruct_all_nld:computer_security|0|0 - extended|global_mmlu_instruct_ca_nld:computer_security|0|0 - extended|global_mmlu_instruct_cs_nld:computer_security|0|0 - extended|global_mmlu_instruct_unk_nld:computer_security|0|0 - extended|global_mmlu_instruct_all_nor:computer_security|0|0 - extended|global_mmlu_instruct_ca_nor:computer_security|0|0 - extended|global_mmlu_instruct_cs_nor:computer_security|0|0 - extended|global_mmlu_instruct_unk_nor:computer_security|0|0 - extended|global_mmlu_instruct_all_pol:computer_security|0|0 - extended|global_mmlu_instruct_ca_pol:computer_security|0|0 - extended|global_mmlu_instruct_cs_pol:computer_security|0|0 - extended|global_mmlu_instruct_unk_pol:computer_security|0|0 - extended|global_mmlu_instruct_all_por:computer_security|0|0 - extended|global_mmlu_instruct_ca_por:computer_security|0|0 - extended|global_mmlu_instruct_cs_por:computer_security|0|0 - extended|global_mmlu_instruct_unk_por:computer_security|0|0 - extended|global_mmlu_instruct_all_rus:computer_security|0|0 - extended|global_mmlu_instruct_ca_rus:computer_security|0|0 - extended|global_mmlu_instruct_cs_rus:computer_security|0|0 - extended|global_mmlu_instruct_unk_rus:computer_security|0|0 - extended|global_mmlu_instruct_all_srp:computer_security|0|0 - extended|global_mmlu_instruct_ca_srp:computer_security|0|0 - extended|global_mmlu_instruct_cs_srp:computer_security|0|0 - extended|global_mmlu_instruct_unk_srp:computer_security|0|0 - extended|global_mmlu_instruct_all_swe:computer_security|0|0 - extended|global_mmlu_instruct_ca_swe:computer_security|0|0 - extended|global_mmlu_instruct_cs_swe:computer_security|0|0 - extended|global_mmlu_instruct_unk_swe:computer_security|0|0 - extended|global_mmlu_instruct_all_swa:computer_security|0|0 - extended|global_mmlu_instruct_ca_swa:computer_security|0|0 - extended|global_mmlu_instruct_cs_swa:computer_security|0|0 - extended|global_mmlu_instruct_unk_swa:computer_security|0|0 - extended|global_mmlu_instruct_all_tel:computer_security|0|0 - extended|global_mmlu_instruct_ca_tel:computer_security|0|0 - extended|global_mmlu_instruct_cs_tel:computer_security|0|0 - extended|global_mmlu_instruct_unk_tel:computer_security|0|0 - extended|global_mmlu_instruct_all_tha:computer_security|0|0 - extended|global_mmlu_instruct_ca_tha:computer_security|0|0 - extended|global_mmlu_instruct_cs_tha:computer_security|0|0 - extended|global_mmlu_instruct_unk_tha:computer_security|0|0 - extended|global_mmlu_instruct_all_tur:computer_security|0|0 - extended|global_mmlu_instruct_ca_tur:computer_security|0|0 - extended|global_mmlu_instruct_cs_tur:computer_security|0|0 - extended|global_mmlu_instruct_unk_tur:computer_security|0|0 - extended|global_mmlu_instruct_all_ukr:computer_security|0|0 - extended|global_mmlu_instruct_ca_ukr:computer_security|0|0 - extended|global_mmlu_instruct_cs_ukr:computer_security|0|0 - extended|global_mmlu_instruct_unk_ukr:computer_security|0|0 - extended|global_mmlu_instruct_all_urd:computer_security|0|0 - extended|global_mmlu_instruct_ca_urd:computer_security|0|0 - extended|global_mmlu_instruct_cs_urd:computer_security|0|0 - extended|global_mmlu_instruct_unk_urd:computer_security|0|0 - extended|global_mmlu_instruct_all_vie:computer_security|0|0 - extended|global_mmlu_instruct_ca_vie:computer_security|0|0 - extended|global_mmlu_instruct_cs_vie:computer_security|0|0 - extended|global_mmlu_instruct_unk_vie:computer_security|0|0 - extended|global_mmlu_instruct_all_ara:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_ara:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_ara:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_ara:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_ben:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_ben:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_ben:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_ben:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_deu:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_deu:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_deu:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_deu:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_spa:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_spa:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_spa:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_spa:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_eng:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_eng:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_eng:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_eng:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_fra:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_fra:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_fra:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_fra:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_hin:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_hin:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_hin:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_hin:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_ind:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_ind:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_ind:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_ind:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_ita:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_ita:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_ita:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_ita:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_jpn:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_jpn:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_jpn:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_jpn:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_nld:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_nld:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_nld:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_nld:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_nor:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_nor:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_nor:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_nor:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_pol:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_pol:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_pol:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_pol:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_por:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_por:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_por:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_por:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_rus:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_rus:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_rus:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_rus:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_srp:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_srp:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_srp:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_srp:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_swe:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_swe:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_swe:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_swe:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_swa:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_swa:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_swa:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_swa:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_tel:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_tel:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_tel:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_tel:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_tha:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_tha:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_tha:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_tha:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_tur:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_tur:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_tur:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_tur:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_ukr:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_ukr:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_ukr:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_ukr:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_urd:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_urd:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_urd:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_urd:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_vie:conceptual_physics|0|0 - extended|global_mmlu_instruct_ca_vie:conceptual_physics|0|0 - extended|global_mmlu_instruct_cs_vie:conceptual_physics|0|0 - extended|global_mmlu_instruct_unk_vie:conceptual_physics|0|0 - extended|global_mmlu_instruct_all_ara:econometrics|0|0 - extended|global_mmlu_instruct_ca_ara:econometrics|0|0 - extended|global_mmlu_instruct_cs_ara:econometrics|0|0 - extended|global_mmlu_instruct_unk_ara:econometrics|0|0 - extended|global_mmlu_instruct_all_ben:econometrics|0|0 - extended|global_mmlu_instruct_ca_ben:econometrics|0|0 - extended|global_mmlu_instruct_cs_ben:econometrics|0|0 - extended|global_mmlu_instruct_unk_ben:econometrics|0|0 - extended|global_mmlu_instruct_all_deu:econometrics|0|0 - extended|global_mmlu_instruct_ca_deu:econometrics|0|0 - extended|global_mmlu_instruct_cs_deu:econometrics|0|0 - extended|global_mmlu_instruct_unk_deu:econometrics|0|0 - extended|global_mmlu_instruct_all_spa:econometrics|0|0 - extended|global_mmlu_instruct_ca_spa:econometrics|0|0 - extended|global_mmlu_instruct_cs_spa:econometrics|0|0 - extended|global_mmlu_instruct_unk_spa:econometrics|0|0 - extended|global_mmlu_instruct_all_eng:econometrics|0|0 - extended|global_mmlu_instruct_ca_eng:econometrics|0|0 - extended|global_mmlu_instruct_cs_eng:econometrics|0|0 - extended|global_mmlu_instruct_unk_eng:econometrics|0|0 - extended|global_mmlu_instruct_all_fra:econometrics|0|0 - extended|global_mmlu_instruct_ca_fra:econometrics|0|0 - extended|global_mmlu_instruct_cs_fra:econometrics|0|0 - extended|global_mmlu_instruct_unk_fra:econometrics|0|0 - extended|global_mmlu_instruct_all_hin:econometrics|0|0 - extended|global_mmlu_instruct_ca_hin:econometrics|0|0 - extended|global_mmlu_instruct_cs_hin:econometrics|0|0 - extended|global_mmlu_instruct_unk_hin:econometrics|0|0 - extended|global_mmlu_instruct_all_ind:econometrics|0|0 - extended|global_mmlu_instruct_ca_ind:econometrics|0|0 - extended|global_mmlu_instruct_cs_ind:econometrics|0|0 - extended|global_mmlu_instruct_unk_ind:econometrics|0|0 - extended|global_mmlu_instruct_all_ita:econometrics|0|0 - extended|global_mmlu_instruct_ca_ita:econometrics|0|0 - extended|global_mmlu_instruct_cs_ita:econometrics|0|0 - extended|global_mmlu_instruct_unk_ita:econometrics|0|0 - extended|global_mmlu_instruct_all_jpn:econometrics|0|0 - extended|global_mmlu_instruct_ca_jpn:econometrics|0|0 - extended|global_mmlu_instruct_cs_jpn:econometrics|0|0 - extended|global_mmlu_instruct_unk_jpn:econometrics|0|0 - extended|global_mmlu_instruct_all_nld:econometrics|0|0 - extended|global_mmlu_instruct_ca_nld:econometrics|0|0 - extended|global_mmlu_instruct_cs_nld:econometrics|0|0 - extended|global_mmlu_instruct_unk_nld:econometrics|0|0 - extended|global_mmlu_instruct_all_nor:econometrics|0|0 - extended|global_mmlu_instruct_ca_nor:econometrics|0|0 - extended|global_mmlu_instruct_cs_nor:econometrics|0|0 - extended|global_mmlu_instruct_unk_nor:econometrics|0|0 - extended|global_mmlu_instruct_all_pol:econometrics|0|0 - extended|global_mmlu_instruct_ca_pol:econometrics|0|0 - extended|global_mmlu_instruct_cs_pol:econometrics|0|0 - extended|global_mmlu_instruct_unk_pol:econometrics|0|0 - extended|global_mmlu_instruct_all_por:econometrics|0|0 - extended|global_mmlu_instruct_ca_por:econometrics|0|0 - extended|global_mmlu_instruct_cs_por:econometrics|0|0 - extended|global_mmlu_instruct_unk_por:econometrics|0|0 - extended|global_mmlu_instruct_all_rus:econometrics|0|0 - extended|global_mmlu_instruct_ca_rus:econometrics|0|0 - extended|global_mmlu_instruct_cs_rus:econometrics|0|0 - extended|global_mmlu_instruct_unk_rus:econometrics|0|0 - extended|global_mmlu_instruct_all_srp:econometrics|0|0 - extended|global_mmlu_instruct_ca_srp:econometrics|0|0 - extended|global_mmlu_instruct_cs_srp:econometrics|0|0 - extended|global_mmlu_instruct_unk_srp:econometrics|0|0 - extended|global_mmlu_instruct_all_swe:econometrics|0|0 - extended|global_mmlu_instruct_ca_swe:econometrics|0|0 - extended|global_mmlu_instruct_cs_swe:econometrics|0|0 - extended|global_mmlu_instruct_unk_swe:econometrics|0|0 - extended|global_mmlu_instruct_all_swa:econometrics|0|0 - extended|global_mmlu_instruct_ca_swa:econometrics|0|0 - extended|global_mmlu_instruct_cs_swa:econometrics|0|0 - extended|global_mmlu_instruct_unk_swa:econometrics|0|0 - extended|global_mmlu_instruct_all_tel:econometrics|0|0 - extended|global_mmlu_instruct_ca_tel:econometrics|0|0 - extended|global_mmlu_instruct_cs_tel:econometrics|0|0 - extended|global_mmlu_instruct_unk_tel:econometrics|0|0 - extended|global_mmlu_instruct_all_tha:econometrics|0|0 - extended|global_mmlu_instruct_ca_tha:econometrics|0|0 - extended|global_mmlu_instruct_cs_tha:econometrics|0|0 - extended|global_mmlu_instruct_unk_tha:econometrics|0|0 - extended|global_mmlu_instruct_all_tur:econometrics|0|0 - extended|global_mmlu_instruct_ca_tur:econometrics|0|0 - extended|global_mmlu_instruct_cs_tur:econometrics|0|0 - extended|global_mmlu_instruct_unk_tur:econometrics|0|0 - extended|global_mmlu_instruct_all_ukr:econometrics|0|0 - extended|global_mmlu_instruct_ca_ukr:econometrics|0|0 - extended|global_mmlu_instruct_cs_ukr:econometrics|0|0 - extended|global_mmlu_instruct_unk_ukr:econometrics|0|0 - extended|global_mmlu_instruct_all_urd:econometrics|0|0 - extended|global_mmlu_instruct_ca_urd:econometrics|0|0 - extended|global_mmlu_instruct_cs_urd:econometrics|0|0 - extended|global_mmlu_instruct_unk_urd:econometrics|0|0 - extended|global_mmlu_instruct_all_vie:econometrics|0|0 - extended|global_mmlu_instruct_ca_vie:econometrics|0|0 - extended|global_mmlu_instruct_cs_vie:econometrics|0|0 - extended|global_mmlu_instruct_unk_vie:econometrics|0|0 - extended|global_mmlu_instruct_all_ara:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_ara:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_ara:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_ara:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_ben:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_ben:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_ben:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_ben:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_deu:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_deu:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_deu:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_deu:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_spa:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_spa:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_spa:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_spa:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_eng:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_eng:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_eng:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_eng:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_fra:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_fra:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_fra:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_fra:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_hin:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_hin:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_hin:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_hin:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_ind:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_ind:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_ind:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_ind:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_ita:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_ita:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_ita:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_ita:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_jpn:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_jpn:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_jpn:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_jpn:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_nld:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_nld:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_nld:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_nld:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_nor:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_nor:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_nor:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_nor:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_pol:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_pol:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_pol:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_pol:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_por:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_por:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_por:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_por:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_rus:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_rus:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_rus:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_rus:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_srp:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_srp:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_srp:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_srp:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_swe:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_swe:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_swe:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_swe:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_swa:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_swa:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_swa:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_swa:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_tel:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_tel:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_tel:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_tel:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_tha:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_tha:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_tha:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_tha:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_tur:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_tur:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_tur:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_tur:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_ukr:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_ukr:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_ukr:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_ukr:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_urd:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_urd:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_urd:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_urd:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_vie:electrical_engineering|0|0 - extended|global_mmlu_instruct_ca_vie:electrical_engineering|0|0 - extended|global_mmlu_instruct_cs_vie:electrical_engineering|0|0 - extended|global_mmlu_instruct_unk_vie:electrical_engineering|0|0 - extended|global_mmlu_instruct_all_ara:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_ara:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_ara:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_ara:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_ben:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_ben:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_ben:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_ben:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_deu:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_deu:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_deu:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_deu:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_spa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_spa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_spa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_spa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_eng:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_eng:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_eng:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_eng:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_fra:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_fra:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_fra:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_fra:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_hin:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_hin:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_hin:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_hin:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_ind:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_ind:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_ind:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_ind:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_ita:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_ita:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_ita:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_ita:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_jpn:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_jpn:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_jpn:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_jpn:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_nld:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_nld:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_nld:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_nld:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_nor:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_nor:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_nor:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_nor:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_pol:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_pol:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_pol:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_pol:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_por:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_por:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_por:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_por:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_rus:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_rus:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_rus:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_rus:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_srp:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_srp:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_srp:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_srp:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_swe:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_swe:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_swe:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_swe:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_swa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_swa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_swa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_swa:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_tel:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_tel:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_tel:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_tel:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_tha:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_tha:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_tha:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_tha:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_tur:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_tur:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_tur:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_tur:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_ukr:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_ukr:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_ukr:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_ukr:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_urd:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_urd:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_urd:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_urd:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_vie:elementary_mathematics|0|0 - extended|global_mmlu_instruct_ca_vie:elementary_mathematics|0|0 - extended|global_mmlu_instruct_cs_vie:elementary_mathematics|0|0 - extended|global_mmlu_instruct_unk_vie:elementary_mathematics|0|0 - extended|global_mmlu_instruct_all_ara:formal_logic|0|0 - extended|global_mmlu_instruct_ca_ara:formal_logic|0|0 - extended|global_mmlu_instruct_cs_ara:formal_logic|0|0 - extended|global_mmlu_instruct_unk_ara:formal_logic|0|0 - extended|global_mmlu_instruct_all_ben:formal_logic|0|0 - extended|global_mmlu_instruct_ca_ben:formal_logic|0|0 - extended|global_mmlu_instruct_cs_ben:formal_logic|0|0 - extended|global_mmlu_instruct_unk_ben:formal_logic|0|0 - extended|global_mmlu_instruct_all_deu:formal_logic|0|0 - extended|global_mmlu_instruct_ca_deu:formal_logic|0|0 - extended|global_mmlu_instruct_cs_deu:formal_logic|0|0 - extended|global_mmlu_instruct_unk_deu:formal_logic|0|0 - extended|global_mmlu_instruct_all_spa:formal_logic|0|0 - extended|global_mmlu_instruct_ca_spa:formal_logic|0|0 - extended|global_mmlu_instruct_cs_spa:formal_logic|0|0 - extended|global_mmlu_instruct_unk_spa:formal_logic|0|0 - extended|global_mmlu_instruct_all_eng:formal_logic|0|0 - extended|global_mmlu_instruct_ca_eng:formal_logic|0|0 - extended|global_mmlu_instruct_cs_eng:formal_logic|0|0 - extended|global_mmlu_instruct_unk_eng:formal_logic|0|0 - extended|global_mmlu_instruct_all_fra:formal_logic|0|0 - extended|global_mmlu_instruct_ca_fra:formal_logic|0|0 - extended|global_mmlu_instruct_cs_fra:formal_logic|0|0 - extended|global_mmlu_instruct_unk_fra:formal_logic|0|0 - extended|global_mmlu_instruct_all_hin:formal_logic|0|0 - extended|global_mmlu_instruct_ca_hin:formal_logic|0|0 - extended|global_mmlu_instruct_cs_hin:formal_logic|0|0 - extended|global_mmlu_instruct_unk_hin:formal_logic|0|0 - extended|global_mmlu_instruct_all_ind:formal_logic|0|0 - extended|global_mmlu_instruct_ca_ind:formal_logic|0|0 - extended|global_mmlu_instruct_cs_ind:formal_logic|0|0 - extended|global_mmlu_instruct_unk_ind:formal_logic|0|0 - extended|global_mmlu_instruct_all_ita:formal_logic|0|0 - extended|global_mmlu_instruct_ca_ita:formal_logic|0|0 - extended|global_mmlu_instruct_cs_ita:formal_logic|0|0 - extended|global_mmlu_instruct_unk_ita:formal_logic|0|0 - extended|global_mmlu_instruct_all_jpn:formal_logic|0|0 - extended|global_mmlu_instruct_ca_jpn:formal_logic|0|0 - extended|global_mmlu_instruct_cs_jpn:formal_logic|0|0 - extended|global_mmlu_instruct_unk_jpn:formal_logic|0|0 - extended|global_mmlu_instruct_all_nld:formal_logic|0|0 - extended|global_mmlu_instruct_ca_nld:formal_logic|0|0 - extended|global_mmlu_instruct_cs_nld:formal_logic|0|0 - extended|global_mmlu_instruct_unk_nld:formal_logic|0|0 - extended|global_mmlu_instruct_all_nor:formal_logic|0|0 - extended|global_mmlu_instruct_ca_nor:formal_logic|0|0 - extended|global_mmlu_instruct_cs_nor:formal_logic|0|0 - extended|global_mmlu_instruct_unk_nor:formal_logic|0|0 - extended|global_mmlu_instruct_all_pol:formal_logic|0|0 - extended|global_mmlu_instruct_ca_pol:formal_logic|0|0 - extended|global_mmlu_instruct_cs_pol:formal_logic|0|0 - extended|global_mmlu_instruct_unk_pol:formal_logic|0|0 - extended|global_mmlu_instruct_all_por:formal_logic|0|0 - extended|global_mmlu_instruct_ca_por:formal_logic|0|0 - extended|global_mmlu_instruct_cs_por:formal_logic|0|0 - extended|global_mmlu_instruct_unk_por:formal_logic|0|0 - extended|global_mmlu_instruct_all_rus:formal_logic|0|0 - extended|global_mmlu_instruct_ca_rus:formal_logic|0|0 - extended|global_mmlu_instruct_cs_rus:formal_logic|0|0 - extended|global_mmlu_instruct_unk_rus:formal_logic|0|0 - extended|global_mmlu_instruct_all_srp:formal_logic|0|0 - extended|global_mmlu_instruct_ca_srp:formal_logic|0|0 - extended|global_mmlu_instruct_cs_srp:formal_logic|0|0 - extended|global_mmlu_instruct_unk_srp:formal_logic|0|0 - extended|global_mmlu_instruct_all_swe:formal_logic|0|0 - extended|global_mmlu_instruct_ca_swe:formal_logic|0|0 - extended|global_mmlu_instruct_cs_swe:formal_logic|0|0 - extended|global_mmlu_instruct_unk_swe:formal_logic|0|0 - extended|global_mmlu_instruct_all_swa:formal_logic|0|0 - extended|global_mmlu_instruct_ca_swa:formal_logic|0|0 - extended|global_mmlu_instruct_cs_swa:formal_logic|0|0 - extended|global_mmlu_instruct_unk_swa:formal_logic|0|0 - extended|global_mmlu_instruct_all_tel:formal_logic|0|0 - extended|global_mmlu_instruct_ca_tel:formal_logic|0|0 - extended|global_mmlu_instruct_cs_tel:formal_logic|0|0 - extended|global_mmlu_instruct_unk_tel:formal_logic|0|0 - extended|global_mmlu_instruct_all_tha:formal_logic|0|0 - extended|global_mmlu_instruct_ca_tha:formal_logic|0|0 - extended|global_mmlu_instruct_cs_tha:formal_logic|0|0 - extended|global_mmlu_instruct_unk_tha:formal_logic|0|0 - extended|global_mmlu_instruct_all_tur:formal_logic|0|0 - extended|global_mmlu_instruct_ca_tur:formal_logic|0|0 - extended|global_mmlu_instruct_cs_tur:formal_logic|0|0 - extended|global_mmlu_instruct_unk_tur:formal_logic|0|0 - extended|global_mmlu_instruct_all_ukr:formal_logic|0|0 - extended|global_mmlu_instruct_ca_ukr:formal_logic|0|0 - extended|global_mmlu_instruct_cs_ukr:formal_logic|0|0 - extended|global_mmlu_instruct_unk_ukr:formal_logic|0|0 - extended|global_mmlu_instruct_all_urd:formal_logic|0|0 - extended|global_mmlu_instruct_ca_urd:formal_logic|0|0 - extended|global_mmlu_instruct_cs_urd:formal_logic|0|0 - extended|global_mmlu_instruct_unk_urd:formal_logic|0|0 - extended|global_mmlu_instruct_all_vie:formal_logic|0|0 - extended|global_mmlu_instruct_ca_vie:formal_logic|0|0 - extended|global_mmlu_instruct_cs_vie:formal_logic|0|0 - extended|global_mmlu_instruct_unk_vie:formal_logic|0|0 - extended|global_mmlu_instruct_all_ara:global_facts|0|0 - extended|global_mmlu_instruct_ca_ara:global_facts|0|0 - extended|global_mmlu_instruct_cs_ara:global_facts|0|0 - extended|global_mmlu_instruct_unk_ara:global_facts|0|0 - extended|global_mmlu_instruct_all_ben:global_facts|0|0 - extended|global_mmlu_instruct_ca_ben:global_facts|0|0 - extended|global_mmlu_instruct_cs_ben:global_facts|0|0 - extended|global_mmlu_instruct_unk_ben:global_facts|0|0 - extended|global_mmlu_instruct_all_deu:global_facts|0|0 - extended|global_mmlu_instruct_ca_deu:global_facts|0|0 - extended|global_mmlu_instruct_cs_deu:global_facts|0|0 - extended|global_mmlu_instruct_unk_deu:global_facts|0|0 - extended|global_mmlu_instruct_all_spa:global_facts|0|0 - extended|global_mmlu_instruct_ca_spa:global_facts|0|0 - extended|global_mmlu_instruct_cs_spa:global_facts|0|0 - extended|global_mmlu_instruct_unk_spa:global_facts|0|0 - extended|global_mmlu_instruct_all_eng:global_facts|0|0 - extended|global_mmlu_instruct_ca_eng:global_facts|0|0 - extended|global_mmlu_instruct_cs_eng:global_facts|0|0 - extended|global_mmlu_instruct_unk_eng:global_facts|0|0 - extended|global_mmlu_instruct_all_fra:global_facts|0|0 - extended|global_mmlu_instruct_ca_fra:global_facts|0|0 - extended|global_mmlu_instruct_cs_fra:global_facts|0|0 - extended|global_mmlu_instruct_unk_fra:global_facts|0|0 - extended|global_mmlu_instruct_all_hin:global_facts|0|0 - extended|global_mmlu_instruct_ca_hin:global_facts|0|0 - extended|global_mmlu_instruct_cs_hin:global_facts|0|0 - extended|global_mmlu_instruct_unk_hin:global_facts|0|0 - extended|global_mmlu_instruct_all_ind:global_facts|0|0 - extended|global_mmlu_instruct_ca_ind:global_facts|0|0 - extended|global_mmlu_instruct_cs_ind:global_facts|0|0 - extended|global_mmlu_instruct_unk_ind:global_facts|0|0 - extended|global_mmlu_instruct_all_ita:global_facts|0|0 - extended|global_mmlu_instruct_ca_ita:global_facts|0|0 - extended|global_mmlu_instruct_cs_ita:global_facts|0|0 - extended|global_mmlu_instruct_unk_ita:global_facts|0|0 - extended|global_mmlu_instruct_all_jpn:global_facts|0|0 - extended|global_mmlu_instruct_ca_jpn:global_facts|0|0 - extended|global_mmlu_instruct_cs_jpn:global_facts|0|0 - extended|global_mmlu_instruct_unk_jpn:global_facts|0|0 - extended|global_mmlu_instruct_all_nld:global_facts|0|0 - extended|global_mmlu_instruct_ca_nld:global_facts|0|0 - extended|global_mmlu_instruct_cs_nld:global_facts|0|0 - extended|global_mmlu_instruct_unk_nld:global_facts|0|0 - extended|global_mmlu_instruct_all_nor:global_facts|0|0 - extended|global_mmlu_instruct_ca_nor:global_facts|0|0 - extended|global_mmlu_instruct_cs_nor:global_facts|0|0 - extended|global_mmlu_instruct_unk_nor:global_facts|0|0 - extended|global_mmlu_instruct_all_pol:global_facts|0|0 - extended|global_mmlu_instruct_ca_pol:global_facts|0|0 - extended|global_mmlu_instruct_cs_pol:global_facts|0|0 - extended|global_mmlu_instruct_unk_pol:global_facts|0|0 - extended|global_mmlu_instruct_all_por:global_facts|0|0 - extended|global_mmlu_instruct_ca_por:global_facts|0|0 - extended|global_mmlu_instruct_cs_por:global_facts|0|0 - extended|global_mmlu_instruct_unk_por:global_facts|0|0 - extended|global_mmlu_instruct_all_rus:global_facts|0|0 - extended|global_mmlu_instruct_ca_rus:global_facts|0|0 - extended|global_mmlu_instruct_cs_rus:global_facts|0|0 - extended|global_mmlu_instruct_unk_rus:global_facts|0|0 - extended|global_mmlu_instruct_all_srp:global_facts|0|0 - extended|global_mmlu_instruct_ca_srp:global_facts|0|0 - extended|global_mmlu_instruct_cs_srp:global_facts|0|0 - extended|global_mmlu_instruct_unk_srp:global_facts|0|0 - extended|global_mmlu_instruct_all_swe:global_facts|0|0 - extended|global_mmlu_instruct_ca_swe:global_facts|0|0 - extended|global_mmlu_instruct_cs_swe:global_facts|0|0 - extended|global_mmlu_instruct_unk_swe:global_facts|0|0 - extended|global_mmlu_instruct_all_swa:global_facts|0|0 - extended|global_mmlu_instruct_ca_swa:global_facts|0|0 - extended|global_mmlu_instruct_cs_swa:global_facts|0|0 - extended|global_mmlu_instruct_unk_swa:global_facts|0|0 - extended|global_mmlu_instruct_all_tel:global_facts|0|0 - extended|global_mmlu_instruct_ca_tel:global_facts|0|0 - extended|global_mmlu_instruct_cs_tel:global_facts|0|0 - extended|global_mmlu_instruct_unk_tel:global_facts|0|0 - extended|global_mmlu_instruct_all_tha:global_facts|0|0 - extended|global_mmlu_instruct_ca_tha:global_facts|0|0 - extended|global_mmlu_instruct_cs_tha:global_facts|0|0 - extended|global_mmlu_instruct_unk_tha:global_facts|0|0 - extended|global_mmlu_instruct_all_tur:global_facts|0|0 - extended|global_mmlu_instruct_ca_tur:global_facts|0|0 - extended|global_mmlu_instruct_cs_tur:global_facts|0|0 - extended|global_mmlu_instruct_unk_tur:global_facts|0|0 - extended|global_mmlu_instruct_all_ukr:global_facts|0|0 - extended|global_mmlu_instruct_ca_ukr:global_facts|0|0 - extended|global_mmlu_instruct_cs_ukr:global_facts|0|0 - extended|global_mmlu_instruct_unk_ukr:global_facts|0|0 - extended|global_mmlu_instruct_all_urd:global_facts|0|0 - extended|global_mmlu_instruct_ca_urd:global_facts|0|0 - extended|global_mmlu_instruct_cs_urd:global_facts|0|0 - extended|global_mmlu_instruct_unk_urd:global_facts|0|0 - extended|global_mmlu_instruct_all_vie:global_facts|0|0 - extended|global_mmlu_instruct_ca_vie:global_facts|0|0 - extended|global_mmlu_instruct_cs_vie:global_facts|0|0 - extended|global_mmlu_instruct_unk_vie:global_facts|0|0 - extended|global_mmlu_instruct_all_ara:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_biology|0|0 - extended|global_mmlu_instruct_all_ben:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_biology|0|0 - extended|global_mmlu_instruct_all_deu:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_biology|0|0 - extended|global_mmlu_instruct_all_spa:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_biology|0|0 - extended|global_mmlu_instruct_all_eng:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_biology|0|0 - extended|global_mmlu_instruct_all_fra:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_biology|0|0 - extended|global_mmlu_instruct_all_hin:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_biology|0|0 - extended|global_mmlu_instruct_all_ind:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_biology|0|0 - extended|global_mmlu_instruct_all_ita:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_biology|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_biology|0|0 - extended|global_mmlu_instruct_all_nld:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_biology|0|0 - extended|global_mmlu_instruct_all_nor:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_biology|0|0 - extended|global_mmlu_instruct_all_pol:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_biology|0|0 - extended|global_mmlu_instruct_all_por:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_por:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_por:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_por:high_school_biology|0|0 - extended|global_mmlu_instruct_all_rus:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_biology|0|0 - extended|global_mmlu_instruct_all_srp:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_biology|0|0 - extended|global_mmlu_instruct_all_swe:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_biology|0|0 - extended|global_mmlu_instruct_all_swa:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_biology|0|0 - extended|global_mmlu_instruct_all_tel:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_biology|0|0 - extended|global_mmlu_instruct_all_tha:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_biology|0|0 - extended|global_mmlu_instruct_all_tur:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_biology|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_biology|0|0 - extended|global_mmlu_instruct_all_urd:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_biology|0|0 - extended|global_mmlu_instruct_all_vie:high_school_biology|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_biology|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_biology|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_biology|0|0 - extended|global_mmlu_instruct_all_ara:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_ben:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_deu:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_spa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_eng:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_fra:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_hin:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_ind:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_ita:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_nld:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_nor:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_pol:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_por:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_por:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_por:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_por:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_rus:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_srp:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_swe:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_swa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_tel:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_tha:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_tur:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_urd:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_vie:high_school_chemistry|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_chemistry|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_chemistry|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_chemistry|0|0 - extended|global_mmlu_instruct_all_ara:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_ben:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_deu:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_spa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_eng:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_fra:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_hin:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_ind:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_ita:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_nld:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_nor:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_pol:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_por:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_por:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_por:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_por:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_rus:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_srp:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_swe:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_swa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_tel:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_tha:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_tur:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_urd:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_vie:high_school_computer_science|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_computer_science|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_computer_science|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_computer_science|0|0 - extended|global_mmlu_instruct_all_ara:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_ben:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_deu:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_spa:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_eng:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_fra:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_hin:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_ind:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_ita:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_nld:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_nor:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_pol:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_por:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_por:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_por:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_por:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_rus:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_srp:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_swe:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_swa:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_tel:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_tha:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_tur:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_urd:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_vie:high_school_european_history|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_european_history|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_european_history|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_european_history|0|0 - extended|global_mmlu_instruct_all_ara:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_geography|0|0 - extended|global_mmlu_instruct_all_ben:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_geography|0|0 - extended|global_mmlu_instruct_all_deu:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_geography|0|0 - extended|global_mmlu_instruct_all_spa:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_geography|0|0 - extended|global_mmlu_instruct_all_eng:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_geography|0|0 - extended|global_mmlu_instruct_all_fra:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_geography|0|0 - extended|global_mmlu_instruct_all_hin:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_geography|0|0 - extended|global_mmlu_instruct_all_ind:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_geography|0|0 - extended|global_mmlu_instruct_all_ita:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_geography|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_geography|0|0 - extended|global_mmlu_instruct_all_nld:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_geography|0|0 - extended|global_mmlu_instruct_all_nor:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_geography|0|0 - extended|global_mmlu_instruct_all_pol:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_geography|0|0 - extended|global_mmlu_instruct_all_por:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_por:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_por:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_por:high_school_geography|0|0 - extended|global_mmlu_instruct_all_rus:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_geography|0|0 - extended|global_mmlu_instruct_all_srp:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_geography|0|0 - extended|global_mmlu_instruct_all_swe:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_geography|0|0 - extended|global_mmlu_instruct_all_swa:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_geography|0|0 - extended|global_mmlu_instruct_all_tel:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_geography|0|0 - extended|global_mmlu_instruct_all_tha:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_geography|0|0 - extended|global_mmlu_instruct_all_tur:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_geography|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_geography|0|0 - extended|global_mmlu_instruct_all_urd:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_geography|0|0 - extended|global_mmlu_instruct_all_vie:high_school_geography|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_geography|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_geography|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_geography|0|0 - extended|global_mmlu_instruct_all_ara:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_ben:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_deu:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_spa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_eng:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_fra:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_hin:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_ind:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_ita:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_nld:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_nor:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_pol:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_por:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_por:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_por:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_por:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_rus:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_srp:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_swe:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_swa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_tel:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_tha:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_tur:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_urd:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_vie:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_government_and_politics|0|0 - extended|global_mmlu_instruct_all_ara:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_ben:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_deu:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_spa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_eng:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_fra:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_hin:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_ind:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_ita:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_nld:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_nor:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_pol:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_por:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_por:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_por:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_por:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_rus:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_srp:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_swe:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_swa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_tel:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_tha:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_tur:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_urd:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_vie:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_macroeconomics|0|0 - extended|global_mmlu_instruct_all_ara:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_ben:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_deu:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_spa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_eng:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_fra:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_hin:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_ind:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_ita:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_nld:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_nor:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_pol:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_por:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_por:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_por:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_por:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_rus:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_srp:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_swe:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_swa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_tel:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_tha:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_tur:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_urd:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_vie:high_school_mathematics|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_mathematics|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_mathematics|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_mathematics|0|0 - extended|global_mmlu_instruct_all_ara:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_ben:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_deu:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_spa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_eng:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_fra:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_hin:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_ind:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_ita:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_nld:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_nor:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_pol:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_por:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_por:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_por:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_por:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_rus:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_srp:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_swe:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_swa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_tel:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_tha:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_tur:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_urd:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_vie:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_microeconomics|0|0 - extended|global_mmlu_instruct_all_ara:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_physics|0|0 - extended|global_mmlu_instruct_all_ben:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_physics|0|0 - extended|global_mmlu_instruct_all_deu:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_physics|0|0 - extended|global_mmlu_instruct_all_spa:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_physics|0|0 - extended|global_mmlu_instruct_all_eng:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_physics|0|0 - extended|global_mmlu_instruct_all_fra:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_physics|0|0 - extended|global_mmlu_instruct_all_hin:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_physics|0|0 - extended|global_mmlu_instruct_all_ind:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_physics|0|0 - extended|global_mmlu_instruct_all_ita:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_physics|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_physics|0|0 - extended|global_mmlu_instruct_all_nld:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_physics|0|0 - extended|global_mmlu_instruct_all_nor:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_physics|0|0 - extended|global_mmlu_instruct_all_pol:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_physics|0|0 - extended|global_mmlu_instruct_all_por:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_por:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_por:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_por:high_school_physics|0|0 - extended|global_mmlu_instruct_all_rus:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_physics|0|0 - extended|global_mmlu_instruct_all_srp:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_physics|0|0 - extended|global_mmlu_instruct_all_swe:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_physics|0|0 - extended|global_mmlu_instruct_all_swa:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_physics|0|0 - extended|global_mmlu_instruct_all_tel:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_physics|0|0 - extended|global_mmlu_instruct_all_tha:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_physics|0|0 - extended|global_mmlu_instruct_all_tur:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_physics|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_physics|0|0 - extended|global_mmlu_instruct_all_urd:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_physics|0|0 - extended|global_mmlu_instruct_all_vie:high_school_physics|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_physics|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_physics|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_physics|0|0 - extended|global_mmlu_instruct_all_ara:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_ben:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_deu:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_spa:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_eng:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_fra:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_hin:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_ind:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_ita:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_nld:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_nor:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_pol:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_por:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_por:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_por:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_por:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_rus:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_srp:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_swe:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_swa:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_tel:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_tha:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_tur:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_urd:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_vie:high_school_psychology|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_psychology|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_psychology|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_psychology|0|0 - extended|global_mmlu_instruct_all_ara:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_ben:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_deu:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_spa:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_eng:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_fra:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_hin:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_ind:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_ita:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_nld:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_nor:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_pol:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_por:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_por:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_por:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_por:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_rus:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_srp:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_swe:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_swa:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_tel:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_tha:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_tur:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_urd:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_vie:high_school_statistics|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_statistics|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_statistics|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_statistics|0|0 - extended|global_mmlu_instruct_all_ara:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_ben:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_deu:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_spa:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_eng:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_fra:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_hin:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_ind:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_ita:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_nld:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_nor:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_pol:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_por:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_por:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_por:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_por:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_rus:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_srp:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_swe:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_swa:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_tel:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_tha:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_tur:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_urd:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_vie:high_school_us_history|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_us_history|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_us_history|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_us_history|0|0 - extended|global_mmlu_instruct_all_ara:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_ara:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_ara:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_ara:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_ben:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_ben:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_ben:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_ben:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_deu:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_deu:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_deu:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_deu:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_spa:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_spa:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_spa:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_spa:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_eng:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_eng:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_eng:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_eng:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_fra:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_fra:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_fra:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_fra:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_hin:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_hin:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_hin:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_hin:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_ind:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_ind:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_ind:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_ind:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_ita:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_ita:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_ita:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_ita:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_jpn:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_jpn:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_jpn:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_jpn:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_nld:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_nld:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_nld:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_nld:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_nor:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_nor:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_nor:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_nor:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_pol:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_pol:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_pol:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_pol:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_por:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_por:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_por:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_por:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_rus:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_rus:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_rus:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_rus:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_srp:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_srp:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_srp:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_srp:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_swe:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_swe:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_swe:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_swe:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_swa:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_swa:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_swa:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_swa:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_tel:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_tel:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_tel:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_tel:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_tha:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_tha:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_tha:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_tha:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_tur:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_tur:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_tur:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_tur:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_ukr:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_ukr:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_ukr:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_ukr:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_urd:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_urd:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_urd:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_urd:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_vie:high_school_world_history|0|0 - extended|global_mmlu_instruct_ca_vie:high_school_world_history|0|0 - extended|global_mmlu_instruct_cs_vie:high_school_world_history|0|0 - extended|global_mmlu_instruct_unk_vie:high_school_world_history|0|0 - extended|global_mmlu_instruct_all_ara:human_aging|0|0 - extended|global_mmlu_instruct_ca_ara:human_aging|0|0 - extended|global_mmlu_instruct_cs_ara:human_aging|0|0 - extended|global_mmlu_instruct_unk_ara:human_aging|0|0 - extended|global_mmlu_instruct_all_ben:human_aging|0|0 - extended|global_mmlu_instruct_ca_ben:human_aging|0|0 - extended|global_mmlu_instruct_cs_ben:human_aging|0|0 - extended|global_mmlu_instruct_unk_ben:human_aging|0|0 - extended|global_mmlu_instruct_all_deu:human_aging|0|0 - extended|global_mmlu_instruct_ca_deu:human_aging|0|0 - extended|global_mmlu_instruct_cs_deu:human_aging|0|0 - extended|global_mmlu_instruct_unk_deu:human_aging|0|0 - extended|global_mmlu_instruct_all_spa:human_aging|0|0 - extended|global_mmlu_instruct_ca_spa:human_aging|0|0 - extended|global_mmlu_instruct_cs_spa:human_aging|0|0 - extended|global_mmlu_instruct_unk_spa:human_aging|0|0 - extended|global_mmlu_instruct_all_eng:human_aging|0|0 - extended|global_mmlu_instruct_ca_eng:human_aging|0|0 - extended|global_mmlu_instruct_cs_eng:human_aging|0|0 - extended|global_mmlu_instruct_unk_eng:human_aging|0|0 - extended|global_mmlu_instruct_all_fra:human_aging|0|0 - extended|global_mmlu_instruct_ca_fra:human_aging|0|0 - extended|global_mmlu_instruct_cs_fra:human_aging|0|0 - extended|global_mmlu_instruct_unk_fra:human_aging|0|0 - extended|global_mmlu_instruct_all_hin:human_aging|0|0 - extended|global_mmlu_instruct_ca_hin:human_aging|0|0 - extended|global_mmlu_instruct_cs_hin:human_aging|0|0 - extended|global_mmlu_instruct_unk_hin:human_aging|0|0 - extended|global_mmlu_instruct_all_ind:human_aging|0|0 - extended|global_mmlu_instruct_ca_ind:human_aging|0|0 - extended|global_mmlu_instruct_cs_ind:human_aging|0|0 - extended|global_mmlu_instruct_unk_ind:human_aging|0|0 - extended|global_mmlu_instruct_all_ita:human_aging|0|0 - extended|global_mmlu_instruct_ca_ita:human_aging|0|0 - extended|global_mmlu_instruct_cs_ita:human_aging|0|0 - extended|global_mmlu_instruct_unk_ita:human_aging|0|0 - extended|global_mmlu_instruct_all_jpn:human_aging|0|0 - extended|global_mmlu_instruct_ca_jpn:human_aging|0|0 - extended|global_mmlu_instruct_cs_jpn:human_aging|0|0 - extended|global_mmlu_instruct_unk_jpn:human_aging|0|0 - extended|global_mmlu_instruct_all_nld:human_aging|0|0 - extended|global_mmlu_instruct_ca_nld:human_aging|0|0 - extended|global_mmlu_instruct_cs_nld:human_aging|0|0 - extended|global_mmlu_instruct_unk_nld:human_aging|0|0 - extended|global_mmlu_instruct_all_nor:human_aging|0|0 - extended|global_mmlu_instruct_ca_nor:human_aging|0|0 - extended|global_mmlu_instruct_cs_nor:human_aging|0|0 - extended|global_mmlu_instruct_unk_nor:human_aging|0|0 - extended|global_mmlu_instruct_all_pol:human_aging|0|0 - extended|global_mmlu_instruct_ca_pol:human_aging|0|0 - extended|global_mmlu_instruct_cs_pol:human_aging|0|0 - extended|global_mmlu_instruct_unk_pol:human_aging|0|0 - extended|global_mmlu_instruct_all_por:human_aging|0|0 - extended|global_mmlu_instruct_ca_por:human_aging|0|0 - extended|global_mmlu_instruct_cs_por:human_aging|0|0 - extended|global_mmlu_instruct_unk_por:human_aging|0|0 - extended|global_mmlu_instruct_all_rus:human_aging|0|0 - extended|global_mmlu_instruct_ca_rus:human_aging|0|0 - extended|global_mmlu_instruct_cs_rus:human_aging|0|0 - extended|global_mmlu_instruct_unk_rus:human_aging|0|0 - extended|global_mmlu_instruct_all_srp:human_aging|0|0 - extended|global_mmlu_instruct_ca_srp:human_aging|0|0 - extended|global_mmlu_instruct_cs_srp:human_aging|0|0 - extended|global_mmlu_instruct_unk_srp:human_aging|0|0 - extended|global_mmlu_instruct_all_swe:human_aging|0|0 - extended|global_mmlu_instruct_ca_swe:human_aging|0|0 - extended|global_mmlu_instruct_cs_swe:human_aging|0|0 - extended|global_mmlu_instruct_unk_swe:human_aging|0|0 - extended|global_mmlu_instruct_all_swa:human_aging|0|0 - extended|global_mmlu_instruct_ca_swa:human_aging|0|0 - extended|global_mmlu_instruct_cs_swa:human_aging|0|0 - extended|global_mmlu_instruct_unk_swa:human_aging|0|0 - extended|global_mmlu_instruct_all_tel:human_aging|0|0 - extended|global_mmlu_instruct_ca_tel:human_aging|0|0 - extended|global_mmlu_instruct_cs_tel:human_aging|0|0 - extended|global_mmlu_instruct_unk_tel:human_aging|0|0 - extended|global_mmlu_instruct_all_tha:human_aging|0|0 - extended|global_mmlu_instruct_ca_tha:human_aging|0|0 - extended|global_mmlu_instruct_cs_tha:human_aging|0|0 - extended|global_mmlu_instruct_unk_tha:human_aging|0|0 - extended|global_mmlu_instruct_all_tur:human_aging|0|0 - extended|global_mmlu_instruct_ca_tur:human_aging|0|0 - extended|global_mmlu_instruct_cs_tur:human_aging|0|0 - extended|global_mmlu_instruct_unk_tur:human_aging|0|0 - extended|global_mmlu_instruct_all_ukr:human_aging|0|0 - extended|global_mmlu_instruct_ca_ukr:human_aging|0|0 - extended|global_mmlu_instruct_cs_ukr:human_aging|0|0 - extended|global_mmlu_instruct_unk_ukr:human_aging|0|0 - extended|global_mmlu_instruct_all_urd:human_aging|0|0 - extended|global_mmlu_instruct_ca_urd:human_aging|0|0 - extended|global_mmlu_instruct_cs_urd:human_aging|0|0 - extended|global_mmlu_instruct_unk_urd:human_aging|0|0 - extended|global_mmlu_instruct_all_vie:human_aging|0|0 - extended|global_mmlu_instruct_ca_vie:human_aging|0|0 - extended|global_mmlu_instruct_cs_vie:human_aging|0|0 - extended|global_mmlu_instruct_unk_vie:human_aging|0|0 - extended|global_mmlu_instruct_all_ara:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_ara:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_ara:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_ara:human_sexuality|0|0 - extended|global_mmlu_instruct_all_ben:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_ben:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_ben:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_ben:human_sexuality|0|0 - extended|global_mmlu_instruct_all_deu:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_deu:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_deu:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_deu:human_sexuality|0|0 - extended|global_mmlu_instruct_all_spa:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_spa:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_spa:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_spa:human_sexuality|0|0 - extended|global_mmlu_instruct_all_eng:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_eng:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_eng:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_eng:human_sexuality|0|0 - extended|global_mmlu_instruct_all_fra:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_fra:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_fra:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_fra:human_sexuality|0|0 - extended|global_mmlu_instruct_all_hin:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_hin:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_hin:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_hin:human_sexuality|0|0 - extended|global_mmlu_instruct_all_ind:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_ind:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_ind:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_ind:human_sexuality|0|0 - extended|global_mmlu_instruct_all_ita:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_ita:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_ita:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_ita:human_sexuality|0|0 - extended|global_mmlu_instruct_all_jpn:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_jpn:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_jpn:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_jpn:human_sexuality|0|0 - extended|global_mmlu_instruct_all_nld:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_nld:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_nld:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_nld:human_sexuality|0|0 - extended|global_mmlu_instruct_all_nor:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_nor:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_nor:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_nor:human_sexuality|0|0 - extended|global_mmlu_instruct_all_pol:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_pol:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_pol:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_pol:human_sexuality|0|0 - extended|global_mmlu_instruct_all_por:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_por:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_por:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_por:human_sexuality|0|0 - extended|global_mmlu_instruct_all_rus:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_rus:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_rus:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_rus:human_sexuality|0|0 - extended|global_mmlu_instruct_all_srp:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_srp:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_srp:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_srp:human_sexuality|0|0 - extended|global_mmlu_instruct_all_swe:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_swe:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_swe:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_swe:human_sexuality|0|0 - extended|global_mmlu_instruct_all_swa:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_swa:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_swa:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_swa:human_sexuality|0|0 - extended|global_mmlu_instruct_all_tel:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_tel:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_tel:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_tel:human_sexuality|0|0 - extended|global_mmlu_instruct_all_tha:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_tha:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_tha:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_tha:human_sexuality|0|0 - extended|global_mmlu_instruct_all_tur:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_tur:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_tur:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_tur:human_sexuality|0|0 - extended|global_mmlu_instruct_all_ukr:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_ukr:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_ukr:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_ukr:human_sexuality|0|0 - extended|global_mmlu_instruct_all_urd:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_urd:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_urd:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_urd:human_sexuality|0|0 - extended|global_mmlu_instruct_all_vie:human_sexuality|0|0 - extended|global_mmlu_instruct_ca_vie:human_sexuality|0|0 - extended|global_mmlu_instruct_cs_vie:human_sexuality|0|0 - extended|global_mmlu_instruct_unk_vie:human_sexuality|0|0 - extended|global_mmlu_instruct_all_ara:international_law|0|0 - extended|global_mmlu_instruct_ca_ara:international_law|0|0 - extended|global_mmlu_instruct_cs_ara:international_law|0|0 - extended|global_mmlu_instruct_unk_ara:international_law|0|0 - extended|global_mmlu_instruct_all_ben:international_law|0|0 - extended|global_mmlu_instruct_ca_ben:international_law|0|0 - extended|global_mmlu_instruct_cs_ben:international_law|0|0 - extended|global_mmlu_instruct_unk_ben:international_law|0|0 - extended|global_mmlu_instruct_all_deu:international_law|0|0 - extended|global_mmlu_instruct_ca_deu:international_law|0|0 - extended|global_mmlu_instruct_cs_deu:international_law|0|0 - extended|global_mmlu_instruct_unk_deu:international_law|0|0 - extended|global_mmlu_instruct_all_spa:international_law|0|0 - extended|global_mmlu_instruct_ca_spa:international_law|0|0 - extended|global_mmlu_instruct_cs_spa:international_law|0|0 - extended|global_mmlu_instruct_unk_spa:international_law|0|0 - extended|global_mmlu_instruct_all_eng:international_law|0|0 - extended|global_mmlu_instruct_ca_eng:international_law|0|0 - extended|global_mmlu_instruct_cs_eng:international_law|0|0 - extended|global_mmlu_instruct_unk_eng:international_law|0|0 - extended|global_mmlu_instruct_all_fra:international_law|0|0 - extended|global_mmlu_instruct_ca_fra:international_law|0|0 - extended|global_mmlu_instruct_cs_fra:international_law|0|0 - extended|global_mmlu_instruct_unk_fra:international_law|0|0 - extended|global_mmlu_instruct_all_hin:international_law|0|0 - extended|global_mmlu_instruct_ca_hin:international_law|0|0 - extended|global_mmlu_instruct_cs_hin:international_law|0|0 - extended|global_mmlu_instruct_unk_hin:international_law|0|0 - extended|global_mmlu_instruct_all_ind:international_law|0|0 - extended|global_mmlu_instruct_ca_ind:international_law|0|0 - extended|global_mmlu_instruct_cs_ind:international_law|0|0 - extended|global_mmlu_instruct_unk_ind:international_law|0|0 - extended|global_mmlu_instruct_all_ita:international_law|0|0 - extended|global_mmlu_instruct_ca_ita:international_law|0|0 - extended|global_mmlu_instruct_cs_ita:international_law|0|0 - extended|global_mmlu_instruct_unk_ita:international_law|0|0 - extended|global_mmlu_instruct_all_jpn:international_law|0|0 - extended|global_mmlu_instruct_ca_jpn:international_law|0|0 - extended|global_mmlu_instruct_cs_jpn:international_law|0|0 - extended|global_mmlu_instruct_unk_jpn:international_law|0|0 - extended|global_mmlu_instruct_all_nld:international_law|0|0 - extended|global_mmlu_instruct_ca_nld:international_law|0|0 - extended|global_mmlu_instruct_cs_nld:international_law|0|0 - extended|global_mmlu_instruct_unk_nld:international_law|0|0 - extended|global_mmlu_instruct_all_nor:international_law|0|0 - extended|global_mmlu_instruct_ca_nor:international_law|0|0 - extended|global_mmlu_instruct_cs_nor:international_law|0|0 - extended|global_mmlu_instruct_unk_nor:international_law|0|0 - extended|global_mmlu_instruct_all_pol:international_law|0|0 - extended|global_mmlu_instruct_ca_pol:international_law|0|0 - extended|global_mmlu_instruct_cs_pol:international_law|0|0 - extended|global_mmlu_instruct_unk_pol:international_law|0|0 - extended|global_mmlu_instruct_all_por:international_law|0|0 - extended|global_mmlu_instruct_ca_por:international_law|0|0 - extended|global_mmlu_instruct_cs_por:international_law|0|0 - extended|global_mmlu_instruct_unk_por:international_law|0|0 - extended|global_mmlu_instruct_all_rus:international_law|0|0 - extended|global_mmlu_instruct_ca_rus:international_law|0|0 - extended|global_mmlu_instruct_cs_rus:international_law|0|0 - extended|global_mmlu_instruct_unk_rus:international_law|0|0 - extended|global_mmlu_instruct_all_srp:international_law|0|0 - extended|global_mmlu_instruct_ca_srp:international_law|0|0 - extended|global_mmlu_instruct_cs_srp:international_law|0|0 - extended|global_mmlu_instruct_unk_srp:international_law|0|0 - extended|global_mmlu_instruct_all_swe:international_law|0|0 - extended|global_mmlu_instruct_ca_swe:international_law|0|0 - extended|global_mmlu_instruct_cs_swe:international_law|0|0 - extended|global_mmlu_instruct_unk_swe:international_law|0|0 - extended|global_mmlu_instruct_all_swa:international_law|0|0 - extended|global_mmlu_instruct_ca_swa:international_law|0|0 - extended|global_mmlu_instruct_cs_swa:international_law|0|0 - extended|global_mmlu_instruct_unk_swa:international_law|0|0 - extended|global_mmlu_instruct_all_tel:international_law|0|0 - extended|global_mmlu_instruct_ca_tel:international_law|0|0 - extended|global_mmlu_instruct_cs_tel:international_law|0|0 - extended|global_mmlu_instruct_unk_tel:international_law|0|0 - extended|global_mmlu_instruct_all_tha:international_law|0|0 - extended|global_mmlu_instruct_ca_tha:international_law|0|0 - extended|global_mmlu_instruct_cs_tha:international_law|0|0 - extended|global_mmlu_instruct_unk_tha:international_law|0|0 - extended|global_mmlu_instruct_all_tur:international_law|0|0 - extended|global_mmlu_instruct_ca_tur:international_law|0|0 - extended|global_mmlu_instruct_cs_tur:international_law|0|0 - extended|global_mmlu_instruct_unk_tur:international_law|0|0 - extended|global_mmlu_instruct_all_ukr:international_law|0|0 - extended|global_mmlu_instruct_ca_ukr:international_law|0|0 - extended|global_mmlu_instruct_cs_ukr:international_law|0|0 - extended|global_mmlu_instruct_unk_ukr:international_law|0|0 - extended|global_mmlu_instruct_all_urd:international_law|0|0 - extended|global_mmlu_instruct_ca_urd:international_law|0|0 - extended|global_mmlu_instruct_cs_urd:international_law|0|0 - extended|global_mmlu_instruct_unk_urd:international_law|0|0 - extended|global_mmlu_instruct_all_vie:international_law|0|0 - extended|global_mmlu_instruct_ca_vie:international_law|0|0 - extended|global_mmlu_instruct_cs_vie:international_law|0|0 - extended|global_mmlu_instruct_unk_vie:international_law|0|0 - extended|global_mmlu_instruct_all_ara:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_ara:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_ara:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_ara:jurisprudence|0|0 - extended|global_mmlu_instruct_all_ben:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_ben:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_ben:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_ben:jurisprudence|0|0 - extended|global_mmlu_instruct_all_deu:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_deu:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_deu:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_deu:jurisprudence|0|0 - extended|global_mmlu_instruct_all_spa:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_spa:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_spa:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_spa:jurisprudence|0|0 - extended|global_mmlu_instruct_all_eng:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_eng:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_eng:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_eng:jurisprudence|0|0 - extended|global_mmlu_instruct_all_fra:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_fra:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_fra:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_fra:jurisprudence|0|0 - extended|global_mmlu_instruct_all_hin:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_hin:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_hin:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_hin:jurisprudence|0|0 - extended|global_mmlu_instruct_all_ind:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_ind:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_ind:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_ind:jurisprudence|0|0 - extended|global_mmlu_instruct_all_ita:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_ita:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_ita:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_ita:jurisprudence|0|0 - extended|global_mmlu_instruct_all_jpn:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_jpn:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_jpn:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_jpn:jurisprudence|0|0 - extended|global_mmlu_instruct_all_nld:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_nld:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_nld:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_nld:jurisprudence|0|0 - extended|global_mmlu_instruct_all_nor:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_nor:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_nor:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_nor:jurisprudence|0|0 - extended|global_mmlu_instruct_all_pol:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_pol:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_pol:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_pol:jurisprudence|0|0 - extended|global_mmlu_instruct_all_por:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_por:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_por:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_por:jurisprudence|0|0 - extended|global_mmlu_instruct_all_rus:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_rus:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_rus:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_rus:jurisprudence|0|0 - extended|global_mmlu_instruct_all_srp:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_srp:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_srp:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_srp:jurisprudence|0|0 - extended|global_mmlu_instruct_all_swe:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_swe:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_swe:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_swe:jurisprudence|0|0 - extended|global_mmlu_instruct_all_swa:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_swa:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_swa:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_swa:jurisprudence|0|0 - extended|global_mmlu_instruct_all_tel:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_tel:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_tel:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_tel:jurisprudence|0|0 - extended|global_mmlu_instruct_all_tha:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_tha:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_tha:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_tha:jurisprudence|0|0 - extended|global_mmlu_instruct_all_tur:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_tur:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_tur:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_tur:jurisprudence|0|0 - extended|global_mmlu_instruct_all_ukr:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_ukr:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_ukr:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_ukr:jurisprudence|0|0 - extended|global_mmlu_instruct_all_urd:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_urd:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_urd:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_urd:jurisprudence|0|0 - extended|global_mmlu_instruct_all_vie:jurisprudence|0|0 - extended|global_mmlu_instruct_ca_vie:jurisprudence|0|0 - extended|global_mmlu_instruct_cs_vie:jurisprudence|0|0 - extended|global_mmlu_instruct_unk_vie:jurisprudence|0|0 - extended|global_mmlu_instruct_all_ara:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_ara:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_ara:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_ara:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_ben:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_ben:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_ben:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_ben:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_deu:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_deu:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_deu:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_deu:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_spa:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_spa:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_spa:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_spa:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_eng:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_eng:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_eng:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_eng:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_fra:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_fra:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_fra:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_fra:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_hin:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_hin:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_hin:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_hin:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_ind:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_ind:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_ind:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_ind:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_ita:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_ita:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_ita:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_ita:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_jpn:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_jpn:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_jpn:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_jpn:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_nld:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_nld:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_nld:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_nld:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_nor:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_nor:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_nor:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_nor:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_pol:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_pol:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_pol:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_pol:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_por:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_por:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_por:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_por:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_rus:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_rus:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_rus:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_rus:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_srp:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_srp:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_srp:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_srp:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_swe:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_swe:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_swe:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_swe:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_swa:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_swa:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_swa:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_swa:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_tel:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_tel:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_tel:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_tel:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_tha:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_tha:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_tha:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_tha:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_tur:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_tur:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_tur:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_tur:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_ukr:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_ukr:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_ukr:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_ukr:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_urd:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_urd:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_urd:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_urd:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_vie:logical_fallacies|0|0 - extended|global_mmlu_instruct_ca_vie:logical_fallacies|0|0 - extended|global_mmlu_instruct_cs_vie:logical_fallacies|0|0 - extended|global_mmlu_instruct_unk_vie:logical_fallacies|0|0 - extended|global_mmlu_instruct_all_ara:machine_learning|0|0 - extended|global_mmlu_instruct_ca_ara:machine_learning|0|0 - extended|global_mmlu_instruct_cs_ara:machine_learning|0|0 - extended|global_mmlu_instruct_unk_ara:machine_learning|0|0 - extended|global_mmlu_instruct_all_ben:machine_learning|0|0 - extended|global_mmlu_instruct_ca_ben:machine_learning|0|0 - extended|global_mmlu_instruct_cs_ben:machine_learning|0|0 - extended|global_mmlu_instruct_unk_ben:machine_learning|0|0 - extended|global_mmlu_instruct_all_deu:machine_learning|0|0 - extended|global_mmlu_instruct_ca_deu:machine_learning|0|0 - extended|global_mmlu_instruct_cs_deu:machine_learning|0|0 - extended|global_mmlu_instruct_unk_deu:machine_learning|0|0 - extended|global_mmlu_instruct_all_spa:machine_learning|0|0 - extended|global_mmlu_instruct_ca_spa:machine_learning|0|0 - extended|global_mmlu_instruct_cs_spa:machine_learning|0|0 - extended|global_mmlu_instruct_unk_spa:machine_learning|0|0 - extended|global_mmlu_instruct_all_eng:machine_learning|0|0 - extended|global_mmlu_instruct_ca_eng:machine_learning|0|0 - extended|global_mmlu_instruct_cs_eng:machine_learning|0|0 - extended|global_mmlu_instruct_unk_eng:machine_learning|0|0 - extended|global_mmlu_instruct_all_fra:machine_learning|0|0 - extended|global_mmlu_instruct_ca_fra:machine_learning|0|0 - extended|global_mmlu_instruct_cs_fra:machine_learning|0|0 - extended|global_mmlu_instruct_unk_fra:machine_learning|0|0 - extended|global_mmlu_instruct_all_hin:machine_learning|0|0 - extended|global_mmlu_instruct_ca_hin:machine_learning|0|0 - extended|global_mmlu_instruct_cs_hin:machine_learning|0|0 - extended|global_mmlu_instruct_unk_hin:machine_learning|0|0 - extended|global_mmlu_instruct_all_ind:machine_learning|0|0 - extended|global_mmlu_instruct_ca_ind:machine_learning|0|0 - extended|global_mmlu_instruct_cs_ind:machine_learning|0|0 - extended|global_mmlu_instruct_unk_ind:machine_learning|0|0 - extended|global_mmlu_instruct_all_ita:machine_learning|0|0 - extended|global_mmlu_instruct_ca_ita:machine_learning|0|0 - extended|global_mmlu_instruct_cs_ita:machine_learning|0|0 - extended|global_mmlu_instruct_unk_ita:machine_learning|0|0 - extended|global_mmlu_instruct_all_jpn:machine_learning|0|0 - extended|global_mmlu_instruct_ca_jpn:machine_learning|0|0 - extended|global_mmlu_instruct_cs_jpn:machine_learning|0|0 - extended|global_mmlu_instruct_unk_jpn:machine_learning|0|0 - extended|global_mmlu_instruct_all_nld:machine_learning|0|0 - extended|global_mmlu_instruct_ca_nld:machine_learning|0|0 - extended|global_mmlu_instruct_cs_nld:machine_learning|0|0 - extended|global_mmlu_instruct_unk_nld:machine_learning|0|0 - extended|global_mmlu_instruct_all_nor:machine_learning|0|0 - extended|global_mmlu_instruct_ca_nor:machine_learning|0|0 - extended|global_mmlu_instruct_cs_nor:machine_learning|0|0 - extended|global_mmlu_instruct_unk_nor:machine_learning|0|0 - extended|global_mmlu_instruct_all_pol:machine_learning|0|0 - extended|global_mmlu_instruct_ca_pol:machine_learning|0|0 - extended|global_mmlu_instruct_cs_pol:machine_learning|0|0 - extended|global_mmlu_instruct_unk_pol:machine_learning|0|0 - extended|global_mmlu_instruct_all_por:machine_learning|0|0 - extended|global_mmlu_instruct_ca_por:machine_learning|0|0 - extended|global_mmlu_instruct_cs_por:machine_learning|0|0 - extended|global_mmlu_instruct_unk_por:machine_learning|0|0 - extended|global_mmlu_instruct_all_rus:machine_learning|0|0 - extended|global_mmlu_instruct_ca_rus:machine_learning|0|0 - extended|global_mmlu_instruct_cs_rus:machine_learning|0|0 - extended|global_mmlu_instruct_unk_rus:machine_learning|0|0 - extended|global_mmlu_instruct_all_srp:machine_learning|0|0 - extended|global_mmlu_instruct_ca_srp:machine_learning|0|0 - extended|global_mmlu_instruct_cs_srp:machine_learning|0|0 - extended|global_mmlu_instruct_unk_srp:machine_learning|0|0 - extended|global_mmlu_instruct_all_swe:machine_learning|0|0 - extended|global_mmlu_instruct_ca_swe:machine_learning|0|0 - extended|global_mmlu_instruct_cs_swe:machine_learning|0|0 - extended|global_mmlu_instruct_unk_swe:machine_learning|0|0 - extended|global_mmlu_instruct_all_swa:machine_learning|0|0 - extended|global_mmlu_instruct_ca_swa:machine_learning|0|0 - extended|global_mmlu_instruct_cs_swa:machine_learning|0|0 - extended|global_mmlu_instruct_unk_swa:machine_learning|0|0 - extended|global_mmlu_instruct_all_tel:machine_learning|0|0 - extended|global_mmlu_instruct_ca_tel:machine_learning|0|0 - extended|global_mmlu_instruct_cs_tel:machine_learning|0|0 - extended|global_mmlu_instruct_unk_tel:machine_learning|0|0 - extended|global_mmlu_instruct_all_tha:machine_learning|0|0 - extended|global_mmlu_instruct_ca_tha:machine_learning|0|0 - extended|global_mmlu_instruct_cs_tha:machine_learning|0|0 - extended|global_mmlu_instruct_unk_tha:machine_learning|0|0 - extended|global_mmlu_instruct_all_tur:machine_learning|0|0 - extended|global_mmlu_instruct_ca_tur:machine_learning|0|0 - extended|global_mmlu_instruct_cs_tur:machine_learning|0|0 - extended|global_mmlu_instruct_unk_tur:machine_learning|0|0 - extended|global_mmlu_instruct_all_ukr:machine_learning|0|0 - extended|global_mmlu_instruct_ca_ukr:machine_learning|0|0 - extended|global_mmlu_instruct_cs_ukr:machine_learning|0|0 - extended|global_mmlu_instruct_unk_ukr:machine_learning|0|0 - extended|global_mmlu_instruct_all_urd:machine_learning|0|0 - extended|global_mmlu_instruct_ca_urd:machine_learning|0|0 - extended|global_mmlu_instruct_cs_urd:machine_learning|0|0 - extended|global_mmlu_instruct_unk_urd:machine_learning|0|0 - extended|global_mmlu_instruct_all_vie:machine_learning|0|0 - extended|global_mmlu_instruct_ca_vie:machine_learning|0|0 - extended|global_mmlu_instruct_cs_vie:machine_learning|0|0 - extended|global_mmlu_instruct_unk_vie:machine_learning|0|0 - extended|global_mmlu_instruct_all_ara:management|0|0 - extended|global_mmlu_instruct_ca_ara:management|0|0 - extended|global_mmlu_instruct_cs_ara:management|0|0 - extended|global_mmlu_instruct_unk_ara:management|0|0 - extended|global_mmlu_instruct_all_ben:management|0|0 - extended|global_mmlu_instruct_ca_ben:management|0|0 - extended|global_mmlu_instruct_cs_ben:management|0|0 - extended|global_mmlu_instruct_unk_ben:management|0|0 - extended|global_mmlu_instruct_all_deu:management|0|0 - extended|global_mmlu_instruct_ca_deu:management|0|0 - extended|global_mmlu_instruct_cs_deu:management|0|0 - extended|global_mmlu_instruct_unk_deu:management|0|0 - extended|global_mmlu_instruct_all_spa:management|0|0 - extended|global_mmlu_instruct_ca_spa:management|0|0 - extended|global_mmlu_instruct_cs_spa:management|0|0 - extended|global_mmlu_instruct_unk_spa:management|0|0 - extended|global_mmlu_instruct_all_eng:management|0|0 - extended|global_mmlu_instruct_ca_eng:management|0|0 - extended|global_mmlu_instruct_cs_eng:management|0|0 - extended|global_mmlu_instruct_unk_eng:management|0|0 - extended|global_mmlu_instruct_all_fra:management|0|0 - extended|global_mmlu_instruct_ca_fra:management|0|0 - extended|global_mmlu_instruct_cs_fra:management|0|0 - extended|global_mmlu_instruct_unk_fra:management|0|0 - extended|global_mmlu_instruct_all_hin:management|0|0 - extended|global_mmlu_instruct_ca_hin:management|0|0 - extended|global_mmlu_instruct_cs_hin:management|0|0 - extended|global_mmlu_instruct_unk_hin:management|0|0 - extended|global_mmlu_instruct_all_ind:management|0|0 - extended|global_mmlu_instruct_ca_ind:management|0|0 - extended|global_mmlu_instruct_cs_ind:management|0|0 - extended|global_mmlu_instruct_unk_ind:management|0|0 - extended|global_mmlu_instruct_all_ita:management|0|0 - extended|global_mmlu_instruct_ca_ita:management|0|0 - extended|global_mmlu_instruct_cs_ita:management|0|0 - extended|global_mmlu_instruct_unk_ita:management|0|0 - extended|global_mmlu_instruct_all_jpn:management|0|0 - extended|global_mmlu_instruct_ca_jpn:management|0|0 - extended|global_mmlu_instruct_cs_jpn:management|0|0 - extended|global_mmlu_instruct_unk_jpn:management|0|0 - extended|global_mmlu_instruct_all_nld:management|0|0 - extended|global_mmlu_instruct_ca_nld:management|0|0 - extended|global_mmlu_instruct_cs_nld:management|0|0 - extended|global_mmlu_instruct_unk_nld:management|0|0 - extended|global_mmlu_instruct_all_nor:management|0|0 - extended|global_mmlu_instruct_ca_nor:management|0|0 - extended|global_mmlu_instruct_cs_nor:management|0|0 - extended|global_mmlu_instruct_unk_nor:management|0|0 - extended|global_mmlu_instruct_all_pol:management|0|0 - extended|global_mmlu_instruct_ca_pol:management|0|0 - extended|global_mmlu_instruct_cs_pol:management|0|0 - extended|global_mmlu_instruct_unk_pol:management|0|0 - extended|global_mmlu_instruct_all_por:management|0|0 - extended|global_mmlu_instruct_ca_por:management|0|0 - extended|global_mmlu_instruct_cs_por:management|0|0 - extended|global_mmlu_instruct_unk_por:management|0|0 - extended|global_mmlu_instruct_all_rus:management|0|0 - extended|global_mmlu_instruct_ca_rus:management|0|0 - extended|global_mmlu_instruct_cs_rus:management|0|0 - extended|global_mmlu_instruct_unk_rus:management|0|0 - extended|global_mmlu_instruct_all_srp:management|0|0 - extended|global_mmlu_instruct_ca_srp:management|0|0 - extended|global_mmlu_instruct_cs_srp:management|0|0 - extended|global_mmlu_instruct_unk_srp:management|0|0 - extended|global_mmlu_instruct_all_swe:management|0|0 - extended|global_mmlu_instruct_ca_swe:management|0|0 - extended|global_mmlu_instruct_cs_swe:management|0|0 - extended|global_mmlu_instruct_unk_swe:management|0|0 - extended|global_mmlu_instruct_all_swa:management|0|0 - extended|global_mmlu_instruct_ca_swa:management|0|0 - extended|global_mmlu_instruct_cs_swa:management|0|0 - extended|global_mmlu_instruct_unk_swa:management|0|0 - extended|global_mmlu_instruct_all_tel:management|0|0 - extended|global_mmlu_instruct_ca_tel:management|0|0 - extended|global_mmlu_instruct_cs_tel:management|0|0 - extended|global_mmlu_instruct_unk_tel:management|0|0 - extended|global_mmlu_instruct_all_tha:management|0|0 - extended|global_mmlu_instruct_ca_tha:management|0|0 - extended|global_mmlu_instruct_cs_tha:management|0|0 - extended|global_mmlu_instruct_unk_tha:management|0|0 - extended|global_mmlu_instruct_all_tur:management|0|0 - extended|global_mmlu_instruct_ca_tur:management|0|0 - extended|global_mmlu_instruct_cs_tur:management|0|0 - extended|global_mmlu_instruct_unk_tur:management|0|0 - extended|global_mmlu_instruct_all_ukr:management|0|0 - extended|global_mmlu_instruct_ca_ukr:management|0|0 - extended|global_mmlu_instruct_cs_ukr:management|0|0 - extended|global_mmlu_instruct_unk_ukr:management|0|0 - extended|global_mmlu_instruct_all_urd:management|0|0 - extended|global_mmlu_instruct_ca_urd:management|0|0 - extended|global_mmlu_instruct_cs_urd:management|0|0 - extended|global_mmlu_instruct_unk_urd:management|0|0 - extended|global_mmlu_instruct_all_vie:management|0|0 - extended|global_mmlu_instruct_ca_vie:management|0|0 - extended|global_mmlu_instruct_cs_vie:management|0|0 - extended|global_mmlu_instruct_unk_vie:management|0|0 - extended|global_mmlu_instruct_all_ara:marketing|0|0 - extended|global_mmlu_instruct_ca_ara:marketing|0|0 - extended|global_mmlu_instruct_cs_ara:marketing|0|0 - extended|global_mmlu_instruct_unk_ara:marketing|0|0 - extended|global_mmlu_instruct_all_ben:marketing|0|0 - extended|global_mmlu_instruct_ca_ben:marketing|0|0 - extended|global_mmlu_instruct_cs_ben:marketing|0|0 - extended|global_mmlu_instruct_unk_ben:marketing|0|0 - extended|global_mmlu_instruct_all_deu:marketing|0|0 - extended|global_mmlu_instruct_ca_deu:marketing|0|0 - extended|global_mmlu_instruct_cs_deu:marketing|0|0 - extended|global_mmlu_instruct_unk_deu:marketing|0|0 - extended|global_mmlu_instruct_all_spa:marketing|0|0 - extended|global_mmlu_instruct_ca_spa:marketing|0|0 - extended|global_mmlu_instruct_cs_spa:marketing|0|0 - extended|global_mmlu_instruct_unk_spa:marketing|0|0 - extended|global_mmlu_instruct_all_eng:marketing|0|0 - extended|global_mmlu_instruct_ca_eng:marketing|0|0 - extended|global_mmlu_instruct_cs_eng:marketing|0|0 - extended|global_mmlu_instruct_unk_eng:marketing|0|0 - extended|global_mmlu_instruct_all_fra:marketing|0|0 - extended|global_mmlu_instruct_ca_fra:marketing|0|0 - extended|global_mmlu_instruct_cs_fra:marketing|0|0 - extended|global_mmlu_instruct_unk_fra:marketing|0|0 - extended|global_mmlu_instruct_all_hin:marketing|0|0 - extended|global_mmlu_instruct_ca_hin:marketing|0|0 - extended|global_mmlu_instruct_cs_hin:marketing|0|0 - extended|global_mmlu_instruct_unk_hin:marketing|0|0 - extended|global_mmlu_instruct_all_ind:marketing|0|0 - extended|global_mmlu_instruct_ca_ind:marketing|0|0 - extended|global_mmlu_instruct_cs_ind:marketing|0|0 - extended|global_mmlu_instruct_unk_ind:marketing|0|0 - extended|global_mmlu_instruct_all_ita:marketing|0|0 - extended|global_mmlu_instruct_ca_ita:marketing|0|0 - extended|global_mmlu_instruct_cs_ita:marketing|0|0 - extended|global_mmlu_instruct_unk_ita:marketing|0|0 - extended|global_mmlu_instruct_all_jpn:marketing|0|0 - extended|global_mmlu_instruct_ca_jpn:marketing|0|0 - extended|global_mmlu_instruct_cs_jpn:marketing|0|0 - extended|global_mmlu_instruct_unk_jpn:marketing|0|0 - extended|global_mmlu_instruct_all_nld:marketing|0|0 - extended|global_mmlu_instruct_ca_nld:marketing|0|0 - extended|global_mmlu_instruct_cs_nld:marketing|0|0 - extended|global_mmlu_instruct_unk_nld:marketing|0|0 - extended|global_mmlu_instruct_all_nor:marketing|0|0 - extended|global_mmlu_instruct_ca_nor:marketing|0|0 - extended|global_mmlu_instruct_cs_nor:marketing|0|0 - extended|global_mmlu_instruct_unk_nor:marketing|0|0 - extended|global_mmlu_instruct_all_pol:marketing|0|0 - extended|global_mmlu_instruct_ca_pol:marketing|0|0 - extended|global_mmlu_instruct_cs_pol:marketing|0|0 - extended|global_mmlu_instruct_unk_pol:marketing|0|0 - extended|global_mmlu_instruct_all_por:marketing|0|0 - extended|global_mmlu_instruct_ca_por:marketing|0|0 - extended|global_mmlu_instruct_cs_por:marketing|0|0 - extended|global_mmlu_instruct_unk_por:marketing|0|0 - extended|global_mmlu_instruct_all_rus:marketing|0|0 - extended|global_mmlu_instruct_ca_rus:marketing|0|0 - extended|global_mmlu_instruct_cs_rus:marketing|0|0 - extended|global_mmlu_instruct_unk_rus:marketing|0|0 - extended|global_mmlu_instruct_all_srp:marketing|0|0 - extended|global_mmlu_instruct_ca_srp:marketing|0|0 - extended|global_mmlu_instruct_cs_srp:marketing|0|0 - extended|global_mmlu_instruct_unk_srp:marketing|0|0 - extended|global_mmlu_instruct_all_swe:marketing|0|0 - extended|global_mmlu_instruct_ca_swe:marketing|0|0 - extended|global_mmlu_instruct_cs_swe:marketing|0|0 - extended|global_mmlu_instruct_unk_swe:marketing|0|0 - extended|global_mmlu_instruct_all_swa:marketing|0|0 - extended|global_mmlu_instruct_ca_swa:marketing|0|0 - extended|global_mmlu_instruct_cs_swa:marketing|0|0 - extended|global_mmlu_instruct_unk_swa:marketing|0|0 - extended|global_mmlu_instruct_all_tel:marketing|0|0 - extended|global_mmlu_instruct_ca_tel:marketing|0|0 - extended|global_mmlu_instruct_cs_tel:marketing|0|0 - extended|global_mmlu_instruct_unk_tel:marketing|0|0 - extended|global_mmlu_instruct_all_tha:marketing|0|0 - extended|global_mmlu_instruct_ca_tha:marketing|0|0 - extended|global_mmlu_instruct_cs_tha:marketing|0|0 - extended|global_mmlu_instruct_unk_tha:marketing|0|0 - extended|global_mmlu_instruct_all_tur:marketing|0|0 - extended|global_mmlu_instruct_ca_tur:marketing|0|0 - extended|global_mmlu_instruct_cs_tur:marketing|0|0 - extended|global_mmlu_instruct_unk_tur:marketing|0|0 - extended|global_mmlu_instruct_all_ukr:marketing|0|0 - extended|global_mmlu_instruct_ca_ukr:marketing|0|0 - extended|global_mmlu_instruct_cs_ukr:marketing|0|0 - extended|global_mmlu_instruct_unk_ukr:marketing|0|0 - extended|global_mmlu_instruct_all_urd:marketing|0|0 - extended|global_mmlu_instruct_ca_urd:marketing|0|0 - extended|global_mmlu_instruct_cs_urd:marketing|0|0 - extended|global_mmlu_instruct_unk_urd:marketing|0|0 - extended|global_mmlu_instruct_all_vie:marketing|0|0 - extended|global_mmlu_instruct_ca_vie:marketing|0|0 - extended|global_mmlu_instruct_cs_vie:marketing|0|0 - extended|global_mmlu_instruct_unk_vie:marketing|0|0 - extended|global_mmlu_instruct_all_ara:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_ara:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_ara:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_ara:medical_genetics|0|0 - extended|global_mmlu_instruct_all_ben:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_ben:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_ben:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_ben:medical_genetics|0|0 - extended|global_mmlu_instruct_all_deu:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_deu:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_deu:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_deu:medical_genetics|0|0 - extended|global_mmlu_instruct_all_spa:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_spa:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_spa:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_spa:medical_genetics|0|0 - extended|global_mmlu_instruct_all_eng:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_eng:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_eng:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_eng:medical_genetics|0|0 - extended|global_mmlu_instruct_all_fra:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_fra:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_fra:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_fra:medical_genetics|0|0 - extended|global_mmlu_instruct_all_hin:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_hin:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_hin:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_hin:medical_genetics|0|0 - extended|global_mmlu_instruct_all_ind:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_ind:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_ind:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_ind:medical_genetics|0|0 - extended|global_mmlu_instruct_all_ita:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_ita:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_ita:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_ita:medical_genetics|0|0 - extended|global_mmlu_instruct_all_jpn:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_jpn:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_jpn:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_jpn:medical_genetics|0|0 - extended|global_mmlu_instruct_all_nld:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_nld:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_nld:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_nld:medical_genetics|0|0 - extended|global_mmlu_instruct_all_nor:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_nor:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_nor:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_nor:medical_genetics|0|0 - extended|global_mmlu_instruct_all_pol:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_pol:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_pol:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_pol:medical_genetics|0|0 - extended|global_mmlu_instruct_all_por:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_por:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_por:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_por:medical_genetics|0|0 - extended|global_mmlu_instruct_all_rus:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_rus:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_rus:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_rus:medical_genetics|0|0 - extended|global_mmlu_instruct_all_srp:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_srp:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_srp:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_srp:medical_genetics|0|0 - extended|global_mmlu_instruct_all_swe:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_swe:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_swe:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_swe:medical_genetics|0|0 - extended|global_mmlu_instruct_all_swa:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_swa:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_swa:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_swa:medical_genetics|0|0 - extended|global_mmlu_instruct_all_tel:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_tel:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_tel:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_tel:medical_genetics|0|0 - extended|global_mmlu_instruct_all_tha:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_tha:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_tha:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_tha:medical_genetics|0|0 - extended|global_mmlu_instruct_all_tur:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_tur:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_tur:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_tur:medical_genetics|0|0 - extended|global_mmlu_instruct_all_ukr:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_ukr:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_ukr:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_ukr:medical_genetics|0|0 - extended|global_mmlu_instruct_all_urd:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_urd:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_urd:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_urd:medical_genetics|0|0 - extended|global_mmlu_instruct_all_vie:medical_genetics|0|0 - extended|global_mmlu_instruct_ca_vie:medical_genetics|0|0 - extended|global_mmlu_instruct_cs_vie:medical_genetics|0|0 - extended|global_mmlu_instruct_unk_vie:medical_genetics|0|0 - extended|global_mmlu_instruct_all_ara:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_ara:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_ara:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_ara:miscellaneous|0|0 - extended|global_mmlu_instruct_all_ben:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_ben:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_ben:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_ben:miscellaneous|0|0 - extended|global_mmlu_instruct_all_deu:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_deu:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_deu:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_deu:miscellaneous|0|0 - extended|global_mmlu_instruct_all_spa:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_spa:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_spa:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_spa:miscellaneous|0|0 - extended|global_mmlu_instruct_all_eng:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_eng:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_eng:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_eng:miscellaneous|0|0 - extended|global_mmlu_instruct_all_fra:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_fra:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_fra:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_fra:miscellaneous|0|0 - extended|global_mmlu_instruct_all_hin:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_hin:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_hin:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_hin:miscellaneous|0|0 - extended|global_mmlu_instruct_all_ind:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_ind:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_ind:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_ind:miscellaneous|0|0 - extended|global_mmlu_instruct_all_ita:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_ita:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_ita:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_ita:miscellaneous|0|0 - extended|global_mmlu_instruct_all_jpn:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_jpn:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_jpn:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_jpn:miscellaneous|0|0 - extended|global_mmlu_instruct_all_nld:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_nld:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_nld:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_nld:miscellaneous|0|0 - extended|global_mmlu_instruct_all_nor:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_nor:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_nor:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_nor:miscellaneous|0|0 - extended|global_mmlu_instruct_all_pol:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_pol:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_pol:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_pol:miscellaneous|0|0 - extended|global_mmlu_instruct_all_por:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_por:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_por:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_por:miscellaneous|0|0 - extended|global_mmlu_instruct_all_rus:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_rus:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_rus:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_rus:miscellaneous|0|0 - extended|global_mmlu_instruct_all_srp:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_srp:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_srp:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_srp:miscellaneous|0|0 - extended|global_mmlu_instruct_all_swe:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_swe:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_swe:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_swe:miscellaneous|0|0 - extended|global_mmlu_instruct_all_swa:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_swa:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_swa:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_swa:miscellaneous|0|0 - extended|global_mmlu_instruct_all_tel:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_tel:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_tel:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_tel:miscellaneous|0|0 - extended|global_mmlu_instruct_all_tha:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_tha:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_tha:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_tha:miscellaneous|0|0 - extended|global_mmlu_instruct_all_tur:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_tur:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_tur:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_tur:miscellaneous|0|0 - extended|global_mmlu_instruct_all_ukr:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_ukr:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_ukr:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_ukr:miscellaneous|0|0 - extended|global_mmlu_instruct_all_urd:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_urd:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_urd:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_urd:miscellaneous|0|0 - extended|global_mmlu_instruct_all_vie:miscellaneous|0|0 - extended|global_mmlu_instruct_ca_vie:miscellaneous|0|0 - extended|global_mmlu_instruct_cs_vie:miscellaneous|0|0 - extended|global_mmlu_instruct_unk_vie:miscellaneous|0|0 - extended|global_mmlu_instruct_all_ara:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_ara:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_ara:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_ara:moral_disputes|0|0 - extended|global_mmlu_instruct_all_ben:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_ben:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_ben:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_ben:moral_disputes|0|0 - extended|global_mmlu_instruct_all_deu:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_deu:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_deu:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_deu:moral_disputes|0|0 - extended|global_mmlu_instruct_all_spa:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_spa:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_spa:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_spa:moral_disputes|0|0 - extended|global_mmlu_instruct_all_eng:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_eng:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_eng:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_eng:moral_disputes|0|0 - extended|global_mmlu_instruct_all_fra:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_fra:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_fra:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_fra:moral_disputes|0|0 - extended|global_mmlu_instruct_all_hin:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_hin:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_hin:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_hin:moral_disputes|0|0 - extended|global_mmlu_instruct_all_ind:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_ind:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_ind:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_ind:moral_disputes|0|0 - extended|global_mmlu_instruct_all_ita:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_ita:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_ita:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_ita:moral_disputes|0|0 - extended|global_mmlu_instruct_all_jpn:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_jpn:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_jpn:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_jpn:moral_disputes|0|0 - extended|global_mmlu_instruct_all_nld:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_nld:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_nld:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_nld:moral_disputes|0|0 - extended|global_mmlu_instruct_all_nor:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_nor:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_nor:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_nor:moral_disputes|0|0 - extended|global_mmlu_instruct_all_pol:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_pol:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_pol:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_pol:moral_disputes|0|0 - extended|global_mmlu_instruct_all_por:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_por:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_por:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_por:moral_disputes|0|0 - extended|global_mmlu_instruct_all_rus:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_rus:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_rus:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_rus:moral_disputes|0|0 - extended|global_mmlu_instruct_all_srp:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_srp:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_srp:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_srp:moral_disputes|0|0 - extended|global_mmlu_instruct_all_swe:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_swe:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_swe:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_swe:moral_disputes|0|0 - extended|global_mmlu_instruct_all_swa:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_swa:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_swa:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_swa:moral_disputes|0|0 - extended|global_mmlu_instruct_all_tel:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_tel:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_tel:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_tel:moral_disputes|0|0 - extended|global_mmlu_instruct_all_tha:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_tha:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_tha:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_tha:moral_disputes|0|0 - extended|global_mmlu_instruct_all_tur:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_tur:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_tur:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_tur:moral_disputes|0|0 - extended|global_mmlu_instruct_all_ukr:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_ukr:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_ukr:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_ukr:moral_disputes|0|0 - extended|global_mmlu_instruct_all_urd:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_urd:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_urd:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_urd:moral_disputes|0|0 - extended|global_mmlu_instruct_all_vie:moral_disputes|0|0 - extended|global_mmlu_instruct_ca_vie:moral_disputes|0|0 - extended|global_mmlu_instruct_cs_vie:moral_disputes|0|0 - extended|global_mmlu_instruct_unk_vie:moral_disputes|0|0 - extended|global_mmlu_instruct_all_ara:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_ara:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_ara:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_ara:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_ben:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_ben:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_ben:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_ben:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_deu:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_deu:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_deu:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_deu:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_spa:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_spa:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_spa:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_spa:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_eng:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_eng:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_eng:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_eng:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_fra:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_fra:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_fra:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_fra:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_hin:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_hin:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_hin:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_hin:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_ind:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_ind:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_ind:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_ind:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_ita:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_ita:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_ita:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_ita:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_jpn:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_jpn:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_jpn:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_jpn:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_nld:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_nld:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_nld:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_nld:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_nor:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_nor:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_nor:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_nor:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_pol:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_pol:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_pol:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_pol:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_por:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_por:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_por:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_por:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_rus:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_rus:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_rus:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_rus:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_srp:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_srp:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_srp:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_srp:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_swe:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_swe:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_swe:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_swe:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_swa:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_swa:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_swa:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_swa:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_tel:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_tel:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_tel:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_tel:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_tha:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_tha:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_tha:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_tha:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_tur:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_tur:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_tur:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_tur:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_ukr:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_ukr:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_ukr:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_ukr:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_urd:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_urd:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_urd:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_urd:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_vie:moral_scenarios|0|0 - extended|global_mmlu_instruct_ca_vie:moral_scenarios|0|0 - extended|global_mmlu_instruct_cs_vie:moral_scenarios|0|0 - extended|global_mmlu_instruct_unk_vie:moral_scenarios|0|0 - extended|global_mmlu_instruct_all_ara:nutrition|0|0 - extended|global_mmlu_instruct_ca_ara:nutrition|0|0 - extended|global_mmlu_instruct_cs_ara:nutrition|0|0 - extended|global_mmlu_instruct_unk_ara:nutrition|0|0 - extended|global_mmlu_instruct_all_ben:nutrition|0|0 - extended|global_mmlu_instruct_ca_ben:nutrition|0|0 - extended|global_mmlu_instruct_cs_ben:nutrition|0|0 - extended|global_mmlu_instruct_unk_ben:nutrition|0|0 - extended|global_mmlu_instruct_all_deu:nutrition|0|0 - extended|global_mmlu_instruct_ca_deu:nutrition|0|0 - extended|global_mmlu_instruct_cs_deu:nutrition|0|0 - extended|global_mmlu_instruct_unk_deu:nutrition|0|0 - extended|global_mmlu_instruct_all_spa:nutrition|0|0 - extended|global_mmlu_instruct_ca_spa:nutrition|0|0 - extended|global_mmlu_instruct_cs_spa:nutrition|0|0 - extended|global_mmlu_instruct_unk_spa:nutrition|0|0 - extended|global_mmlu_instruct_all_eng:nutrition|0|0 - extended|global_mmlu_instruct_ca_eng:nutrition|0|0 - extended|global_mmlu_instruct_cs_eng:nutrition|0|0 - extended|global_mmlu_instruct_unk_eng:nutrition|0|0 - extended|global_mmlu_instruct_all_fra:nutrition|0|0 - extended|global_mmlu_instruct_ca_fra:nutrition|0|0 - extended|global_mmlu_instruct_cs_fra:nutrition|0|0 - extended|global_mmlu_instruct_unk_fra:nutrition|0|0 - extended|global_mmlu_instruct_all_hin:nutrition|0|0 - extended|global_mmlu_instruct_ca_hin:nutrition|0|0 - extended|global_mmlu_instruct_cs_hin:nutrition|0|0 - extended|global_mmlu_instruct_unk_hin:nutrition|0|0 - extended|global_mmlu_instruct_all_ind:nutrition|0|0 - extended|global_mmlu_instruct_ca_ind:nutrition|0|0 - extended|global_mmlu_instruct_cs_ind:nutrition|0|0 - extended|global_mmlu_instruct_unk_ind:nutrition|0|0 - extended|global_mmlu_instruct_all_ita:nutrition|0|0 - extended|global_mmlu_instruct_ca_ita:nutrition|0|0 - extended|global_mmlu_instruct_cs_ita:nutrition|0|0 - extended|global_mmlu_instruct_unk_ita:nutrition|0|0 - extended|global_mmlu_instruct_all_jpn:nutrition|0|0 - extended|global_mmlu_instruct_ca_jpn:nutrition|0|0 - extended|global_mmlu_instruct_cs_jpn:nutrition|0|0 - extended|global_mmlu_instruct_unk_jpn:nutrition|0|0 - extended|global_mmlu_instruct_all_nld:nutrition|0|0 - extended|global_mmlu_instruct_ca_nld:nutrition|0|0 - extended|global_mmlu_instruct_cs_nld:nutrition|0|0 - extended|global_mmlu_instruct_unk_nld:nutrition|0|0 - extended|global_mmlu_instruct_all_nor:nutrition|0|0 - extended|global_mmlu_instruct_ca_nor:nutrition|0|0 - extended|global_mmlu_instruct_cs_nor:nutrition|0|0 - extended|global_mmlu_instruct_unk_nor:nutrition|0|0 - extended|global_mmlu_instruct_all_pol:nutrition|0|0 - extended|global_mmlu_instruct_ca_pol:nutrition|0|0 - extended|global_mmlu_instruct_cs_pol:nutrition|0|0 - extended|global_mmlu_instruct_unk_pol:nutrition|0|0 - extended|global_mmlu_instruct_all_por:nutrition|0|0 - extended|global_mmlu_instruct_ca_por:nutrition|0|0 - extended|global_mmlu_instruct_cs_por:nutrition|0|0 - extended|global_mmlu_instruct_unk_por:nutrition|0|0 - extended|global_mmlu_instruct_all_rus:nutrition|0|0 - extended|global_mmlu_instruct_ca_rus:nutrition|0|0 - extended|global_mmlu_instruct_cs_rus:nutrition|0|0 - extended|global_mmlu_instruct_unk_rus:nutrition|0|0 - extended|global_mmlu_instruct_all_srp:nutrition|0|0 - extended|global_mmlu_instruct_ca_srp:nutrition|0|0 - extended|global_mmlu_instruct_cs_srp:nutrition|0|0 - extended|global_mmlu_instruct_unk_srp:nutrition|0|0 - extended|global_mmlu_instruct_all_swe:nutrition|0|0 - extended|global_mmlu_instruct_ca_swe:nutrition|0|0 - extended|global_mmlu_instruct_cs_swe:nutrition|0|0 - extended|global_mmlu_instruct_unk_swe:nutrition|0|0 - extended|global_mmlu_instruct_all_swa:nutrition|0|0 - extended|global_mmlu_instruct_ca_swa:nutrition|0|0 - extended|global_mmlu_instruct_cs_swa:nutrition|0|0 - extended|global_mmlu_instruct_unk_swa:nutrition|0|0 - extended|global_mmlu_instruct_all_tel:nutrition|0|0 - extended|global_mmlu_instruct_ca_tel:nutrition|0|0 - extended|global_mmlu_instruct_cs_tel:nutrition|0|0 - extended|global_mmlu_instruct_unk_tel:nutrition|0|0 - extended|global_mmlu_instruct_all_tha:nutrition|0|0 - extended|global_mmlu_instruct_ca_tha:nutrition|0|0 - extended|global_mmlu_instruct_cs_tha:nutrition|0|0 - extended|global_mmlu_instruct_unk_tha:nutrition|0|0 - extended|global_mmlu_instruct_all_tur:nutrition|0|0 - extended|global_mmlu_instruct_ca_tur:nutrition|0|0 - extended|global_mmlu_instruct_cs_tur:nutrition|0|0 - extended|global_mmlu_instruct_unk_tur:nutrition|0|0 - extended|global_mmlu_instruct_all_ukr:nutrition|0|0 - extended|global_mmlu_instruct_ca_ukr:nutrition|0|0 - extended|global_mmlu_instruct_cs_ukr:nutrition|0|0 - extended|global_mmlu_instruct_unk_ukr:nutrition|0|0 - extended|global_mmlu_instruct_all_urd:nutrition|0|0 - extended|global_mmlu_instruct_ca_urd:nutrition|0|0 - extended|global_mmlu_instruct_cs_urd:nutrition|0|0 - extended|global_mmlu_instruct_unk_urd:nutrition|0|0 - extended|global_mmlu_instruct_all_vie:nutrition|0|0 - extended|global_mmlu_instruct_ca_vie:nutrition|0|0 - extended|global_mmlu_instruct_cs_vie:nutrition|0|0 - extended|global_mmlu_instruct_unk_vie:nutrition|0|0 - extended|global_mmlu_instruct_all_ara:philosophy|0|0 - extended|global_mmlu_instruct_ca_ara:philosophy|0|0 - extended|global_mmlu_instruct_cs_ara:philosophy|0|0 - extended|global_mmlu_instruct_unk_ara:philosophy|0|0 - extended|global_mmlu_instruct_all_ben:philosophy|0|0 - extended|global_mmlu_instruct_ca_ben:philosophy|0|0 - extended|global_mmlu_instruct_cs_ben:philosophy|0|0 - extended|global_mmlu_instruct_unk_ben:philosophy|0|0 - extended|global_mmlu_instruct_all_deu:philosophy|0|0 - extended|global_mmlu_instruct_ca_deu:philosophy|0|0 - extended|global_mmlu_instruct_cs_deu:philosophy|0|0 - extended|global_mmlu_instruct_unk_deu:philosophy|0|0 - extended|global_mmlu_instruct_all_spa:philosophy|0|0 - extended|global_mmlu_instruct_ca_spa:philosophy|0|0 - extended|global_mmlu_instruct_cs_spa:philosophy|0|0 - extended|global_mmlu_instruct_unk_spa:philosophy|0|0 - extended|global_mmlu_instruct_all_eng:philosophy|0|0 - extended|global_mmlu_instruct_ca_eng:philosophy|0|0 - extended|global_mmlu_instruct_cs_eng:philosophy|0|0 - extended|global_mmlu_instruct_unk_eng:philosophy|0|0 - extended|global_mmlu_instruct_all_fra:philosophy|0|0 - extended|global_mmlu_instruct_ca_fra:philosophy|0|0 - extended|global_mmlu_instruct_cs_fra:philosophy|0|0 - extended|global_mmlu_instruct_unk_fra:philosophy|0|0 - extended|global_mmlu_instruct_all_hin:philosophy|0|0 - extended|global_mmlu_instruct_ca_hin:philosophy|0|0 - extended|global_mmlu_instruct_cs_hin:philosophy|0|0 - extended|global_mmlu_instruct_unk_hin:philosophy|0|0 - extended|global_mmlu_instruct_all_ind:philosophy|0|0 - extended|global_mmlu_instruct_ca_ind:philosophy|0|0 - extended|global_mmlu_instruct_cs_ind:philosophy|0|0 - extended|global_mmlu_instruct_unk_ind:philosophy|0|0 - extended|global_mmlu_instruct_all_ita:philosophy|0|0 - extended|global_mmlu_instruct_ca_ita:philosophy|0|0 - extended|global_mmlu_instruct_cs_ita:philosophy|0|0 - extended|global_mmlu_instruct_unk_ita:philosophy|0|0 - extended|global_mmlu_instruct_all_jpn:philosophy|0|0 - extended|global_mmlu_instruct_ca_jpn:philosophy|0|0 - extended|global_mmlu_instruct_cs_jpn:philosophy|0|0 - extended|global_mmlu_instruct_unk_jpn:philosophy|0|0 - extended|global_mmlu_instruct_all_nld:philosophy|0|0 - extended|global_mmlu_instruct_ca_nld:philosophy|0|0 - extended|global_mmlu_instruct_cs_nld:philosophy|0|0 - extended|global_mmlu_instruct_unk_nld:philosophy|0|0 - extended|global_mmlu_instruct_all_nor:philosophy|0|0 - extended|global_mmlu_instruct_ca_nor:philosophy|0|0 - extended|global_mmlu_instruct_cs_nor:philosophy|0|0 - extended|global_mmlu_instruct_unk_nor:philosophy|0|0 - extended|global_mmlu_instruct_all_pol:philosophy|0|0 - extended|global_mmlu_instruct_ca_pol:philosophy|0|0 - extended|global_mmlu_instruct_cs_pol:philosophy|0|0 - extended|global_mmlu_instruct_unk_pol:philosophy|0|0 - extended|global_mmlu_instruct_all_por:philosophy|0|0 - extended|global_mmlu_instruct_ca_por:philosophy|0|0 - extended|global_mmlu_instruct_cs_por:philosophy|0|0 - extended|global_mmlu_instruct_unk_por:philosophy|0|0 - extended|global_mmlu_instruct_all_rus:philosophy|0|0 - extended|global_mmlu_instruct_ca_rus:philosophy|0|0 - extended|global_mmlu_instruct_cs_rus:philosophy|0|0 - extended|global_mmlu_instruct_unk_rus:philosophy|0|0 - extended|global_mmlu_instruct_all_srp:philosophy|0|0 - extended|global_mmlu_instruct_ca_srp:philosophy|0|0 - extended|global_mmlu_instruct_cs_srp:philosophy|0|0 - extended|global_mmlu_instruct_unk_srp:philosophy|0|0 - extended|global_mmlu_instruct_all_swe:philosophy|0|0 - extended|global_mmlu_instruct_ca_swe:philosophy|0|0 - extended|global_mmlu_instruct_cs_swe:philosophy|0|0 - extended|global_mmlu_instruct_unk_swe:philosophy|0|0 - extended|global_mmlu_instruct_all_swa:philosophy|0|0 - extended|global_mmlu_instruct_ca_swa:philosophy|0|0 - extended|global_mmlu_instruct_cs_swa:philosophy|0|0 - extended|global_mmlu_instruct_unk_swa:philosophy|0|0 - extended|global_mmlu_instruct_all_tel:philosophy|0|0 - extended|global_mmlu_instruct_ca_tel:philosophy|0|0 - extended|global_mmlu_instruct_cs_tel:philosophy|0|0 - extended|global_mmlu_instruct_unk_tel:philosophy|0|0 - extended|global_mmlu_instruct_all_tha:philosophy|0|0 - extended|global_mmlu_instruct_ca_tha:philosophy|0|0 - extended|global_mmlu_instruct_cs_tha:philosophy|0|0 - extended|global_mmlu_instruct_unk_tha:philosophy|0|0 - extended|global_mmlu_instruct_all_tur:philosophy|0|0 - extended|global_mmlu_instruct_ca_tur:philosophy|0|0 - extended|global_mmlu_instruct_cs_tur:philosophy|0|0 - extended|global_mmlu_instruct_unk_tur:philosophy|0|0 - extended|global_mmlu_instruct_all_ukr:philosophy|0|0 - extended|global_mmlu_instruct_ca_ukr:philosophy|0|0 - extended|global_mmlu_instruct_cs_ukr:philosophy|0|0 - extended|global_mmlu_instruct_unk_ukr:philosophy|0|0 - extended|global_mmlu_instruct_all_urd:philosophy|0|0 - extended|global_mmlu_instruct_ca_urd:philosophy|0|0 - extended|global_mmlu_instruct_cs_urd:philosophy|0|0 - extended|global_mmlu_instruct_unk_urd:philosophy|0|0 - extended|global_mmlu_instruct_all_vie:philosophy|0|0 - extended|global_mmlu_instruct_ca_vie:philosophy|0|0 - extended|global_mmlu_instruct_cs_vie:philosophy|0|0 - extended|global_mmlu_instruct_unk_vie:philosophy|0|0 - extended|global_mmlu_instruct_all_ara:prehistory|0|0 - extended|global_mmlu_instruct_ca_ara:prehistory|0|0 - extended|global_mmlu_instruct_cs_ara:prehistory|0|0 - extended|global_mmlu_instruct_unk_ara:prehistory|0|0 - extended|global_mmlu_instruct_all_ben:prehistory|0|0 - extended|global_mmlu_instruct_ca_ben:prehistory|0|0 - extended|global_mmlu_instruct_cs_ben:prehistory|0|0 - extended|global_mmlu_instruct_unk_ben:prehistory|0|0 - extended|global_mmlu_instruct_all_deu:prehistory|0|0 - extended|global_mmlu_instruct_ca_deu:prehistory|0|0 - extended|global_mmlu_instruct_cs_deu:prehistory|0|0 - extended|global_mmlu_instruct_unk_deu:prehistory|0|0 - extended|global_mmlu_instruct_all_spa:prehistory|0|0 - extended|global_mmlu_instruct_ca_spa:prehistory|0|0 - extended|global_mmlu_instruct_cs_spa:prehistory|0|0 - extended|global_mmlu_instruct_unk_spa:prehistory|0|0 - extended|global_mmlu_instruct_all_eng:prehistory|0|0 - extended|global_mmlu_instruct_ca_eng:prehistory|0|0 - extended|global_mmlu_instruct_cs_eng:prehistory|0|0 - extended|global_mmlu_instruct_unk_eng:prehistory|0|0 - extended|global_mmlu_instruct_all_fra:prehistory|0|0 - extended|global_mmlu_instruct_ca_fra:prehistory|0|0 - extended|global_mmlu_instruct_cs_fra:prehistory|0|0 - extended|global_mmlu_instruct_unk_fra:prehistory|0|0 - extended|global_mmlu_instruct_all_hin:prehistory|0|0 - extended|global_mmlu_instruct_ca_hin:prehistory|0|0 - extended|global_mmlu_instruct_cs_hin:prehistory|0|0 - extended|global_mmlu_instruct_unk_hin:prehistory|0|0 - extended|global_mmlu_instruct_all_ind:prehistory|0|0 - extended|global_mmlu_instruct_ca_ind:prehistory|0|0 - extended|global_mmlu_instruct_cs_ind:prehistory|0|0 - extended|global_mmlu_instruct_unk_ind:prehistory|0|0 - extended|global_mmlu_instruct_all_ita:prehistory|0|0 - extended|global_mmlu_instruct_ca_ita:prehistory|0|0 - extended|global_mmlu_instruct_cs_ita:prehistory|0|0 - extended|global_mmlu_instruct_unk_ita:prehistory|0|0 - extended|global_mmlu_instruct_all_jpn:prehistory|0|0 - extended|global_mmlu_instruct_ca_jpn:prehistory|0|0 - extended|global_mmlu_instruct_cs_jpn:prehistory|0|0 - extended|global_mmlu_instruct_unk_jpn:prehistory|0|0 - extended|global_mmlu_instruct_all_nld:prehistory|0|0 - extended|global_mmlu_instruct_ca_nld:prehistory|0|0 - extended|global_mmlu_instruct_cs_nld:prehistory|0|0 - extended|global_mmlu_instruct_unk_nld:prehistory|0|0 - extended|global_mmlu_instruct_all_nor:prehistory|0|0 - extended|global_mmlu_instruct_ca_nor:prehistory|0|0 - extended|global_mmlu_instruct_cs_nor:prehistory|0|0 - extended|global_mmlu_instruct_unk_nor:prehistory|0|0 - extended|global_mmlu_instruct_all_pol:prehistory|0|0 - extended|global_mmlu_instruct_ca_pol:prehistory|0|0 - extended|global_mmlu_instruct_cs_pol:prehistory|0|0 - extended|global_mmlu_instruct_unk_pol:prehistory|0|0 - extended|global_mmlu_instruct_all_por:prehistory|0|0 - extended|global_mmlu_instruct_ca_por:prehistory|0|0 - extended|global_mmlu_instruct_cs_por:prehistory|0|0 - extended|global_mmlu_instruct_unk_por:prehistory|0|0 - extended|global_mmlu_instruct_all_rus:prehistory|0|0 - extended|global_mmlu_instruct_ca_rus:prehistory|0|0 - extended|global_mmlu_instruct_cs_rus:prehistory|0|0 - extended|global_mmlu_instruct_unk_rus:prehistory|0|0 - extended|global_mmlu_instruct_all_srp:prehistory|0|0 - extended|global_mmlu_instruct_ca_srp:prehistory|0|0 - extended|global_mmlu_instruct_cs_srp:prehistory|0|0 - extended|global_mmlu_instruct_unk_srp:prehistory|0|0 - extended|global_mmlu_instruct_all_swe:prehistory|0|0 - extended|global_mmlu_instruct_ca_swe:prehistory|0|0 - extended|global_mmlu_instruct_cs_swe:prehistory|0|0 - extended|global_mmlu_instruct_unk_swe:prehistory|0|0 - extended|global_mmlu_instruct_all_swa:prehistory|0|0 - extended|global_mmlu_instruct_ca_swa:prehistory|0|0 - extended|global_mmlu_instruct_cs_swa:prehistory|0|0 - extended|global_mmlu_instruct_unk_swa:prehistory|0|0 - extended|global_mmlu_instruct_all_tel:prehistory|0|0 - extended|global_mmlu_instruct_ca_tel:prehistory|0|0 - extended|global_mmlu_instruct_cs_tel:prehistory|0|0 - extended|global_mmlu_instruct_unk_tel:prehistory|0|0 - extended|global_mmlu_instruct_all_tha:prehistory|0|0 - extended|global_mmlu_instruct_ca_tha:prehistory|0|0 - extended|global_mmlu_instruct_cs_tha:prehistory|0|0 - extended|global_mmlu_instruct_unk_tha:prehistory|0|0 - extended|global_mmlu_instruct_all_tur:prehistory|0|0 - extended|global_mmlu_instruct_ca_tur:prehistory|0|0 - extended|global_mmlu_instruct_cs_tur:prehistory|0|0 - extended|global_mmlu_instruct_unk_tur:prehistory|0|0 - extended|global_mmlu_instruct_all_ukr:prehistory|0|0 - extended|global_mmlu_instruct_ca_ukr:prehistory|0|0 - extended|global_mmlu_instruct_cs_ukr:prehistory|0|0 - extended|global_mmlu_instruct_unk_ukr:prehistory|0|0 - extended|global_mmlu_instruct_all_urd:prehistory|0|0 - extended|global_mmlu_instruct_ca_urd:prehistory|0|0 - extended|global_mmlu_instruct_cs_urd:prehistory|0|0 - extended|global_mmlu_instruct_unk_urd:prehistory|0|0 - extended|global_mmlu_instruct_all_vie:prehistory|0|0 - extended|global_mmlu_instruct_ca_vie:prehistory|0|0 - extended|global_mmlu_instruct_cs_vie:prehistory|0|0 - extended|global_mmlu_instruct_unk_vie:prehistory|0|0 - extended|global_mmlu_instruct_all_ara:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_ara:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_ara:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_ara:professional_accounting|0|0 - extended|global_mmlu_instruct_all_ben:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_ben:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_ben:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_ben:professional_accounting|0|0 - extended|global_mmlu_instruct_all_deu:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_deu:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_deu:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_deu:professional_accounting|0|0 - extended|global_mmlu_instruct_all_spa:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_spa:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_spa:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_spa:professional_accounting|0|0 - extended|global_mmlu_instruct_all_eng:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_eng:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_eng:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_eng:professional_accounting|0|0 - extended|global_mmlu_instruct_all_fra:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_fra:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_fra:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_fra:professional_accounting|0|0 - extended|global_mmlu_instruct_all_hin:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_hin:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_hin:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_hin:professional_accounting|0|0 - extended|global_mmlu_instruct_all_ind:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_ind:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_ind:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_ind:professional_accounting|0|0 - extended|global_mmlu_instruct_all_ita:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_ita:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_ita:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_ita:professional_accounting|0|0 - extended|global_mmlu_instruct_all_jpn:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_jpn:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_jpn:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_jpn:professional_accounting|0|0 - extended|global_mmlu_instruct_all_nld:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_nld:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_nld:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_nld:professional_accounting|0|0 - extended|global_mmlu_instruct_all_nor:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_nor:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_nor:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_nor:professional_accounting|0|0 - extended|global_mmlu_instruct_all_pol:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_pol:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_pol:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_pol:professional_accounting|0|0 - extended|global_mmlu_instruct_all_por:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_por:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_por:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_por:professional_accounting|0|0 - extended|global_mmlu_instruct_all_rus:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_rus:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_rus:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_rus:professional_accounting|0|0 - extended|global_mmlu_instruct_all_srp:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_srp:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_srp:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_srp:professional_accounting|0|0 - extended|global_mmlu_instruct_all_swe:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_swe:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_swe:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_swe:professional_accounting|0|0 - extended|global_mmlu_instruct_all_swa:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_swa:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_swa:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_swa:professional_accounting|0|0 - extended|global_mmlu_instruct_all_tel:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_tel:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_tel:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_tel:professional_accounting|0|0 - extended|global_mmlu_instruct_all_tha:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_tha:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_tha:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_tha:professional_accounting|0|0 - extended|global_mmlu_instruct_all_tur:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_tur:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_tur:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_tur:professional_accounting|0|0 - extended|global_mmlu_instruct_all_ukr:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_ukr:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_ukr:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_ukr:professional_accounting|0|0 - extended|global_mmlu_instruct_all_urd:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_urd:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_urd:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_urd:professional_accounting|0|0 - extended|global_mmlu_instruct_all_vie:professional_accounting|0|0 - extended|global_mmlu_instruct_ca_vie:professional_accounting|0|0 - extended|global_mmlu_instruct_cs_vie:professional_accounting|0|0 - extended|global_mmlu_instruct_unk_vie:professional_accounting|0|0 - extended|global_mmlu_instruct_all_ara:professional_law|0|0 - extended|global_mmlu_instruct_ca_ara:professional_law|0|0 - extended|global_mmlu_instruct_cs_ara:professional_law|0|0 - extended|global_mmlu_instruct_unk_ara:professional_law|0|0 - extended|global_mmlu_instruct_all_ben:professional_law|0|0 - extended|global_mmlu_instruct_ca_ben:professional_law|0|0 - extended|global_mmlu_instruct_cs_ben:professional_law|0|0 - extended|global_mmlu_instruct_unk_ben:professional_law|0|0 - extended|global_mmlu_instruct_all_deu:professional_law|0|0 - extended|global_mmlu_instruct_ca_deu:professional_law|0|0 - extended|global_mmlu_instruct_cs_deu:professional_law|0|0 - extended|global_mmlu_instruct_unk_deu:professional_law|0|0 - extended|global_mmlu_instruct_all_spa:professional_law|0|0 - extended|global_mmlu_instruct_ca_spa:professional_law|0|0 - extended|global_mmlu_instruct_cs_spa:professional_law|0|0 - extended|global_mmlu_instruct_unk_spa:professional_law|0|0 - extended|global_mmlu_instruct_all_eng:professional_law|0|0 - extended|global_mmlu_instruct_ca_eng:professional_law|0|0 - extended|global_mmlu_instruct_cs_eng:professional_law|0|0 - extended|global_mmlu_instruct_unk_eng:professional_law|0|0 - extended|global_mmlu_instruct_all_fra:professional_law|0|0 - extended|global_mmlu_instruct_ca_fra:professional_law|0|0 - extended|global_mmlu_instruct_cs_fra:professional_law|0|0 - extended|global_mmlu_instruct_unk_fra:professional_law|0|0 - extended|global_mmlu_instruct_all_hin:professional_law|0|0 - extended|global_mmlu_instruct_ca_hin:professional_law|0|0 - extended|global_mmlu_instruct_cs_hin:professional_law|0|0 - extended|global_mmlu_instruct_unk_hin:professional_law|0|0 - extended|global_mmlu_instruct_all_ind:professional_law|0|0 - extended|global_mmlu_instruct_ca_ind:professional_law|0|0 - extended|global_mmlu_instruct_cs_ind:professional_law|0|0 - extended|global_mmlu_instruct_unk_ind:professional_law|0|0 - extended|global_mmlu_instruct_all_ita:professional_law|0|0 - extended|global_mmlu_instruct_ca_ita:professional_law|0|0 - extended|global_mmlu_instruct_cs_ita:professional_law|0|0 - extended|global_mmlu_instruct_unk_ita:professional_law|0|0 - extended|global_mmlu_instruct_all_jpn:professional_law|0|0 - extended|global_mmlu_instruct_ca_jpn:professional_law|0|0 - extended|global_mmlu_instruct_cs_jpn:professional_law|0|0 - extended|global_mmlu_instruct_unk_jpn:professional_law|0|0 - extended|global_mmlu_instruct_all_nld:professional_law|0|0 - extended|global_mmlu_instruct_ca_nld:professional_law|0|0 - extended|global_mmlu_instruct_cs_nld:professional_law|0|0 - extended|global_mmlu_instruct_unk_nld:professional_law|0|0 - extended|global_mmlu_instruct_all_nor:professional_law|0|0 - extended|global_mmlu_instruct_ca_nor:professional_law|0|0 - extended|global_mmlu_instruct_cs_nor:professional_law|0|0 - extended|global_mmlu_instruct_unk_nor:professional_law|0|0 - extended|global_mmlu_instruct_all_pol:professional_law|0|0 - extended|global_mmlu_instruct_ca_pol:professional_law|0|0 - extended|global_mmlu_instruct_cs_pol:professional_law|0|0 - extended|global_mmlu_instruct_unk_pol:professional_law|0|0 - extended|global_mmlu_instruct_all_por:professional_law|0|0 - extended|global_mmlu_instruct_ca_por:professional_law|0|0 - extended|global_mmlu_instruct_cs_por:professional_law|0|0 - extended|global_mmlu_instruct_unk_por:professional_law|0|0 - extended|global_mmlu_instruct_all_rus:professional_law|0|0 - extended|global_mmlu_instruct_ca_rus:professional_law|0|0 - extended|global_mmlu_instruct_cs_rus:professional_law|0|0 - extended|global_mmlu_instruct_unk_rus:professional_law|0|0 - extended|global_mmlu_instruct_all_srp:professional_law|0|0 - extended|global_mmlu_instruct_ca_srp:professional_law|0|0 - extended|global_mmlu_instruct_cs_srp:professional_law|0|0 - extended|global_mmlu_instruct_unk_srp:professional_law|0|0 - extended|global_mmlu_instruct_all_swe:professional_law|0|0 - extended|global_mmlu_instruct_ca_swe:professional_law|0|0 - extended|global_mmlu_instruct_cs_swe:professional_law|0|0 - extended|global_mmlu_instruct_unk_swe:professional_law|0|0 - extended|global_mmlu_instruct_all_swa:professional_law|0|0 - extended|global_mmlu_instruct_ca_swa:professional_law|0|0 - extended|global_mmlu_instruct_cs_swa:professional_law|0|0 - extended|global_mmlu_instruct_unk_swa:professional_law|0|0 - extended|global_mmlu_instruct_all_tel:professional_law|0|0 - extended|global_mmlu_instruct_ca_tel:professional_law|0|0 - extended|global_mmlu_instruct_cs_tel:professional_law|0|0 - extended|global_mmlu_instruct_unk_tel:professional_law|0|0 - extended|global_mmlu_instruct_all_tha:professional_law|0|0 - extended|global_mmlu_instruct_ca_tha:professional_law|0|0 - extended|global_mmlu_instruct_cs_tha:professional_law|0|0 - extended|global_mmlu_instruct_unk_tha:professional_law|0|0 - extended|global_mmlu_instruct_all_tur:professional_law|0|0 - extended|global_mmlu_instruct_ca_tur:professional_law|0|0 - extended|global_mmlu_instruct_cs_tur:professional_law|0|0 - extended|global_mmlu_instruct_unk_tur:professional_law|0|0 - extended|global_mmlu_instruct_all_ukr:professional_law|0|0 - extended|global_mmlu_instruct_ca_ukr:professional_law|0|0 - extended|global_mmlu_instruct_cs_ukr:professional_law|0|0 - extended|global_mmlu_instruct_unk_ukr:professional_law|0|0 - extended|global_mmlu_instruct_all_urd:professional_law|0|0 - extended|global_mmlu_instruct_ca_urd:professional_law|0|0 - extended|global_mmlu_instruct_cs_urd:professional_law|0|0 - extended|global_mmlu_instruct_unk_urd:professional_law|0|0 - extended|global_mmlu_instruct_all_vie:professional_law|0|0 - extended|global_mmlu_instruct_ca_vie:professional_law|0|0 - extended|global_mmlu_instruct_cs_vie:professional_law|0|0 - extended|global_mmlu_instruct_unk_vie:professional_law|0|0 - extended|global_mmlu_instruct_all_ara:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_ara:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_ara:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_ara:professional_medicine|0|0 - extended|global_mmlu_instruct_all_ben:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_ben:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_ben:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_ben:professional_medicine|0|0 - extended|global_mmlu_instruct_all_deu:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_deu:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_deu:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_deu:professional_medicine|0|0 - extended|global_mmlu_instruct_all_spa:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_spa:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_spa:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_spa:professional_medicine|0|0 - extended|global_mmlu_instruct_all_eng:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_eng:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_eng:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_eng:professional_medicine|0|0 - extended|global_mmlu_instruct_all_fra:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_fra:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_fra:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_fra:professional_medicine|0|0 - extended|global_mmlu_instruct_all_hin:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_hin:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_hin:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_hin:professional_medicine|0|0 - extended|global_mmlu_instruct_all_ind:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_ind:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_ind:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_ind:professional_medicine|0|0 - extended|global_mmlu_instruct_all_ita:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_ita:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_ita:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_ita:professional_medicine|0|0 - extended|global_mmlu_instruct_all_jpn:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_jpn:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_jpn:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_jpn:professional_medicine|0|0 - extended|global_mmlu_instruct_all_nld:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_nld:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_nld:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_nld:professional_medicine|0|0 - extended|global_mmlu_instruct_all_nor:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_nor:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_nor:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_nor:professional_medicine|0|0 - extended|global_mmlu_instruct_all_pol:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_pol:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_pol:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_pol:professional_medicine|0|0 - extended|global_mmlu_instruct_all_por:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_por:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_por:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_por:professional_medicine|0|0 - extended|global_mmlu_instruct_all_rus:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_rus:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_rus:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_rus:professional_medicine|0|0 - extended|global_mmlu_instruct_all_srp:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_srp:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_srp:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_srp:professional_medicine|0|0 - extended|global_mmlu_instruct_all_swe:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_swe:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_swe:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_swe:professional_medicine|0|0 - extended|global_mmlu_instruct_all_swa:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_swa:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_swa:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_swa:professional_medicine|0|0 - extended|global_mmlu_instruct_all_tel:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_tel:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_tel:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_tel:professional_medicine|0|0 - extended|global_mmlu_instruct_all_tha:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_tha:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_tha:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_tha:professional_medicine|0|0 - extended|global_mmlu_instruct_all_tur:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_tur:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_tur:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_tur:professional_medicine|0|0 - extended|global_mmlu_instruct_all_ukr:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_ukr:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_ukr:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_ukr:professional_medicine|0|0 - extended|global_mmlu_instruct_all_urd:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_urd:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_urd:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_urd:professional_medicine|0|0 - extended|global_mmlu_instruct_all_vie:professional_medicine|0|0 - extended|global_mmlu_instruct_ca_vie:professional_medicine|0|0 - extended|global_mmlu_instruct_cs_vie:professional_medicine|0|0 - extended|global_mmlu_instruct_unk_vie:professional_medicine|0|0 - extended|global_mmlu_instruct_all_ara:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_ara:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_ara:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_ara:professional_psychology|0|0 - extended|global_mmlu_instruct_all_ben:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_ben:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_ben:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_ben:professional_psychology|0|0 - extended|global_mmlu_instruct_all_deu:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_deu:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_deu:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_deu:professional_psychology|0|0 - extended|global_mmlu_instruct_all_spa:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_spa:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_spa:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_spa:professional_psychology|0|0 - extended|global_mmlu_instruct_all_eng:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_eng:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_eng:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_eng:professional_psychology|0|0 - extended|global_mmlu_instruct_all_fra:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_fra:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_fra:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_fra:professional_psychology|0|0 - extended|global_mmlu_instruct_all_hin:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_hin:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_hin:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_hin:professional_psychology|0|0 - extended|global_mmlu_instruct_all_ind:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_ind:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_ind:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_ind:professional_psychology|0|0 - extended|global_mmlu_instruct_all_ita:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_ita:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_ita:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_ita:professional_psychology|0|0 - extended|global_mmlu_instruct_all_jpn:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_jpn:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_jpn:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_jpn:professional_psychology|0|0 - extended|global_mmlu_instruct_all_nld:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_nld:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_nld:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_nld:professional_psychology|0|0 - extended|global_mmlu_instruct_all_nor:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_nor:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_nor:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_nor:professional_psychology|0|0 - extended|global_mmlu_instruct_all_pol:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_pol:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_pol:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_pol:professional_psychology|0|0 - extended|global_mmlu_instruct_all_por:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_por:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_por:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_por:professional_psychology|0|0 - extended|global_mmlu_instruct_all_rus:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_rus:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_rus:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_rus:professional_psychology|0|0 - extended|global_mmlu_instruct_all_srp:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_srp:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_srp:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_srp:professional_psychology|0|0 - extended|global_mmlu_instruct_all_swe:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_swe:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_swe:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_swe:professional_psychology|0|0 - extended|global_mmlu_instruct_all_swa:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_swa:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_swa:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_swa:professional_psychology|0|0 - extended|global_mmlu_instruct_all_tel:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_tel:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_tel:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_tel:professional_psychology|0|0 - extended|global_mmlu_instruct_all_tha:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_tha:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_tha:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_tha:professional_psychology|0|0 - extended|global_mmlu_instruct_all_tur:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_tur:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_tur:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_tur:professional_psychology|0|0 - extended|global_mmlu_instruct_all_ukr:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_ukr:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_ukr:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_ukr:professional_psychology|0|0 - extended|global_mmlu_instruct_all_urd:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_urd:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_urd:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_urd:professional_psychology|0|0 - extended|global_mmlu_instruct_all_vie:professional_psychology|0|0 - extended|global_mmlu_instruct_ca_vie:professional_psychology|0|0 - extended|global_mmlu_instruct_cs_vie:professional_psychology|0|0 - extended|global_mmlu_instruct_unk_vie:professional_psychology|0|0 - extended|global_mmlu_instruct_all_ara:public_relations|0|0 - extended|global_mmlu_instruct_ca_ara:public_relations|0|0 - extended|global_mmlu_instruct_cs_ara:public_relations|0|0 - extended|global_mmlu_instruct_unk_ara:public_relations|0|0 - extended|global_mmlu_instruct_all_ben:public_relations|0|0 - extended|global_mmlu_instruct_ca_ben:public_relations|0|0 - extended|global_mmlu_instruct_cs_ben:public_relations|0|0 - extended|global_mmlu_instruct_unk_ben:public_relations|0|0 - extended|global_mmlu_instruct_all_deu:public_relations|0|0 - extended|global_mmlu_instruct_ca_deu:public_relations|0|0 - extended|global_mmlu_instruct_cs_deu:public_relations|0|0 - extended|global_mmlu_instruct_unk_deu:public_relations|0|0 - extended|global_mmlu_instruct_all_spa:public_relations|0|0 - extended|global_mmlu_instruct_ca_spa:public_relations|0|0 - extended|global_mmlu_instruct_cs_spa:public_relations|0|0 - extended|global_mmlu_instruct_unk_spa:public_relations|0|0 - extended|global_mmlu_instruct_all_eng:public_relations|0|0 - extended|global_mmlu_instruct_ca_eng:public_relations|0|0 - extended|global_mmlu_instruct_cs_eng:public_relations|0|0 - extended|global_mmlu_instruct_unk_eng:public_relations|0|0 - extended|global_mmlu_instruct_all_fra:public_relations|0|0 - extended|global_mmlu_instruct_ca_fra:public_relations|0|0 - extended|global_mmlu_instruct_cs_fra:public_relations|0|0 - extended|global_mmlu_instruct_unk_fra:public_relations|0|0 - extended|global_mmlu_instruct_all_hin:public_relations|0|0 - extended|global_mmlu_instruct_ca_hin:public_relations|0|0 - extended|global_mmlu_instruct_cs_hin:public_relations|0|0 - extended|global_mmlu_instruct_unk_hin:public_relations|0|0 - extended|global_mmlu_instruct_all_ind:public_relations|0|0 - extended|global_mmlu_instruct_ca_ind:public_relations|0|0 - extended|global_mmlu_instruct_cs_ind:public_relations|0|0 - extended|global_mmlu_instruct_unk_ind:public_relations|0|0 - extended|global_mmlu_instruct_all_ita:public_relations|0|0 - extended|global_mmlu_instruct_ca_ita:public_relations|0|0 - extended|global_mmlu_instruct_cs_ita:public_relations|0|0 - extended|global_mmlu_instruct_unk_ita:public_relations|0|0 - extended|global_mmlu_instruct_all_jpn:public_relations|0|0 - extended|global_mmlu_instruct_ca_jpn:public_relations|0|0 - extended|global_mmlu_instruct_cs_jpn:public_relations|0|0 - extended|global_mmlu_instruct_unk_jpn:public_relations|0|0 - extended|global_mmlu_instruct_all_nld:public_relations|0|0 - extended|global_mmlu_instruct_ca_nld:public_relations|0|0 - extended|global_mmlu_instruct_cs_nld:public_relations|0|0 - extended|global_mmlu_instruct_unk_nld:public_relations|0|0 - extended|global_mmlu_instruct_all_nor:public_relations|0|0 - extended|global_mmlu_instruct_ca_nor:public_relations|0|0 - extended|global_mmlu_instruct_cs_nor:public_relations|0|0 - extended|global_mmlu_instruct_unk_nor:public_relations|0|0 - extended|global_mmlu_instruct_all_pol:public_relations|0|0 - extended|global_mmlu_instruct_ca_pol:public_relations|0|0 - extended|global_mmlu_instruct_cs_pol:public_relations|0|0 - extended|global_mmlu_instruct_unk_pol:public_relations|0|0 - extended|global_mmlu_instruct_all_por:public_relations|0|0 - extended|global_mmlu_instruct_ca_por:public_relations|0|0 - extended|global_mmlu_instruct_cs_por:public_relations|0|0 - extended|global_mmlu_instruct_unk_por:public_relations|0|0 - extended|global_mmlu_instruct_all_rus:public_relations|0|0 - extended|global_mmlu_instruct_ca_rus:public_relations|0|0 - extended|global_mmlu_instruct_cs_rus:public_relations|0|0 - extended|global_mmlu_instruct_unk_rus:public_relations|0|0 - extended|global_mmlu_instruct_all_srp:public_relations|0|0 - extended|global_mmlu_instruct_ca_srp:public_relations|0|0 - extended|global_mmlu_instruct_cs_srp:public_relations|0|0 - extended|global_mmlu_instruct_unk_srp:public_relations|0|0 - extended|global_mmlu_instruct_all_swe:public_relations|0|0 - extended|global_mmlu_instruct_ca_swe:public_relations|0|0 - extended|global_mmlu_instruct_cs_swe:public_relations|0|0 - extended|global_mmlu_instruct_unk_swe:public_relations|0|0 - extended|global_mmlu_instruct_all_swa:public_relations|0|0 - extended|global_mmlu_instruct_ca_swa:public_relations|0|0 - extended|global_mmlu_instruct_cs_swa:public_relations|0|0 - extended|global_mmlu_instruct_unk_swa:public_relations|0|0 - extended|global_mmlu_instruct_all_tel:public_relations|0|0 - extended|global_mmlu_instruct_ca_tel:public_relations|0|0 - extended|global_mmlu_instruct_cs_tel:public_relations|0|0 - extended|global_mmlu_instruct_unk_tel:public_relations|0|0 - extended|global_mmlu_instruct_all_tha:public_relations|0|0 - extended|global_mmlu_instruct_ca_tha:public_relations|0|0 - extended|global_mmlu_instruct_cs_tha:public_relations|0|0 - extended|global_mmlu_instruct_unk_tha:public_relations|0|0 - extended|global_mmlu_instruct_all_tur:public_relations|0|0 - extended|global_mmlu_instruct_ca_tur:public_relations|0|0 - extended|global_mmlu_instruct_cs_tur:public_relations|0|0 - extended|global_mmlu_instruct_unk_tur:public_relations|0|0 - extended|global_mmlu_instruct_all_ukr:public_relations|0|0 - extended|global_mmlu_instruct_ca_ukr:public_relations|0|0 - extended|global_mmlu_instruct_cs_ukr:public_relations|0|0 - extended|global_mmlu_instruct_unk_ukr:public_relations|0|0 - extended|global_mmlu_instruct_all_urd:public_relations|0|0 - extended|global_mmlu_instruct_ca_urd:public_relations|0|0 - extended|global_mmlu_instruct_cs_urd:public_relations|0|0 - extended|global_mmlu_instruct_unk_urd:public_relations|0|0 - extended|global_mmlu_instruct_all_vie:public_relations|0|0 - extended|global_mmlu_instruct_ca_vie:public_relations|0|0 - extended|global_mmlu_instruct_cs_vie:public_relations|0|0 - extended|global_mmlu_instruct_unk_vie:public_relations|0|0 - extended|global_mmlu_instruct_all_ara:security_studies|0|0 - extended|global_mmlu_instruct_ca_ara:security_studies|0|0 - extended|global_mmlu_instruct_cs_ara:security_studies|0|0 - extended|global_mmlu_instruct_unk_ara:security_studies|0|0 - extended|global_mmlu_instruct_all_ben:security_studies|0|0 - extended|global_mmlu_instruct_ca_ben:security_studies|0|0 - extended|global_mmlu_instruct_cs_ben:security_studies|0|0 - extended|global_mmlu_instruct_unk_ben:security_studies|0|0 - extended|global_mmlu_instruct_all_deu:security_studies|0|0 - extended|global_mmlu_instruct_ca_deu:security_studies|0|0 - extended|global_mmlu_instruct_cs_deu:security_studies|0|0 - extended|global_mmlu_instruct_unk_deu:security_studies|0|0 - extended|global_mmlu_instruct_all_spa:security_studies|0|0 - extended|global_mmlu_instruct_ca_spa:security_studies|0|0 - extended|global_mmlu_instruct_cs_spa:security_studies|0|0 - extended|global_mmlu_instruct_unk_spa:security_studies|0|0 - extended|global_mmlu_instruct_all_eng:security_studies|0|0 - extended|global_mmlu_instruct_ca_eng:security_studies|0|0 - extended|global_mmlu_instruct_cs_eng:security_studies|0|0 - extended|global_mmlu_instruct_unk_eng:security_studies|0|0 - extended|global_mmlu_instruct_all_fra:security_studies|0|0 - extended|global_mmlu_instruct_ca_fra:security_studies|0|0 - extended|global_mmlu_instruct_cs_fra:security_studies|0|0 - extended|global_mmlu_instruct_unk_fra:security_studies|0|0 - extended|global_mmlu_instruct_all_hin:security_studies|0|0 - extended|global_mmlu_instruct_ca_hin:security_studies|0|0 - extended|global_mmlu_instruct_cs_hin:security_studies|0|0 - extended|global_mmlu_instruct_unk_hin:security_studies|0|0 - extended|global_mmlu_instruct_all_ind:security_studies|0|0 - extended|global_mmlu_instruct_ca_ind:security_studies|0|0 - extended|global_mmlu_instruct_cs_ind:security_studies|0|0 - extended|global_mmlu_instruct_unk_ind:security_studies|0|0 - extended|global_mmlu_instruct_all_ita:security_studies|0|0 - extended|global_mmlu_instruct_ca_ita:security_studies|0|0 - extended|global_mmlu_instruct_cs_ita:security_studies|0|0 - extended|global_mmlu_instruct_unk_ita:security_studies|0|0 - extended|global_mmlu_instruct_all_jpn:security_studies|0|0 - extended|global_mmlu_instruct_ca_jpn:security_studies|0|0 - extended|global_mmlu_instruct_cs_jpn:security_studies|0|0 - extended|global_mmlu_instruct_unk_jpn:security_studies|0|0 - extended|global_mmlu_instruct_all_nld:security_studies|0|0 - extended|global_mmlu_instruct_ca_nld:security_studies|0|0 - extended|global_mmlu_instruct_cs_nld:security_studies|0|0 - extended|global_mmlu_instruct_unk_nld:security_studies|0|0 - extended|global_mmlu_instruct_all_nor:security_studies|0|0 - extended|global_mmlu_instruct_ca_nor:security_studies|0|0 - extended|global_mmlu_instruct_cs_nor:security_studies|0|0 - extended|global_mmlu_instruct_unk_nor:security_studies|0|0 - extended|global_mmlu_instruct_all_pol:security_studies|0|0 - extended|global_mmlu_instruct_ca_pol:security_studies|0|0 - extended|global_mmlu_instruct_cs_pol:security_studies|0|0 - extended|global_mmlu_instruct_unk_pol:security_studies|0|0 - extended|global_mmlu_instruct_all_por:security_studies|0|0 - extended|global_mmlu_instruct_ca_por:security_studies|0|0 - extended|global_mmlu_instruct_cs_por:security_studies|0|0 - extended|global_mmlu_instruct_unk_por:security_studies|0|0 - extended|global_mmlu_instruct_all_rus:security_studies|0|0 - extended|global_mmlu_instruct_ca_rus:security_studies|0|0 - extended|global_mmlu_instruct_cs_rus:security_studies|0|0 - extended|global_mmlu_instruct_unk_rus:security_studies|0|0 - extended|global_mmlu_instruct_all_srp:security_studies|0|0 - extended|global_mmlu_instruct_ca_srp:security_studies|0|0 - extended|global_mmlu_instruct_cs_srp:security_studies|0|0 - extended|global_mmlu_instruct_unk_srp:security_studies|0|0 - extended|global_mmlu_instruct_all_swe:security_studies|0|0 - extended|global_mmlu_instruct_ca_swe:security_studies|0|0 - extended|global_mmlu_instruct_cs_swe:security_studies|0|0 - extended|global_mmlu_instruct_unk_swe:security_studies|0|0 - extended|global_mmlu_instruct_all_swa:security_studies|0|0 - extended|global_mmlu_instruct_ca_swa:security_studies|0|0 - extended|global_mmlu_instruct_cs_swa:security_studies|0|0 - extended|global_mmlu_instruct_unk_swa:security_studies|0|0 - extended|global_mmlu_instruct_all_tel:security_studies|0|0 - extended|global_mmlu_instruct_ca_tel:security_studies|0|0 - extended|global_mmlu_instruct_cs_tel:security_studies|0|0 - extended|global_mmlu_instruct_unk_tel:security_studies|0|0 - extended|global_mmlu_instruct_all_tha:security_studies|0|0 - extended|global_mmlu_instruct_ca_tha:security_studies|0|0 - extended|global_mmlu_instruct_cs_tha:security_studies|0|0 - extended|global_mmlu_instruct_unk_tha:security_studies|0|0 - extended|global_mmlu_instruct_all_tur:security_studies|0|0 - extended|global_mmlu_instruct_ca_tur:security_studies|0|0 - extended|global_mmlu_instruct_cs_tur:security_studies|0|0 - extended|global_mmlu_instruct_unk_tur:security_studies|0|0 - extended|global_mmlu_instruct_all_ukr:security_studies|0|0 - extended|global_mmlu_instruct_ca_ukr:security_studies|0|0 - extended|global_mmlu_instruct_cs_ukr:security_studies|0|0 - extended|global_mmlu_instruct_unk_ukr:security_studies|0|0 - extended|global_mmlu_instruct_all_urd:security_studies|0|0 - extended|global_mmlu_instruct_ca_urd:security_studies|0|0 - extended|global_mmlu_instruct_cs_urd:security_studies|0|0 - extended|global_mmlu_instruct_unk_urd:security_studies|0|0 - extended|global_mmlu_instruct_all_vie:security_studies|0|0 - extended|global_mmlu_instruct_ca_vie:security_studies|0|0 - extended|global_mmlu_instruct_cs_vie:security_studies|0|0 - extended|global_mmlu_instruct_unk_vie:security_studies|0|0 - extended|global_mmlu_instruct_all_ara:sociology|0|0 - extended|global_mmlu_instruct_ca_ara:sociology|0|0 - extended|global_mmlu_instruct_cs_ara:sociology|0|0 - extended|global_mmlu_instruct_unk_ara:sociology|0|0 - extended|global_mmlu_instruct_all_ben:sociology|0|0 - extended|global_mmlu_instruct_ca_ben:sociology|0|0 - extended|global_mmlu_instruct_cs_ben:sociology|0|0 - extended|global_mmlu_instruct_unk_ben:sociology|0|0 - extended|global_mmlu_instruct_all_deu:sociology|0|0 - extended|global_mmlu_instruct_ca_deu:sociology|0|0 - extended|global_mmlu_instruct_cs_deu:sociology|0|0 - extended|global_mmlu_instruct_unk_deu:sociology|0|0 - extended|global_mmlu_instruct_all_spa:sociology|0|0 - extended|global_mmlu_instruct_ca_spa:sociology|0|0 - extended|global_mmlu_instruct_cs_spa:sociology|0|0 - extended|global_mmlu_instruct_unk_spa:sociology|0|0 - extended|global_mmlu_instruct_all_eng:sociology|0|0 - extended|global_mmlu_instruct_ca_eng:sociology|0|0 - extended|global_mmlu_instruct_cs_eng:sociology|0|0 - extended|global_mmlu_instruct_unk_eng:sociology|0|0 - extended|global_mmlu_instruct_all_fra:sociology|0|0 - extended|global_mmlu_instruct_ca_fra:sociology|0|0 - extended|global_mmlu_instruct_cs_fra:sociology|0|0 - extended|global_mmlu_instruct_unk_fra:sociology|0|0 - extended|global_mmlu_instruct_all_hin:sociology|0|0 - extended|global_mmlu_instruct_ca_hin:sociology|0|0 - extended|global_mmlu_instruct_cs_hin:sociology|0|0 - extended|global_mmlu_instruct_unk_hin:sociology|0|0 - extended|global_mmlu_instruct_all_ind:sociology|0|0 - extended|global_mmlu_instruct_ca_ind:sociology|0|0 - extended|global_mmlu_instruct_cs_ind:sociology|0|0 - extended|global_mmlu_instruct_unk_ind:sociology|0|0 - extended|global_mmlu_instruct_all_ita:sociology|0|0 - extended|global_mmlu_instruct_ca_ita:sociology|0|0 - extended|global_mmlu_instruct_cs_ita:sociology|0|0 - extended|global_mmlu_instruct_unk_ita:sociology|0|0 - extended|global_mmlu_instruct_all_jpn:sociology|0|0 - extended|global_mmlu_instruct_ca_jpn:sociology|0|0 - extended|global_mmlu_instruct_cs_jpn:sociology|0|0 - extended|global_mmlu_instruct_unk_jpn:sociology|0|0 - extended|global_mmlu_instruct_all_nld:sociology|0|0 - extended|global_mmlu_instruct_ca_nld:sociology|0|0 - extended|global_mmlu_instruct_cs_nld:sociology|0|0 - extended|global_mmlu_instruct_unk_nld:sociology|0|0 - extended|global_mmlu_instruct_all_nor:sociology|0|0 - extended|global_mmlu_instruct_ca_nor:sociology|0|0 - extended|global_mmlu_instruct_cs_nor:sociology|0|0 - extended|global_mmlu_instruct_unk_nor:sociology|0|0 - extended|global_mmlu_instruct_all_pol:sociology|0|0 - extended|global_mmlu_instruct_ca_pol:sociology|0|0 - extended|global_mmlu_instruct_cs_pol:sociology|0|0 - extended|global_mmlu_instruct_unk_pol:sociology|0|0 - extended|global_mmlu_instruct_all_por:sociology|0|0 - extended|global_mmlu_instruct_ca_por:sociology|0|0 - extended|global_mmlu_instruct_cs_por:sociology|0|0 - extended|global_mmlu_instruct_unk_por:sociology|0|0 - extended|global_mmlu_instruct_all_rus:sociology|0|0 - extended|global_mmlu_instruct_ca_rus:sociology|0|0 - extended|global_mmlu_instruct_cs_rus:sociology|0|0 - extended|global_mmlu_instruct_unk_rus:sociology|0|0 - extended|global_mmlu_instruct_all_srp:sociology|0|0 - extended|global_mmlu_instruct_ca_srp:sociology|0|0 - extended|global_mmlu_instruct_cs_srp:sociology|0|0 - extended|global_mmlu_instruct_unk_srp:sociology|0|0 - extended|global_mmlu_instruct_all_swe:sociology|0|0 - extended|global_mmlu_instruct_ca_swe:sociology|0|0 - extended|global_mmlu_instruct_cs_swe:sociology|0|0 - extended|global_mmlu_instruct_unk_swe:sociology|0|0 - extended|global_mmlu_instruct_all_swa:sociology|0|0 - extended|global_mmlu_instruct_ca_swa:sociology|0|0 - extended|global_mmlu_instruct_cs_swa:sociology|0|0 - extended|global_mmlu_instruct_unk_swa:sociology|0|0 - extended|global_mmlu_instruct_all_tel:sociology|0|0 - extended|global_mmlu_instruct_ca_tel:sociology|0|0 - extended|global_mmlu_instruct_cs_tel:sociology|0|0 - extended|global_mmlu_instruct_unk_tel:sociology|0|0 - extended|global_mmlu_instruct_all_tha:sociology|0|0 - extended|global_mmlu_instruct_ca_tha:sociology|0|0 - extended|global_mmlu_instruct_cs_tha:sociology|0|0 - extended|global_mmlu_instruct_unk_tha:sociology|0|0 - extended|global_mmlu_instruct_all_tur:sociology|0|0 - extended|global_mmlu_instruct_ca_tur:sociology|0|0 - extended|global_mmlu_instruct_cs_tur:sociology|0|0 - extended|global_mmlu_instruct_unk_tur:sociology|0|0 - extended|global_mmlu_instruct_all_ukr:sociology|0|0 - extended|global_mmlu_instruct_ca_ukr:sociology|0|0 - extended|global_mmlu_instruct_cs_ukr:sociology|0|0 - extended|global_mmlu_instruct_unk_ukr:sociology|0|0 - extended|global_mmlu_instruct_all_urd:sociology|0|0 - extended|global_mmlu_instruct_ca_urd:sociology|0|0 - extended|global_mmlu_instruct_cs_urd:sociology|0|0 - extended|global_mmlu_instruct_unk_urd:sociology|0|0 - extended|global_mmlu_instruct_all_vie:sociology|0|0 - extended|global_mmlu_instruct_ca_vie:sociology|0|0 - extended|global_mmlu_instruct_cs_vie:sociology|0|0 - extended|global_mmlu_instruct_unk_vie:sociology|0|0 - extended|global_mmlu_instruct_all_ara:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_ara:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_ara:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_ara:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_ben:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_ben:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_ben:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_ben:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_deu:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_deu:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_deu:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_deu:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_spa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_spa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_spa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_spa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_eng:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_eng:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_eng:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_eng:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_fra:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_fra:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_fra:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_fra:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_hin:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_hin:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_hin:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_hin:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_ind:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_ind:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_ind:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_ind:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_ita:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_ita:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_ita:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_ita:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_jpn:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_jpn:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_jpn:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_jpn:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_nld:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_nld:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_nld:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_nld:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_nor:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_nor:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_nor:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_nor:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_pol:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_pol:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_pol:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_pol:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_por:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_por:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_por:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_por:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_rus:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_rus:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_rus:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_rus:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_srp:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_srp:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_srp:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_srp:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_swe:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_swe:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_swe:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_swe:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_swa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_swa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_swa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_swa:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_tel:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_tel:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_tel:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_tel:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_tha:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_tha:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_tha:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_tha:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_tur:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_tur:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_tur:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_tur:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_ukr:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_ukr:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_ukr:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_ukr:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_urd:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_urd:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_urd:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_urd:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_vie:us_foreign_policy|0|0 - extended|global_mmlu_instruct_ca_vie:us_foreign_policy|0|0 - extended|global_mmlu_instruct_cs_vie:us_foreign_policy|0|0 - extended|global_mmlu_instruct_unk_vie:us_foreign_policy|0|0 - extended|global_mmlu_instruct_all_ara:virology|0|0 - extended|global_mmlu_instruct_ca_ara:virology|0|0 - extended|global_mmlu_instruct_cs_ara:virology|0|0 - extended|global_mmlu_instruct_unk_ara:virology|0|0 - extended|global_mmlu_instruct_all_ben:virology|0|0 - extended|global_mmlu_instruct_ca_ben:virology|0|0 - extended|global_mmlu_instruct_cs_ben:virology|0|0 - extended|global_mmlu_instruct_unk_ben:virology|0|0 - extended|global_mmlu_instruct_all_deu:virology|0|0 - extended|global_mmlu_instruct_ca_deu:virology|0|0 - extended|global_mmlu_instruct_cs_deu:virology|0|0 - extended|global_mmlu_instruct_unk_deu:virology|0|0 - extended|global_mmlu_instruct_all_spa:virology|0|0 - extended|global_mmlu_instruct_ca_spa:virology|0|0 - extended|global_mmlu_instruct_cs_spa:virology|0|0 - extended|global_mmlu_instruct_unk_spa:virology|0|0 - extended|global_mmlu_instruct_all_eng:virology|0|0 - extended|global_mmlu_instruct_ca_eng:virology|0|0 - extended|global_mmlu_instruct_cs_eng:virology|0|0 - extended|global_mmlu_instruct_unk_eng:virology|0|0 - extended|global_mmlu_instruct_all_fra:virology|0|0 - extended|global_mmlu_instruct_ca_fra:virology|0|0 - extended|global_mmlu_instruct_cs_fra:virology|0|0 - extended|global_mmlu_instruct_unk_fra:virology|0|0 - extended|global_mmlu_instruct_all_hin:virology|0|0 - extended|global_mmlu_instruct_ca_hin:virology|0|0 - extended|global_mmlu_instruct_cs_hin:virology|0|0 - extended|global_mmlu_instruct_unk_hin:virology|0|0 - extended|global_mmlu_instruct_all_ind:virology|0|0 - extended|global_mmlu_instruct_ca_ind:virology|0|0 - extended|global_mmlu_instruct_cs_ind:virology|0|0 - extended|global_mmlu_instruct_unk_ind:virology|0|0 - extended|global_mmlu_instruct_all_ita:virology|0|0 - extended|global_mmlu_instruct_ca_ita:virology|0|0 - extended|global_mmlu_instruct_cs_ita:virology|0|0 - extended|global_mmlu_instruct_unk_ita:virology|0|0 - extended|global_mmlu_instruct_all_jpn:virology|0|0 - extended|global_mmlu_instruct_ca_jpn:virology|0|0 - extended|global_mmlu_instruct_cs_jpn:virology|0|0 - extended|global_mmlu_instruct_unk_jpn:virology|0|0 - extended|global_mmlu_instruct_all_nld:virology|0|0 - extended|global_mmlu_instruct_ca_nld:virology|0|0 - extended|global_mmlu_instruct_cs_nld:virology|0|0 - extended|global_mmlu_instruct_unk_nld:virology|0|0 - extended|global_mmlu_instruct_all_nor:virology|0|0 - extended|global_mmlu_instruct_ca_nor:virology|0|0 - extended|global_mmlu_instruct_cs_nor:virology|0|0 - extended|global_mmlu_instruct_unk_nor:virology|0|0 - extended|global_mmlu_instruct_all_pol:virology|0|0 - extended|global_mmlu_instruct_ca_pol:virology|0|0 - extended|global_mmlu_instruct_cs_pol:virology|0|0 - extended|global_mmlu_instruct_unk_pol:virology|0|0 - extended|global_mmlu_instruct_all_por:virology|0|0 - extended|global_mmlu_instruct_ca_por:virology|0|0 - extended|global_mmlu_instruct_cs_por:virology|0|0 - extended|global_mmlu_instruct_unk_por:virology|0|0 - extended|global_mmlu_instruct_all_rus:virology|0|0 - extended|global_mmlu_instruct_ca_rus:virology|0|0 - extended|global_mmlu_instruct_cs_rus:virology|0|0 - extended|global_mmlu_instruct_unk_rus:virology|0|0 - extended|global_mmlu_instruct_all_srp:virology|0|0 - extended|global_mmlu_instruct_ca_srp:virology|0|0 - extended|global_mmlu_instruct_cs_srp:virology|0|0 - extended|global_mmlu_instruct_unk_srp:virology|0|0 - extended|global_mmlu_instruct_all_swe:virology|0|0 - extended|global_mmlu_instruct_ca_swe:virology|0|0 - extended|global_mmlu_instruct_cs_swe:virology|0|0 - extended|global_mmlu_instruct_unk_swe:virology|0|0 - extended|global_mmlu_instruct_all_swa:virology|0|0 - extended|global_mmlu_instruct_ca_swa:virology|0|0 - extended|global_mmlu_instruct_cs_swa:virology|0|0 - extended|global_mmlu_instruct_unk_swa:virology|0|0 - extended|global_mmlu_instruct_all_tel:virology|0|0 - extended|global_mmlu_instruct_ca_tel:virology|0|0 - extended|global_mmlu_instruct_cs_tel:virology|0|0 - extended|global_mmlu_instruct_unk_tel:virology|0|0 - extended|global_mmlu_instruct_all_tha:virology|0|0 - extended|global_mmlu_instruct_ca_tha:virology|0|0 - extended|global_mmlu_instruct_cs_tha:virology|0|0 - extended|global_mmlu_instruct_unk_tha:virology|0|0 - extended|global_mmlu_instruct_all_tur:virology|0|0 - extended|global_mmlu_instruct_ca_tur:virology|0|0 - extended|global_mmlu_instruct_cs_tur:virology|0|0 - extended|global_mmlu_instruct_unk_tur:virology|0|0 - extended|global_mmlu_instruct_all_ukr:virology|0|0 - extended|global_mmlu_instruct_ca_ukr:virology|0|0 - extended|global_mmlu_instruct_cs_ukr:virology|0|0 - extended|global_mmlu_instruct_unk_ukr:virology|0|0 - extended|global_mmlu_instruct_all_urd:virology|0|0 - extended|global_mmlu_instruct_ca_urd:virology|0|0 - extended|global_mmlu_instruct_cs_urd:virology|0|0 - extended|global_mmlu_instruct_unk_urd:virology|0|0 - extended|global_mmlu_instruct_all_vie:virology|0|0 - extended|global_mmlu_instruct_ca_vie:virology|0|0 - extended|global_mmlu_instruct_cs_vie:virology|0|0 - extended|global_mmlu_instruct_unk_vie:virology|0|0 - extended|global_mmlu_instruct_all_ara:world_religions|0|0 - extended|global_mmlu_instruct_ca_ara:world_religions|0|0 - extended|global_mmlu_instruct_cs_ara:world_religions|0|0 - extended|global_mmlu_instruct_unk_ara:world_religions|0|0 - extended|global_mmlu_instruct_all_ben:world_religions|0|0 - extended|global_mmlu_instruct_ca_ben:world_religions|0|0 - extended|global_mmlu_instruct_cs_ben:world_religions|0|0 - extended|global_mmlu_instruct_unk_ben:world_religions|0|0 - extended|global_mmlu_instruct_all_deu:world_religions|0|0 - extended|global_mmlu_instruct_ca_deu:world_religions|0|0 - extended|global_mmlu_instruct_cs_deu:world_religions|0|0 - extended|global_mmlu_instruct_unk_deu:world_religions|0|0 - extended|global_mmlu_instruct_all_spa:world_religions|0|0 - extended|global_mmlu_instruct_ca_spa:world_religions|0|0 - extended|global_mmlu_instruct_cs_spa:world_religions|0|0 - extended|global_mmlu_instruct_unk_spa:world_religions|0|0 - extended|global_mmlu_instruct_all_eng:world_religions|0|0 - extended|global_mmlu_instruct_ca_eng:world_religions|0|0 - extended|global_mmlu_instruct_cs_eng:world_religions|0|0 - extended|global_mmlu_instruct_unk_eng:world_religions|0|0 - extended|global_mmlu_instruct_all_fra:world_religions|0|0 - extended|global_mmlu_instruct_ca_fra:world_religions|0|0 - extended|global_mmlu_instruct_cs_fra:world_religions|0|0 - extended|global_mmlu_instruct_unk_fra:world_religions|0|0 - extended|global_mmlu_instruct_all_hin:world_religions|0|0 - extended|global_mmlu_instruct_ca_hin:world_religions|0|0 - extended|global_mmlu_instruct_cs_hin:world_religions|0|0 - extended|global_mmlu_instruct_unk_hin:world_religions|0|0 - extended|global_mmlu_instruct_all_ind:world_religions|0|0 - extended|global_mmlu_instruct_ca_ind:world_religions|0|0 - extended|global_mmlu_instruct_cs_ind:world_religions|0|0 - extended|global_mmlu_instruct_unk_ind:world_religions|0|0 - extended|global_mmlu_instruct_all_ita:world_religions|0|0 - extended|global_mmlu_instruct_ca_ita:world_religions|0|0 - extended|global_mmlu_instruct_cs_ita:world_religions|0|0 - extended|global_mmlu_instruct_unk_ita:world_religions|0|0 - extended|global_mmlu_instruct_all_jpn:world_religions|0|0 - extended|global_mmlu_instruct_ca_jpn:world_religions|0|0 - extended|global_mmlu_instruct_cs_jpn:world_religions|0|0 - extended|global_mmlu_instruct_unk_jpn:world_religions|0|0 - extended|global_mmlu_instruct_all_nld:world_religions|0|0 - extended|global_mmlu_instruct_ca_nld:world_religions|0|0 - extended|global_mmlu_instruct_cs_nld:world_religions|0|0 - extended|global_mmlu_instruct_unk_nld:world_religions|0|0 - extended|global_mmlu_instruct_all_nor:world_religions|0|0 - extended|global_mmlu_instruct_ca_nor:world_religions|0|0 - extended|global_mmlu_instruct_cs_nor:world_religions|0|0 - extended|global_mmlu_instruct_unk_nor:world_religions|0|0 - extended|global_mmlu_instruct_all_pol:world_religions|0|0 - extended|global_mmlu_instruct_ca_pol:world_religions|0|0 - extended|global_mmlu_instruct_cs_pol:world_religions|0|0 - extended|global_mmlu_instruct_unk_pol:world_religions|0|0 - extended|global_mmlu_instruct_all_por:world_religions|0|0 - extended|global_mmlu_instruct_ca_por:world_religions|0|0 - extended|global_mmlu_instruct_cs_por:world_religions|0|0 - extended|global_mmlu_instruct_unk_por:world_religions|0|0 - extended|global_mmlu_instruct_all_rus:world_religions|0|0 - extended|global_mmlu_instruct_ca_rus:world_religions|0|0 - extended|global_mmlu_instruct_cs_rus:world_religions|0|0 - extended|global_mmlu_instruct_unk_rus:world_religions|0|0 - extended|global_mmlu_instruct_all_srp:world_religions|0|0 - extended|global_mmlu_instruct_ca_srp:world_religions|0|0 - extended|global_mmlu_instruct_cs_srp:world_religions|0|0 - extended|global_mmlu_instruct_unk_srp:world_religions|0|0 - extended|global_mmlu_instruct_all_swe:world_religions|0|0 - extended|global_mmlu_instruct_ca_swe:world_religions|0|0 - extended|global_mmlu_instruct_cs_swe:world_religions|0|0 - extended|global_mmlu_instruct_unk_swe:world_religions|0|0 - extended|global_mmlu_instruct_all_swa:world_religions|0|0 - extended|global_mmlu_instruct_ca_swa:world_religions|0|0 - extended|global_mmlu_instruct_cs_swa:world_religions|0|0 - extended|global_mmlu_instruct_unk_swa:world_religions|0|0 - extended|global_mmlu_instruct_all_tel:world_religions|0|0 - extended|global_mmlu_instruct_ca_tel:world_religions|0|0 - extended|global_mmlu_instruct_cs_tel:world_religions|0|0 - extended|global_mmlu_instruct_unk_tel:world_religions|0|0 - extended|global_mmlu_instruct_all_tha:world_religions|0|0 - extended|global_mmlu_instruct_ca_tha:world_religions|0|0 - extended|global_mmlu_instruct_cs_tha:world_religions|0|0 - extended|global_mmlu_instruct_unk_tha:world_religions|0|0 - extended|global_mmlu_instruct_all_tur:world_religions|0|0 - extended|global_mmlu_instruct_ca_tur:world_religions|0|0 - extended|global_mmlu_instruct_cs_tur:world_religions|0|0 - extended|global_mmlu_instruct_unk_tur:world_religions|0|0 - extended|global_mmlu_instruct_all_ukr:world_religions|0|0 - extended|global_mmlu_instruct_ca_ukr:world_religions|0|0 - extended|global_mmlu_instruct_cs_ukr:world_religions|0|0 - extended|global_mmlu_instruct_unk_ukr:world_religions|0|0 - extended|global_mmlu_instruct_all_urd:world_religions|0|0 - extended|global_mmlu_instruct_ca_urd:world_religions|0|0 - extended|global_mmlu_instruct_cs_urd:world_religions|0|0 - extended|global_mmlu_instruct_unk_urd:world_religions|0|0 - extended|global_mmlu_instruct_all_vie:world_religions|0|0 - extended|global_mmlu_instruct_ca_vie:world_religions|0|0 - extended|global_mmlu_instruct_cs_vie:world_religions|0|0 - extended|global_mmlu_instruct_unk_vie:world_religions|0|0 +extended|belebele_instruct_deu_Latn|0|0 +extended|belebele_instruct_fra_Latn|0|0 +extended|belebele_instruct_ita_Latn|0|0 +extended|belebele_instruct_por_Latn|0|0 +extended|belebele_instruct_spa_Latn|0|0 +extended|global_mmlu_instruct_amh|0|0 +extended|global_mmlu_instruct_ara|0|0 +extended|global_mmlu_instruct_ben|0|0 +extended|global_mmlu_instruct_ces|0|0 +extended|global_mmlu_instruct_deu|0|0 +extended|global_mmlu_instruct_ell|0|0 +extended|global_mmlu_instruct_eng|0|0 +extended|global_mmlu_instruct_spa|0|0 +extended|global_mmlu_instruct_fas|0|0 +extended|global_mmlu_instruct_fra|0|0 +extended|global_mmlu_instruct_hau|0|0 +extended|global_mmlu_instruct_heb|0|0 +extended|global_mmlu_instruct_hin|0|0 +extended|global_mmlu_instruct_ind|0|0 +extended|global_mmlu_instruct_ibo|0|0 +extended|global_mmlu_instruct_ita|0|0 +extended|global_mmlu_instruct_jpn|0|0 +extended|global_mmlu_instruct_kor|0|0 +extended|global_mmlu_instruct_kir|0|0 +extended|global_mmlu_instruct_lit|0|0 +extended|global_mmlu_instruct_mlg|0|0 +extended|global_mmlu_instruct_msa|0|0 +extended|global_mmlu_instruct_nep|0|0 +extended|global_mmlu_instruct_nld|0|0 +extended|global_mmlu_instruct_nor|0|0 +extended|global_mmlu_instruct_pol|0|0 +extended|global_mmlu_instruct_por|0|0 +extended|global_mmlu_instruct_ron|0|0 +extended|global_mmlu_instruct_rus|0|0 +extended|global_mmlu_instruct_sin|0|0 +extended|global_mmlu_instruct_sna|0|0 +extended|global_mmlu_instruct_som|0|0 +extended|global_mmlu_instruct_srp|0|0 +extended|global_mmlu_instruct_swe|0|0 +extended|global_mmlu_instruct_swa|0|0 +extended|global_mmlu_instruct_tel|0|0 +extended|global_mmlu_instruct_tur|0|0 +extended|global_mmlu_instruct_ukr|0|0 +extended|global_mmlu_instruct_vie|0|0 +extended|global_mmlu_instruct_yor|0|0 +extended|global_mmlu_instruct_zho|0|0 +extended|mmlu_pro|0|0 diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py index adbdf05be..c5cd125db 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -20,10 +20,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from functools import partial import numpy as np -from langcodes import standardize_tag from lighteval.metrics.dynamic_metrics import ( IndicesExtractionConfig, @@ -128,67 +126,6 @@ def belebele_prompt(line, task_name: str = None): TASKS_TABLE.extend(BELEBELE_TASKS) -MMLU_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - - class GlobalMMLUPrompt: def __init__(self, lang): self.lang = lang @@ -225,22 +162,13 @@ def prompt(self, line, task_name: str = None): GLOBAL_MMLU_TASKS = [ LightevalTaskConfig( - name=f"global_mmlu_instruct_{sensitivity_label.lower()}_{language.value}:{subset}", + name=f"global_mmlu_instruct_{language.value}", prompt_function=GlobalMMLUPrompt(language.value).prompt, suite=["extended"], hf_repo="CohereForAI/Global-MMLU", - hf_subset=standardize_tag(language.value), + hf_subset=lang, evaluation_splits=("test",), few_shots_split="dev", - hf_filter=partial( - lambda subset, sensitivity_label, x: x["subject"].lower() == subset - and ( - sensitivity_label == "ALL" or sensitivity_label in x["cultural_sensitivity_label"].replace("-", "UNK") - ) - and all(x[f"option_{opt}"] is not None and x[f"option_{opt}"].strip() for opt in "abcd"), - subset, - sensitivity_label, - ), metric=[ SampleLevelMetric( metric_name="pass@1:1_samples", @@ -263,41 +191,50 @@ def prompt(self, line, task_name: str = None): generation_size=32768, # needed for reasoning models like R1 stop_sequence=[], # no stop sequence, will use eos token ) - for subset in MMLU_SUBSETS - for language in [ - Language.ARABIC, - Language.BENGALI, - Language.GERMAN, - Language.SPANISH, - Language.ENGLISH, - Language.FRENCH, - # Language.HEBREW, - Language.HINDI, - Language.INDONESIAN, - Language.ITALIAN, - Language.JAPANESE, - # Language.KOREAN, - # Language.MALAY, - Language.DUTCH, - Language.NORWEGIAN, - Language.POLISH, - Language.PORTUGUESE, - # Language.ROMANIAN, - Language.RUSSIAN, - Language.SERBIAN, - Language.SWEDISH, - Language.SWAHILI, - # Language.TAMIL, - Language.TELUGU, - Language.THAI, - Language.TURKISH, - Language.UKRAINIAN, - Language.URDU, - Language.VIETNAMESE, - # Language.YORUBA, - # Language.ZULU, missing literals + for lang, language in [ + ("am", Language.AMHARIC), + ("ar", Language.ARABIC), + ("bn", Language.BENGALI), + ("cs", Language.CZECH), + ("de", Language.GERMAN), + ("el", Language.GREEK), + ("en", Language.ENGLISH), + ("es", Language.SPANISH), + ("fa", Language.PERSIAN), + # ("fil", Language.FILIPINO), + ("fr", Language.FRENCH), + ("ha", Language.HAUSA), + ("he", Language.HEBREW), + ("hi", Language.HINDI), + ("id", Language.INDONESIAN), + ("ig", Language.IGBO), + ("it", Language.ITALIAN), + ("ja", Language.JAPANESE), + ("ko", Language.KOREAN), + ("ky", Language.KYRGYZ), + ("lt", Language.LITHUANIAN), + ("mg", Language.MALAGASY), + ("ms", Language.MALAY), + ("ne", Language.NEPALI), + ("nl", Language.DUTCH), + ("ny", Language.NORWEGIAN), + ("pl", Language.POLISH), + ("pt", Language.PORTUGUESE), + ("ro", Language.ROMANIAN), + ("ru", Language.RUSSIAN), + ("si", Language.SINHALA), + ("sn", Language.SHONA), + ("so", Language.SOMALI), + ("sr", Language.SERBIAN), + ("sv", Language.SWEDISH), + ("sw", Language.SWAHILI), + ("te", Language.TELUGU), + ("tr", Language.TURKISH), + ("uk", Language.UKRAINIAN), + ("vi", Language.VIETNAMESE), + ("yo", Language.YORUBA), + ("zh", Language.CHINESE), ] - for sensitivity_label in ["ALL", "CA", "CS", "UNK"] ] TASKS_TABLE.extend(GLOBAL_MMLU_TASKS) @@ -353,3 +290,4 @@ def mmlu_pro(line, task_name: str = None): ) TASKS_TABLE.append(mmlu_pro) +print(TASKS_TABLE) From 031e09ced1fc754ef4bee603fcbb04d290b30f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Fourrier?= <22726840+clefourrier@users.noreply.github.com> Date: Thu, 26 Jun 2025 08:50:44 +0200 Subject: [PATCH 07/15] Fix GPQA and index extractive metric (#829) * too many false positives with the current gpqa metric extraction, making it more string * fixing whitespace and instruction in prompt * better to have a strict extraction for index extraction in general actually * added comment * fix tests, need to invert condition --- .../metrics/utils/extractive_match_utils.py | 5 +++-- src/lighteval/tasks/default_prompts.py | 15 ++++++++++++--- tests/metrics/test_extractive_match.py | 4 +++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/lighteval/metrics/utils/extractive_match_utils.py b/src/lighteval/metrics/utils/extractive_match_utils.py index b8c529d05..13a7e6281 100644 --- a/src/lighteval/metrics/utils/extractive_match_utils.py +++ b/src/lighteval/metrics/utils/extractive_match_utils.py @@ -85,11 +85,12 @@ class IndicesExtractionConfig: Attributes: prefix_for_extraction (ChoicePrefix): The style to use for extracting choice indices (e.g. A,B,C or 1,2,3) - try_extract_without_anchor (bool): Whether to try extracting indices without requiring specific anchors like "answer:" or "final answer is" + try_extract_without_anchor (bool): Whether to try extracting indices without requiring specific anchors like "answer:" or "final answer is". + Recommended False for indices extraction, as some indices (for example `A` which is also a word) can lead to a lot of false positives. """ prefix_for_extraction: ChoicePrefix - try_extract_without_anchor: bool = True + try_extract_without_anchor: bool = False ExtractionTarget = LatexExtractionConfig | ExprExtractionConfig | IndicesExtractionConfig diff --git a/src/lighteval/tasks/default_prompts.py b/src/lighteval/tasks/default_prompts.py index 786c4a0b1..910d0790b 100644 --- a/src/lighteval/tasks/default_prompts.py +++ b/src/lighteval/tasks/default_prompts.py @@ -899,15 +899,24 @@ def gpqa_instruct(line, task_name: str = None): gold_index = random.randint(0, 3) choices = [line["Incorrect Answer 1"], line["Incorrect Answer 2"], line["Incorrect Answer 3"]] choices.insert(gold_index, line["Correct Answer"]) - query_template = "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}" - query = query_template.format(A=choices[0], B=choices[1], C=choices[2], D=choices[3], Question=line["Question"]) + instruction = "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering." + query_template = "{Instruction}\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}" + query = query_template.format( + # Stripping to avoid accidental extra whitespaces, present in GPQA + A=choices[0].strip(), + B=choices[1].strip(), + C=choices[2].strip(), + D=choices[3].strip(), + Question=line["Question"].strip(), + Instruction=instruction, + ) return Doc( task_name=task_name, query=query, choices=LETTER_INDICES[: len(choices)], gold_index=gold_index, - instruction=query, + instruction=instruction, ) diff --git a/tests/metrics/test_extractive_match.py b/tests/metrics/test_extractive_match.py index c3a12c813..8a3ce9c8a 100644 --- a/tests/metrics/test_extractive_match.py +++ b/tests/metrics/test_extractive_match.py @@ -56,7 +56,9 @@ def compare_strings( elif match_type == "expr": extraction_targets.append(ExprExtractionConfig()) elif match_type == "NativeLetters": - extraction_targets.append(IndicesExtractionConfig(prefix_for_extraction="NativeLetters")) + extraction_targets.append( + IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True) + ) extraction_targets = tuple(extraction_targets) # Convert to tuple From bc5a5641b46a3689f4fcf076395eedf6f5f0cfc2 Mon Sep 17 00:00:00 2001 From: Sampo Pyysalo Date: Thu, 26 Jun 2025 09:50:54 +0300 Subject: [PATCH 08/15] Complete TranslationLiterals for Language.ESTONIAN (#779) Translations provided by Kairit Sirts --- .../tasks/templates/utils/translation_literals.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lighteval/tasks/templates/utils/translation_literals.py b/src/lighteval/tasks/templates/utils/translation_literals.py index efbb2841a..5121ae585 100644 --- a/src/lighteval/tasks/templates/utils/translation_literals.py +++ b/src/lighteval/tasks/templates/utils/translation_literals.py @@ -392,10 +392,19 @@ def __getattribute__(self, name: str) -> str: ), Language.ESPERANTO: TranslationLiterals(language=Language.ESPERANTO), Language.ESTONIAN: TranslationLiterals( - # From https://github.com/EleutherAI/lm-evaluation-harness/blob/0845b588303f1f59af98dd1c5bdbd78a9e75a1e2/lm_eval/tasks/xcopa/utils.py language=Language.ESTONIAN, + question_word="küsimus", + answer="vastus", + confirmation_word="eks", + yes="jah", + no="ei", + also="samuti", cause_word="sest", - effect_word="seetõttu", + effect_word="seega", + or_word="või", + true="õige", + false="vale", + neither="mitte kumbki", ), Language.EWE: TranslationLiterals(language=Language.EWE), Language.FAROESE: TranslationLiterals(language=Language.FAROESE), From bae95440564c17254ac2664cf8f30878a61b1cd6 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 08:01:02 +0000 Subject: [PATCH 09/15] belebele split --- src/lighteval/tasks/extended/misc/instruct.py | 175 +++++++++++++++++- 1 file changed, 171 insertions(+), 4 deletions(-) diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py index c5cd125db..f5ce60350 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -33,6 +33,7 @@ ) from lighteval.tasks.default_prompts import LETTER_INDICES from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.multilingual.tasks import LangCodeLanguage, iso_639_3_ind_to_iso_639_3_macro from lighteval.tasks.requests import Doc from lighteval.utils.language import Language @@ -48,6 +49,11 @@ } +def belebele_prompt_en_instruct(line, task_name: str = None): + line["dialect"] == "eng_Latn" + return belebele_prompt(line, task_name) + + def belebele_prompt(line, task_name: str = None): lang_to_template = { "eng_Latn": "Given the following passage, query, and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of A, B, C, or D. Think step by step before answering.\n\n###\nPassage:\n{Passage}\n###\nQuery:\n{Question}\n###\nChoices:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", @@ -60,7 +66,7 @@ def belebele_prompt(line, task_name: str = None): gold_index = int(line["correct_answer_num"]) - 1 choices = [line["mc_answer1"], line["mc_answer2"], line["mc_answer3"], line["mc_answer4"]] - query_template = lang_to_template.get(line["dialect"], "eng_Latn") + query_template = lang_to_template[line["dialect"]] query = query_template.format( A=choices[0], B=choices[1], @@ -80,9 +86,9 @@ def belebele_prompt(line, task_name: str = None): ) -BELEBELE_TASKS = [ +BELEBELE_TASKS_NATIVE_INSTRUCT = [ LightevalTaskConfig( - name=f"belebele_instruct_{lang}_Latn", + name=f"belebele_native_instruct_{lang}_Latn", prompt_function=belebele_prompt, suite=["extended"], hf_repo="facebook/belebele", @@ -123,7 +129,168 @@ def belebele_prompt(line, task_name: str = None): "spa", ] ] -TASKS_TABLE.extend(BELEBELE_TASKS) + +BELEBELE_TASKS_EN_INSTRUCT = [ + LightevalTaskConfig( + name=f"belebele_en_instruct_{lang}", + prompt_function=belebele_prompt_en_instruct, + suite=["extended"], + hf_repo="facebook/belebele", + hf_subset=f"{lang}_Latn", + evaluation_splits=["test"], + hf_avail_splits=["test"], + few_shots_split=None, + few_shots_select=None, + generation_size=32768, # needed for reasoning models like R1 + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=iso_639_3_ind_to_iso_639_3_macro[LangCodeLanguage.get(lang).to_alpha3()], + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], + stop_sequence=[], # no stop sequence, will use eos token + trust_dataset=True, + version=1, + ) + for lang in [ + "acm_Arab", + "arz_Arab", + "ceb_Latn", + "fin_Latn", + "hin_Deva", + "ita_Latn", + "khm_Khmr", + "lvs_Latn", + "npi_Deva", + "pol_Latn", + "slv_Latn", + "swe_Latn", + # "tso_Latn", + # "xho_Latn", + "afr_Latn", + "asm_Beng", + "ces_Latn", + "fra_Latn", + "hin_Latn", + "jav_Latn", + # "kin_Latn", + "mal_Mlym", + "npi_Latn", + "por_Latn", + # "sna_Latn", + "swh_Latn", + "tur_Latn", + "yor_Latn", + "als_Latn", + "azj_Latn", + "ckb_Arab", + # "fuv_Latn", + "hrv_Latn", + "jpn_Jpan", + "kir_Cyrl", + "mar_Deva", + # "nso_Latn", + "snd_Arab", + "tam_Taml", + "ukr_Cyrl", + "zho_Hans", + "amh_Ethi", + # "bam_Latn", + "dan_Latn", + # "gaz_Latn", + "hun_Latn", + # "kac_Latn", + "kor_Hang", + "mkd_Cyrl", + # "nya_Latn", + "ron_Latn", + "som_Latn", + "tel_Telu", + "urd_Arab", + "zho_Hant", + "apc_Arab", + "ben_Beng", + "deu_Latn", + # "grn_Latn", + "hye_Armn", + "kan_Knda", + "lao_Laoo", + "mlt_Latn", + "ory_Orya", + "rus_Cyrl", + # "sot_Latn", + "tgk_Cyrl", + "urd_Latn", + "zsm_Latn", + "arb_Arab", + "ben_Latn", + "ell_Grek", + "guj_Gujr", + # "ibo_Latn", + "kat_Geor", + # "lin_Latn", + # "mri_Latn", + "pan_Guru", + # "shn_Mymr", + "spa_Latn", + "tgl_Latn", + "uzn_Latn", + # "zul_Latn", + "arb_Latn", + # "bod_Tibt", + "eng_Latn", + # "hat_Latn", + # "ilo_Latn", + "kaz_Cyrl", + "lit_Latn", + "mya_Mymr", + "pbt_Arab", + "sin_Latn", + "srp_Cyrl", + "tha_Thai", + "vie_Latn", + "ars_Arab", + "bul_Cyrl", + "est_Latn", + # "hau_Latn", + "ind_Latn", + # "kea_Latn", + # "lug_Latn", + "nld_Latn", + "pes_Arab", + "sin_Sinh", + # "ssw_Latn", + # "tir_Ethi", + "war_Latn", + "ary_Arab", + "cat_Latn", + "eus_Latn", + "heb_Hebr", + "isl_Latn", + # "khk_Cyrl", + # "luo_Latn", + "nob_Latn", + "plt_Latn", + "slk_Latn", + # "sun_Latn", + # "tsn_Latn", + # "wol_Latn", + ] +] +TASKS_TABLE.extend(BELEBELE_TASKS_NATIVE_INSTRUCT + BELEBELE_TASKS_EN_INSTRUCT) class GlobalMMLUPrompt: From 78a5c238256b74f32058e2402759b00639866001 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 08:45:21 +0000 Subject: [PATCH 10/15] updated prompts/instruction management --- examples/tasks/instruct_multilingual.txt | 102 +++++++++++++++++- src/lighteval/tasks/extended/misc/instruct.py | 12 +-- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/examples/tasks/instruct_multilingual.txt b/examples/tasks/instruct_multilingual.txt index 96f60e42d..85a900ddf 100644 --- a/examples/tasks/instruct_multilingual.txt +++ b/examples/tasks/instruct_multilingual.txt @@ -1,8 +1,100 @@ -extended|belebele_instruct_deu_Latn|0|0 -extended|belebele_instruct_fra_Latn|0|0 -extended|belebele_instruct_ita_Latn|0|0 -extended|belebele_instruct_por_Latn|0|0 -extended|belebele_instruct_spa_Latn|0|0 +extended|belebele_native_instruct_deu_Latn|0|0 +extended|belebele_native_instruct_fra_Latn|0|0 +extended|belebele_native_instruct_ita_Latn|0|0 +extended|belebele_native_instruct_por_Latn|0|0 +extended|belebele_native_instruct_spa_Latn|0|0 +extended|belebele_en_instruct_acm_Arab|0|0 +extended|belebele_en_instruct_arz_Arab|0|0 +extended|belebele_en_instruct_ceb_Latn|0|0 +extended|belebele_en_instruct_fin_Latn|0|0 +extended|belebele_en_instruct_hin_Deva|0|0 +extended|belebele_en_instruct_ita_Latn|0|0 +extended|belebele_en_instruct_khm_Khmr|0|0 +extended|belebele_en_instruct_lvs_Latn|0|0 +extended|belebele_en_instruct_npi_Deva|0|0 +extended|belebele_en_instruct_pol_Latn|0|0 +extended|belebele_en_instruct_slv_Latn|0|0 +extended|belebele_en_instruct_swe_Latn|0|0 +extended|belebele_en_instruct_afr_Latn|0|0 +extended|belebele_en_instruct_asm_Beng|0|0 +extended|belebele_en_instruct_ces_Latn|0|0 +extended|belebele_en_instruct_fra_Latn|0|0 +extended|belebele_en_instruct_hin_Latn|0|0 +extended|belebele_en_instruct_jav_Latn|0|0 +extended|belebele_en_instruct_mal_Mlym|0|0 +extended|belebele_en_instruct_npi_Latn|0|0 +extended|belebele_en_instruct_por_Latn|0|0 +extended|belebele_en_instruct_swh_Latn|0|0 +extended|belebele_en_instruct_tur_Latn|0|0 +extended|belebele_en_instruct_yor_Latn|0|0 +extended|belebele_en_instruct_als_Latn|0|0 +extended|belebele_en_instruct_azj_Latn|0|0 +extended|belebele_en_instruct_ckb_Arab|0|0 +extended|belebele_en_instruct_hrv_Latn|0|0 +extended|belebele_en_instruct_jpn_Jpan|0|0 +extended|belebele_en_instruct_kir_Cyrl|0|0 +extended|belebele_en_instruct_mar_Deva|0|0 +extended|belebele_en_instruct_snd_Arab|0|0 +extended|belebele_en_instruct_tam_Taml|0|0 +extended|belebele_en_instruct_ukr_Cyrl|0|0 +extended|belebele_en_instruct_zho_Hans|0|0 +extended|belebele_en_instruct_amh_Ethi|0|0 +extended|belebele_en_instruct_dan_Latn|0|0 +extended|belebele_en_instruct_hun_Latn|0|0 +extended|belebele_en_instruct_kor_Hang|0|0 +extended|belebele_en_instruct_mkd_Cyrl|0|0 +extended|belebele_en_instruct_ron_Latn|0|0 +extended|belebele_en_instruct_som_Latn|0|0 +extended|belebele_en_instruct_tel_Telu|0|0 +extended|belebele_en_instruct_urd_Arab|0|0 +extended|belebele_en_instruct_zho_Hant|0|0 +extended|belebele_en_instruct_apc_Arab|0|0 +extended|belebele_en_instruct_ben_Beng|0|0 +extended|belebele_en_instruct_deu_Latn|0|0 +extended|belebele_en_instruct_hye_Armn|0|0 +extended|belebele_en_instruct_kan_Knda|0|0 +extended|belebele_en_instruct_lao_Laoo|0|0 +extended|belebele_en_instruct_mlt_Latn|0|0 +extended|belebele_en_instruct_ory_Orya|0|0 +extended|belebele_en_instruct_rus_Cyrl|0|0 +extended|belebele_en_instruct_tgk_Cyrl|0|0 +extended|belebele_en_instruct_urd_Latn|0|0 +extended|belebele_en_instruct_zsm_Latn|0|0 +extended|belebele_en_instruct_arb_Arab|0|0 +extended|belebele_en_instruct_ben_Latn|0|0 +extended|belebele_en_instruct_ell_Grek|0|0 +extended|belebele_en_instruct_guj_Gujr|0|0 +extended|belebele_en_instruct_kat_Geor|0|0 +extended|belebele_en_instruct_pan_Guru|0|0 +extended|belebele_en_instruct_spa_Latn|0|0 +extended|belebele_en_instruct_tgl_Latn|0|0 +extended|belebele_en_instruct_uzn_Latn|0|0 +extended|belebele_en_instruct_arb_Latn|0|0 +extended|belebele_en_instruct_eng_Latn|0|0 +extended|belebele_en_instruct_kaz_Cyrl|0|0 +extended|belebele_en_instruct_lit_Latn|0|0 +extended|belebele_en_instruct_mya_Mymr|0|0 +extended|belebele_en_instruct_pbt_Arab|0|0 +extended|belebele_en_instruct_sin_Latn|0|0 +extended|belebele_en_instruct_srp_Cyrl|0|0 +extended|belebele_en_instruct_tha_Thai|0|0 +extended|belebele_en_instruct_vie_Latn|0|0 +extended|belebele_en_instruct_ars_Arab|0|0 +extended|belebele_en_instruct_bul_Cyrl|0|0 +extended|belebele_en_instruct_est_Latn|0|0 +extended|belebele_en_instruct_ind_Latn|0|0 +extended|belebele_en_instruct_nld_Latn|0|0 +extended|belebele_en_instruct_pes_Arab|0|0 +extended|belebele_en_instruct_sin_Sinh|0|0 +extended|belebele_en_instruct_war_Latn|0|0 +extended|belebele_en_instruct_ary_Arab|0|0 +extended|belebele_en_instruct_cat_Latn|0|0 +extended|belebele_en_instruct_eus_Latn|0|0 +extended|belebele_en_instruct_heb_Hebr|0|0 +extended|belebele_en_instruct_isl_Latn|0|0 +extended|belebele_en_instruct_nob_Latn|0|0 +extended|belebele_en_instruct_plt_Latn|0|0 +extended|belebele_en_instruct_slk_Latn|0|0 extended|global_mmlu_instruct_amh|0|0 extended|global_mmlu_instruct_ara|0|0 extended|global_mmlu_instruct_ben|0|0 diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py index f5ce60350..08a33f5d8 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -50,7 +50,7 @@ def belebele_prompt_en_instruct(line, task_name: str = None): - line["dialect"] == "eng_Latn" + line["dialect"] = "eng_Latn" return belebele_prompt(line, task_name) @@ -75,7 +75,7 @@ def belebele_prompt(line, task_name: str = None): Passage=line["flores_passage"], Question=line["question"], ) - instruction = query_template.split("\n\n###")[0] + instruction = query_template.split("###\n")[0] return Doc( task_name=task_name, @@ -136,7 +136,7 @@ def belebele_prompt(line, task_name: str = None): prompt_function=belebele_prompt_en_instruct, suite=["extended"], hf_repo="facebook/belebele", - hf_subset=f"{lang}_Latn", + hf_subset=lang, evaluation_splits=["test"], hf_avail_splits=["test"], few_shots_split=None, @@ -302,7 +302,7 @@ def __init__(self, lang): "fra": "A partir de la question et des choix de réponses suivants, indiquez la lettre correspondant à la bonne réponse. La dernière ligne de votre réponse doit avoir le format suivant : 'Réponse: '$LETTRE' (sans les guillemets) où LETTRE est l'une des lettres: A, B, C ou D. Réfléchissez étape par étape avant de répondre.\n\n###\nRequête:\n{Question}\n###\nChoix:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", "ita": "Dato il seguente quesito e le diverse opzioni per una risposta, indicare la lettera corrispondente alla risposta corretta. L'ultima riga della risposta deve avere il seguente formato: 'Risposta: $LETTERA' (senza virgolette), e LETTERA è necessariamente una tra A, B, C, D. Prima di rispondere, è importante che si ragioni passo per passo.\n\n###\nQuesito:\n{Question}\n###\nOpzioni:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", "por": "Tendo em conta a seguinte pergunta e opções de resposta, indique a letra correspondente à resposta correta. A última linha da sua resposta deve ter o seguinte formato: 'Resposta: $LETRA' (sem aspas) em que LETRA é uma de A, B, C ou D. Pense passo a passo antes de responder.\n\n###\nPergunta:\n{Question}\n###\nOpções:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - "spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\\###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\n###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", } def prompt(self, line, task_name: str = None): @@ -316,7 +316,7 @@ def prompt(self, line, task_name: str = None): D=choices[3], Question=line["question"], ) - instruction = query_template.split("\n\n###")[0] + instruction = query_template.split("###\n")[0] return Doc( task_name=task_name, @@ -457,4 +457,4 @@ def mmlu_pro(line, task_name: str = None): ) TASKS_TABLE.append(mmlu_pro) -print(TASKS_TABLE) +# print("\n".join([f"extended|{task.name}|0|0" for task in TASKS_TABLE])) From 91ad573780840d6e43d008723a0d95c60dd18e37 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 09:17:43 +0000 Subject: [PATCH 11/15] tmp --- examples/tasks/instruct_multilingual_test.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 examples/tasks/instruct_multilingual_test.txt diff --git a/examples/tasks/instruct_multilingual_test.txt b/examples/tasks/instruct_multilingual_test.txt new file mode 100644 index 000000000..fb1cc47e6 --- /dev/null +++ b/examples/tasks/instruct_multilingual_test.txt @@ -0,0 +1,14 @@ +extended|global_mmlu_instruct_ara|0|0 +extended|global_mmlu_instruct_ben|0|0 +extended|global_mmlu_instruct_deu|0|0 +extended|global_mmlu_instruct_eng|0|0 +extended|global_mmlu_instruct_spa|0|0 +extended|global_mmlu_instruct_fra|0|0 +extended|global_mmlu_instruct_hin|0|0 +extended|global_mmlu_instruct_ind|0|0 +extended|global_mmlu_instruct_ita|0|0 +extended|global_mmlu_instruct_jpn|0|0 +extended|global_mmlu_instruct_kor|0|0 +extended|global_mmlu_instruct_por|0|0 +extended|global_mmlu_instruct_swa|0|0 +extended|global_mmlu_instruct_zho|0|0 From 6f5d92f14539713adbd2d9c4fa10822e63bef58b Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 09:26:10 +0000 Subject: [PATCH 12/15] added enforce eager --- src/lighteval/models/vllm/vllm_model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py index d1bdf7328..256d0219e 100644 --- a/src/lighteval/models/vllm/vllm_model.py +++ b/src/lighteval/models/vllm/vllm_model.py @@ -105,6 +105,7 @@ class VLLMModelConfig(ModelConfig): max_num_batched_tokens: PositiveInt = 2048 # maximum number of tokens per batch subfolder: str | None = None is_async: bool = False # Whether to use the async version or sync version of the model + enforce_eager: bool = False class VLLMModel(LightevalModel): @@ -187,6 +188,7 @@ def _create_auto_model(self, config: VLLMModelConfig) -> Optional[LLM]: "seed": int(config.seed), "max_num_seqs": int(config.max_num_seqs), "max_num_batched_tokens": int(config.max_num_batched_tokens), + "enforce_eager": bool(config.enforce_eager), } if config.quantization is not None: From 2a108268fa7045b1a031c81dca9aec693ccc8dc1 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 10:40:17 +0000 Subject: [PATCH 13/15] small prompt fix --- src/lighteval/tasks/extended/misc/instruct.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/misc/instruct.py index 08a33f5d8..33604f924 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/misc/instruct.py @@ -308,7 +308,8 @@ def __init__(self, lang): def prompt(self, line, task_name: str = None): gold_index = LETTER_INDICES.index(line["answer"]) choices = [line["option_a"], line["option_b"], line["option_c"], line["option_d"]] - query_template = self.lang_to_template.get(self.lang, "eng") + lang = self.lang if self.lang in self.lang_to_template.keys() else "eng" + query_template = self.lang_to_template[lang] query = query_template.format( A=choices[0], B=choices[1], From 2114c6167cd2ad368ba19948148d21414d0f806f Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 13:11:16 +0000 Subject: [PATCH 14/15] one file per eval --- examples/tasks/instruct_multilingual.txt | 41 ++++ examples/tasks/instruct_multilingual_test.txt | 20 +- src/lighteval/tasks/extended/__init__.py | 6 +- .../instruct.py => instruct/belebele.py} | 168 ------------- .../tasks/extended/instruct/global_mmlu.py | 232 ++++++++++++++++++ .../tasks/extended/instruct/mmlu_pro.py | 90 +++++++ 6 files changed, 373 insertions(+), 184 deletions(-) rename src/lighteval/tasks/extended/{misc/instruct.py => instruct/belebele.py} (56%) create mode 100644 src/lighteval/tasks/extended/instruct/global_mmlu.py create mode 100644 src/lighteval/tasks/extended/instruct/mmlu_pro.py diff --git a/examples/tasks/instruct_multilingual.txt b/examples/tasks/instruct_multilingual.txt index 85a900ddf..f5bdcb4c2 100644 --- a/examples/tasks/instruct_multilingual.txt +++ b/examples/tasks/instruct_multilingual.txt @@ -136,4 +136,45 @@ extended|global_mmlu_instruct_ukr|0|0 extended|global_mmlu_instruct_vie|0|0 extended|global_mmlu_instruct_yor|0|0 extended|global_mmlu_instruct_zho|0|0 +extended|global_mmlu_lite_instruct_amh|0|0 +extended|global_mmlu_lite_instruct_ara|0|0 +extended|global_mmlu_lite_instruct_ben|0|0 +extended|global_mmlu_lite_instruct_ces|0|0 +extended|global_mmlu_lite_instruct_deu|0|0 +extended|global_mmlu_lite_instruct_ell|0|0 +extended|global_mmlu_lite_instruct_eng|0|0 +extended|global_mmlu_lite_instruct_spa|0|0 +extended|global_mmlu_lite_instruct_fas|0|0 +extended|global_mmlu_lite_instruct_fra|0|0 +extended|global_mmlu_lite_instruct_hau|0|0 +extended|global_mmlu_lite_instruct_heb|0|0 +extended|global_mmlu_lite_instruct_hin|0|0 +extended|global_mmlu_lite_instruct_ind|0|0 +extended|global_mmlu_lite_instruct_ibo|0|0 +extended|global_mmlu_lite_instruct_ita|0|0 +extended|global_mmlu_lite_instruct_jpn|0|0 +extended|global_mmlu_lite_instruct_kor|0|0 +extended|global_mmlu_lite_instruct_kir|0|0 +extended|global_mmlu_lite_instruct_lit|0|0 +extended|global_mmlu_lite_instruct_mlg|0|0 +extended|global_mmlu_lite_instruct_msa|0|0 +extended|global_mmlu_lite_instruct_nep|0|0 +extended|global_mmlu_lite_instruct_nld|0|0 +extended|global_mmlu_lite_instruct_nor|0|0 +extended|global_mmlu_lite_instruct_pol|0|0 +extended|global_mmlu_lite_instruct_por|0|0 +extended|global_mmlu_lite_instruct_ron|0|0 +extended|global_mmlu_lite_instruct_rus|0|0 +extended|global_mmlu_lite_instruct_sin|0|0 +extended|global_mmlu_lite_instruct_sna|0|0 +extended|global_mmlu_lite_instruct_som|0|0 +extended|global_mmlu_lite_instruct_srp|0|0 +extended|global_mmlu_lite_instruct_swe|0|0 +extended|global_mmlu_lite_instruct_swa|0|0 +extended|global_mmlu_lite_instruct_tel|0|0 +extended|global_mmlu_lite_instruct_tur|0|0 +extended|global_mmlu_lite_instruct_ukr|0|0 +extended|global_mmlu_lite_instruct_vie|0|0 +extended|global_mmlu_lite_instruct_yor|0|0 +extended|global_mmlu_lite_instruct_zho|0|0 extended|mmlu_pro|0|0 diff --git a/examples/tasks/instruct_multilingual_test.txt b/examples/tasks/instruct_multilingual_test.txt index fb1cc47e6..fcfb99a9e 100644 --- a/examples/tasks/instruct_multilingual_test.txt +++ b/examples/tasks/instruct_multilingual_test.txt @@ -1,14 +1,6 @@ -extended|global_mmlu_instruct_ara|0|0 -extended|global_mmlu_instruct_ben|0|0 -extended|global_mmlu_instruct_deu|0|0 -extended|global_mmlu_instruct_eng|0|0 -extended|global_mmlu_instruct_spa|0|0 -extended|global_mmlu_instruct_fra|0|0 -extended|global_mmlu_instruct_hin|0|0 -extended|global_mmlu_instruct_ind|0|0 -extended|global_mmlu_instruct_ita|0|0 -extended|global_mmlu_instruct_jpn|0|0 -extended|global_mmlu_instruct_kor|0|0 -extended|global_mmlu_instruct_por|0|0 -extended|global_mmlu_instruct_swa|0|0 -extended|global_mmlu_instruct_zho|0|0 +extended|global_mmlu_lite_instruct_deu|0|0 +extended|global_mmlu_lite_instruct_eng|0|0 +extended|global_mmlu_lite_instruct_spa|0|0 +extended|global_mmlu_lite_instruct_fra|0|0 +extended|global_mmlu_lite_instruct_ita|0|0 +extended|global_mmlu_lite_instruct_por|0|0 diff --git a/src/lighteval/tasks/extended/__init__.py b/src/lighteval/tasks/extended/__init__.py index b2e44433a..9dde9fb16 100644 --- a/src/lighteval/tasks/extended/__init__.py +++ b/src/lighteval/tasks/extended/__init__.py @@ -20,11 +20,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import lighteval.tasks.extended.misc.instruct as instruct +import lighteval.tasks.extended.instruct.belebele as belebele +import lighteval.tasks.extended.instruct.global_mmlu as global_mmlu +import lighteval.tasks.extended.instruct.mmlu_pro as mmlu_pro from lighteval.utils.imports import can_load_extended_tasks -AVAILABLE_EXTENDED_TASKS_MODULES = [instruct] +AVAILABLE_EXTENDED_TASKS_MODULES = [belebele, mmlu_pro, global_mmlu] if can_load_extended_tasks(): import lighteval.tasks.extended.hle.main as hle diff --git a/src/lighteval/tasks/extended/misc/instruct.py b/src/lighteval/tasks/extended/instruct/belebele.py similarity index 56% rename from src/lighteval/tasks/extended/misc/instruct.py rename to src/lighteval/tasks/extended/instruct/belebele.py index 33604f924..a1ecbc741 100644 --- a/src/lighteval/tasks/extended/misc/instruct.py +++ b/src/lighteval/tasks/extended/instruct/belebele.py @@ -291,171 +291,3 @@ def belebele_prompt(line, task_name: str = None): ] ] TASKS_TABLE.extend(BELEBELE_TASKS_NATIVE_INSTRUCT + BELEBELE_TASKS_EN_INSTRUCT) - - -class GlobalMMLUPrompt: - def __init__(self, lang): - self.lang = lang - self.lang_to_template = { - "eng": "Given the following query and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of A, B, C, or D. Think step by step before answering.\n\n###\nQuery:\n{Question}\n###\nChoices:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - "deu": "Gib basierend auf der folgenden Frage und den Antwortmöglichkeiten den Buchstaben aus, der der richtigen Antwort entspricht. Die letzte Zeile deiner Antwort sollte folgendes Format haben: 'Antwort: $BUCHSTABE' (ohne Anführungszeichen), wobei BUCHSTABE einer der folgenden ist: A, B, C oder D. Denke Schritt für Schritt, bevor du antwortest.\n\n###\nFrage:\n{Question}\n###\nAntwortmöglichkeiten:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - "fra": "A partir de la question et des choix de réponses suivants, indiquez la lettre correspondant à la bonne réponse. La dernière ligne de votre réponse doit avoir le format suivant : 'Réponse: '$LETTRE' (sans les guillemets) où LETTRE est l'une des lettres: A, B, C ou D. Réfléchissez étape par étape avant de répondre.\n\n###\nRequête:\n{Question}\n###\nChoix:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - "ita": "Dato il seguente quesito e le diverse opzioni per una risposta, indicare la lettera corrispondente alla risposta corretta. L'ultima riga della risposta deve avere il seguente formato: 'Risposta: $LETTERA' (senza virgolette), e LETTERA è necessariamente una tra A, B, C, D. Prima di rispondere, è importante che si ragioni passo per passo.\n\n###\nQuesito:\n{Question}\n###\nOpzioni:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - "por": "Tendo em conta a seguinte pergunta e opções de resposta, indique a letra correspondente à resposta correta. A última linha da sua resposta deve ter o seguinte formato: 'Resposta: $LETRA' (sem aspas) em que LETRA é uma de A, B, C ou D. Pense passo a passo antes de responder.\n\n###\nPergunta:\n{Question}\n###\nOpções:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - "spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\n###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", - } - - def prompt(self, line, task_name: str = None): - gold_index = LETTER_INDICES.index(line["answer"]) - choices = [line["option_a"], line["option_b"], line["option_c"], line["option_d"]] - lang = self.lang if self.lang in self.lang_to_template.keys() else "eng" - query_template = self.lang_to_template[lang] - query = query_template.format( - A=choices[0], - B=choices[1], - C=choices[2], - D=choices[3], - Question=line["question"], - ) - instruction = query_template.split("###\n")[0] - - return Doc( - task_name=task_name, - query=query, - choices=LETTER_INDICES[: len(choices)], - gold_index=gold_index, - instruction=instruction, - ) - - -GLOBAL_MMLU_TASKS = [ - LightevalTaskConfig( - name=f"global_mmlu_instruct_{language.value}", - prompt_function=GlobalMMLUPrompt(language.value).prompt, - suite=["extended"], - hf_repo="CohereForAI/Global-MMLU", - hf_subset=lang, - evaluation_splits=("test",), - few_shots_split="dev", - metric=[ - SampleLevelMetric( - metric_name="pass@1:1_samples", - sample_level_fn=PassAtK( - k=1, - n=1, - sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( - language=language, - gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - precision=6, - ).sample_level_fn([ref], [pred], doc), - ).compute, - category=MetricCategory.GENERATIVE_SAMPLING, - use_case=MetricUseCase.REASONING, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - ], - generation_size=32768, # needed for reasoning models like R1 - stop_sequence=[], # no stop sequence, will use eos token - ) - for lang, language in [ - ("am", Language.AMHARIC), - ("ar", Language.ARABIC), - ("bn", Language.BENGALI), - ("cs", Language.CZECH), - ("de", Language.GERMAN), - ("el", Language.GREEK), - ("en", Language.ENGLISH), - ("es", Language.SPANISH), - ("fa", Language.PERSIAN), - # ("fil", Language.FILIPINO), - ("fr", Language.FRENCH), - ("ha", Language.HAUSA), - ("he", Language.HEBREW), - ("hi", Language.HINDI), - ("id", Language.INDONESIAN), - ("ig", Language.IGBO), - ("it", Language.ITALIAN), - ("ja", Language.JAPANESE), - ("ko", Language.KOREAN), - ("ky", Language.KYRGYZ), - ("lt", Language.LITHUANIAN), - ("mg", Language.MALAGASY), - ("ms", Language.MALAY), - ("ne", Language.NEPALI), - ("nl", Language.DUTCH), - ("ny", Language.NORWEGIAN), - ("pl", Language.POLISH), - ("pt", Language.PORTUGUESE), - ("ro", Language.ROMANIAN), - ("ru", Language.RUSSIAN), - ("si", Language.SINHALA), - ("sn", Language.SHONA), - ("so", Language.SOMALI), - ("sr", Language.SERBIAN), - ("sv", Language.SWEDISH), - ("sw", Language.SWAHILI), - ("te", Language.TELUGU), - ("tr", Language.TURKISH), - ("uk", Language.UKRAINIAN), - ("vi", Language.VIETNAMESE), - ("yo", Language.YORUBA), - ("zh", Language.CHINESE), - ] -] -TASKS_TABLE.extend(GLOBAL_MMLU_TASKS) - - -def mmlu_pro(line, task_name: str = None): - num_choices = len(line["options"]) - instruction = f"Given the following question about {line['category']} and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of {' ,'.join(LETTER_INDICES[: num_choices - 1])}, or {LETTER_INDICES[num_choices]}. Think step by step before answering.\n\n" - query = f"{instruction}###\nQuery:\n{line['question']}\n###\nChoices:" - query += "".join([f"\n{key}) {choice}" for key, choice in zip(LETTER_INDICES, line["options"])]) - - return Doc( - task_name=task_name, - query=query, - choices=LETTER_INDICES[:num_choices], - gold_index=line["answer_index"], - instruction=instruction, - ) - - -mmlu_pro = LightevalTaskConfig( - name="mmlu_pro", - suite=["extended"], - prompt_function=mmlu_pro, - hf_repo="TIGER-Lab/MMLU-Pro", - hf_subset="default", - hf_avail_splits=["validation", "test"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select=None, - generation_size=32768, # needed for reasoning models like R1 - stop_sequence=[], # no stop sequence, will use eos token - metric=[ - SampleLevelMetric( - metric_name="pass@1:1_samples", - sample_level_fn=PassAtK( - k=1, - n=1, - sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( - language=Language.ENGLISH, - gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - precision=6, - ).sample_level_fn([ref], [pred], doc), - ).compute, - category=MetricCategory.GENERATIVE_SAMPLING, - use_case=MetricUseCase.REASONING, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - ], - trust_dataset=True, - version=0, -) - -TASKS_TABLE.append(mmlu_pro) -# print("\n".join([f"extended|{task.name}|0|0" for task in TASKS_TABLE])) diff --git a/src/lighteval/tasks/extended/instruct/global_mmlu.py b/src/lighteval/tasks/extended/instruct/global_mmlu.py new file mode 100644 index 000000000..2038854c9 --- /dev/null +++ b/src/lighteval/tasks/extended/instruct/global_mmlu.py @@ -0,0 +1,232 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import numpy as np + +from lighteval.metrics.dynamic_metrics import ( + IndicesExtractionConfig, + multilingual_extractive_match_metric, +) +from lighteval.metrics.metrics import MetricCategory, MetricUseCase, SampleLevelMetric +from lighteval.metrics.metrics_sample import ( + PassAtK, +) +from lighteval.tasks.default_prompts import LETTER_INDICES +from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.requests import Doc +from lighteval.utils.language import Language + + +TASKS_TABLE = [] + + +class GlobalMMLUPrompt: + def __init__(self, lang): + self.lang = lang + self.lang_to_template = { + "eng": "Given the following query and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of A, B, C, or D. Think step by step before answering.\n\n###\nQuery:\n{Question}\n###\nChoices:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "deu": "Gib basierend auf der folgenden Frage und den Antwortmöglichkeiten den Buchstaben aus, der der richtigen Antwort entspricht. Die letzte Zeile deiner Antwort sollte folgendes Format haben: 'Antwort: $BUCHSTABE' (ohne Anführungszeichen), wobei BUCHSTABE einer der folgenden ist: A, B, C oder D. Denke Schritt für Schritt, bevor du antwortest.\n\n###\nFrage:\n{Question}\n###\nAntwortmöglichkeiten:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "fra": "A partir de la question et des choix de réponses suivants, indiquez la lettre correspondant à la bonne réponse. La dernière ligne de votre réponse doit avoir le format suivant : 'Réponse: '$LETTRE' (sans les guillemets) où LETTRE est l'une des lettres: A, B, C ou D. Réfléchissez étape par étape avant de répondre.\n\n###\nRequête:\n{Question}\n###\nChoix:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "ita": "Dato il seguente quesito e le diverse opzioni per una risposta, indicare la lettera corrispondente alla risposta corretta. L'ultima riga della risposta deve avere il seguente formato: 'Risposta: $LETTERA' (senza virgolette), e LETTERA è necessariamente una tra A, B, C, D. Prima di rispondere, è importante che si ragioni passo per passo.\n\n###\nQuesito:\n{Question}\n###\nOpzioni:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "por": "Tendo em conta a seguinte pergunta e opções de resposta, indique a letra correspondente à resposta correta. A última linha da sua resposta deve ter o seguinte formato: 'Resposta: $LETRA' (sem aspas) em que LETRA é uma de A, B, C ou D. Pense passo a passo antes de responder.\n\n###\nPergunta:\n{Question}\n###\nOpções:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + "spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\n###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}", + } + + def prompt(self, line, task_name: str = None): + gold_index = LETTER_INDICES.index(line["answer"]) + choices = [line["option_a"], line["option_b"], line["option_c"], line["option_d"]] + lang = self.lang if self.lang in self.lang_to_template.keys() else "eng" + query_template = self.lang_to_template[lang] + query = query_template.format( + A=choices[0], + B=choices[1], + C=choices[2], + D=choices[3], + Question=line["question"], + ) + instruction = query_template.split("###\n")[0] + + return Doc( + task_name=task_name, + query=query, + choices=LETTER_INDICES[: len(choices)], + gold_index=gold_index, + instruction=instruction, + ) + + +GLOBAL_MMLU_TASKS = [ + LightevalTaskConfig( + name=f"global_mmlu_instruct_{language.value}", + prompt_function=GlobalMMLUPrompt(language.value).prompt, + suite=["extended"], + hf_repo="CohereForAI/Global-MMLU", + hf_subset=lang, + evaluation_splits=("test",), + few_shots_split="dev", + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=language, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], + generation_size=32768, # needed for reasoning models like R1 + stop_sequence=[], # no stop sequence, will use eos token + ) + for lang, language in [ + ("am", Language.AMHARIC), + ("ar", Language.ARABIC), + ("bn", Language.BENGALI), + ("cs", Language.CZECH), + ("de", Language.GERMAN), + ("el", Language.GREEK), + ("en", Language.ENGLISH), + ("es", Language.SPANISH), + ("fa", Language.PERSIAN), + # ("fil", Language.FILIPINO), + ("fr", Language.FRENCH), + ("ha", Language.HAUSA), + ("he", Language.HEBREW), + ("hi", Language.HINDI), + ("id", Language.INDONESIAN), + ("ig", Language.IGBO), + ("it", Language.ITALIAN), + ("ja", Language.JAPANESE), + ("ko", Language.KOREAN), + ("ky", Language.KYRGYZ), + ("lt", Language.LITHUANIAN), + ("mg", Language.MALAGASY), + ("ms", Language.MALAY), + ("ne", Language.NEPALI), + ("nl", Language.DUTCH), + ("ny", Language.NORWEGIAN), + ("pl", Language.POLISH), + ("pt", Language.PORTUGUESE), + ("ro", Language.ROMANIAN), + ("ru", Language.RUSSIAN), + ("si", Language.SINHALA), + ("sn", Language.SHONA), + ("so", Language.SOMALI), + ("sr", Language.SERBIAN), + ("sv", Language.SWEDISH), + ("sw", Language.SWAHILI), + ("te", Language.TELUGU), + ("tr", Language.TURKISH), + ("uk", Language.UKRAINIAN), + ("vi", Language.VIETNAMESE), + ("yo", Language.YORUBA), + ("zh", Language.CHINESE), + ] +] +TASKS_TABLE.extend(GLOBAL_MMLU_TASKS) + +GLOBAL_MMLU_LITE_TASKS = [ + LightevalTaskConfig( + name=f"global_mmlu_lite_instruct_{language.value}", + prompt_function=GlobalMMLUPrompt(language.value).prompt, + suite=["extended"], + hf_repo="CohereForAI/Global-MMLU-Lite", + hf_subset=lang, + evaluation_splits=("test",), + few_shots_split="dev", + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=language, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], + generation_size=32768, # needed for reasoning models like R1 + stop_sequence=[], # no stop sequence, will use eos token + ) + for lang, language in [ + ("am", Language.AMHARIC), + ("ar", Language.ARABIC), + ("bn", Language.BENGALI), + ("cs", Language.CZECH), + ("de", Language.GERMAN), + ("el", Language.GREEK), + ("en", Language.ENGLISH), + ("es", Language.SPANISH), + ("fa", Language.PERSIAN), + # ("fil", Language.FILIPINO), + ("fr", Language.FRENCH), + ("ha", Language.HAUSA), + ("he", Language.HEBREW), + ("hi", Language.HINDI), + ("id", Language.INDONESIAN), + ("ig", Language.IGBO), + ("it", Language.ITALIAN), + ("ja", Language.JAPANESE), + ("ko", Language.KOREAN), + ("ky", Language.KYRGYZ), + ("lt", Language.LITHUANIAN), + ("mg", Language.MALAGASY), + ("ms", Language.MALAY), + ("ne", Language.NEPALI), + ("nl", Language.DUTCH), + ("ny", Language.NORWEGIAN), + ("pl", Language.POLISH), + ("pt", Language.PORTUGUESE), + ("ro", Language.ROMANIAN), + ("ru", Language.RUSSIAN), + ("si", Language.SINHALA), + ("sn", Language.SHONA), + ("so", Language.SOMALI), + ("sr", Language.SERBIAN), + ("sv", Language.SWEDISH), + ("sw", Language.SWAHILI), + ("te", Language.TELUGU), + ("tr", Language.TURKISH), + ("uk", Language.UKRAINIAN), + ("vi", Language.VIETNAMESE), + ("yo", Language.YORUBA), + ("zh", Language.CHINESE), + ] +] +TASKS_TABLE.extend(GLOBAL_MMLU_LITE_TASKS) diff --git a/src/lighteval/tasks/extended/instruct/mmlu_pro.py b/src/lighteval/tasks/extended/instruct/mmlu_pro.py new file mode 100644 index 000000000..55d01cef7 --- /dev/null +++ b/src/lighteval/tasks/extended/instruct/mmlu_pro.py @@ -0,0 +1,90 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import numpy as np + +from lighteval.metrics.dynamic_metrics import ( + IndicesExtractionConfig, + multilingual_extractive_match_metric, +) +from lighteval.metrics.metrics import MetricCategory, MetricUseCase, SampleLevelMetric +from lighteval.metrics.metrics_sample import ( + PassAtK, +) +from lighteval.tasks.default_prompts import LETTER_INDICES +from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.requests import Doc +from lighteval.utils.language import Language + + +def mmlu_pro(line, task_name: str = None): + num_choices = len(line["options"]) + instruction = f"Given the following question about {line['category']} and answer choices, output the letter corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of {' ,'.join(LETTER_INDICES[: num_choices - 1])}, or {LETTER_INDICES[num_choices]}. Think step by step before answering.\n\n" + query = f"{instruction}###\nQuery:\n{line['question']}\n###\nChoices:" + query += "".join([f"\n{key}) {choice}" for key, choice in zip(LETTER_INDICES, line["options"])]) + + return Doc( + task_name=task_name, + query=query, + choices=LETTER_INDICES[:num_choices], + gold_index=line["answer_index"], + instruction=instruction, + ) + + +mmlu_pro = LightevalTaskConfig( + name="mmlu_pro", + suite=["extended"], + prompt_function=mmlu_pro, + hf_repo="TIGER-Lab/MMLU-Pro", + hf_subset="default", + hf_avail_splits=["validation", "test"], + evaluation_splits=["test"], + few_shots_split="validation", + few_shots_select=None, + generation_size=32768, # needed for reasoning models like R1 + stop_sequence=[], # no stop sequence, will use eos token + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=Language.ENGLISH, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], + trust_dataset=True, + version=0, +) + +TASKS_TABLE = [mmlu_pro] From ee6149e9383ea10be0f2921715a0fa5d287abffe Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 26 Jun 2025 13:40:42 +0000 Subject: [PATCH 15/15] adding mgsm --- src/lighteval/tasks/extended/__init__.py | 3 +- src/lighteval/tasks/extended/instruct/mgsm.py | 106 ++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/lighteval/tasks/extended/instruct/mgsm.py diff --git a/src/lighteval/tasks/extended/__init__.py b/src/lighteval/tasks/extended/__init__.py index 9dde9fb16..d9c9dc7c8 100644 --- a/src/lighteval/tasks/extended/__init__.py +++ b/src/lighteval/tasks/extended/__init__.py @@ -22,11 +22,12 @@ import lighteval.tasks.extended.instruct.belebele as belebele import lighteval.tasks.extended.instruct.global_mmlu as global_mmlu +import lighteval.tasks.extended.instruct.mgsm as mgsm import lighteval.tasks.extended.instruct.mmlu_pro as mmlu_pro from lighteval.utils.imports import can_load_extended_tasks -AVAILABLE_EXTENDED_TASKS_MODULES = [belebele, mmlu_pro, global_mmlu] +AVAILABLE_EXTENDED_TASKS_MODULES = [belebele, mmlu_pro, mgsm, global_mmlu] if can_load_extended_tasks(): import lighteval.tasks.extended.hle.main as hle diff --git a/src/lighteval/tasks/extended/instruct/mgsm.py b/src/lighteval/tasks/extended/instruct/mgsm.py new file mode 100644 index 000000000..35e92d8b7 --- /dev/null +++ b/src/lighteval/tasks/extended/instruct/mgsm.py @@ -0,0 +1,106 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import numpy as np + +from lighteval.metrics.dynamic_metrics import ( + IndicesExtractionConfig, + multilingual_extractive_match_metric, +) +from lighteval.metrics.metrics import MetricCategory, MetricUseCase, SampleLevelMetric +from lighteval.metrics.metrics_sample import ( + PassAtK, +) +from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.requests import Doc +from lighteval.utils.language import Language + + +TASKS_TABLE = [] + + +def mgsm_prompt(line, task_name: str = None): + query_template = ( + "Given the following question, output the value corresponding to the correct answer. The last line of your response should be of the following format: 'Answer: $NUMBER' (without quotes) where NUMBER is a number. Think step by step before answering.\n\n###\nQuery:\n{Question}", + ) + + query = query_template.format( + Question=line["question"], + ) + instruction = query_template.split("###\n")[0] + + return Doc( + task_name=task_name, + query=query, + choices=[str(line["answer_number"])], + gold_index=0, + instruction=instruction, + ) + + +MGSM_TASKS = [ + LightevalTaskConfig( + name=f"mgsm_instruct_{language.value}", + prompt_function=mgsm_prompt, + suite=("lighteval",), + hf_repo="juletxara/mgsm", + hf_subset=lang, + evaluation_splits=("test",), + few_shots_split="train", + metric=[ + SampleLevelMetric( + metric_name="pass@1:1_samples", + sample_level_fn=PassAtK( + k=1, + n=1, + sample_scoring_function=lambda pred, ref, doc: multilingual_extractive_match_metric( + language=language, + gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], + precision=6, + ).sample_level_fn([ref], [pred], doc), + ).compute, + category=MetricCategory.GENERATIVE_SAMPLING, + use_case=MetricUseCase.REASONING, + corpus_level_fn=np.mean, + higher_is_better=True, + ) + ], + generation_size=32768, # needed for reasoning models like R1 + stop_sequence=[], # no stop sequence, will use eos token + ) + for lang, language in [ + ("bn", Language.BENGALI), + ("de", Language.GERMAN), + ("en", Language.ENGLISH), + ("es", Language.SPANISH), + ("fr", Language.FRENCH), + ("ja", Language.JAPANESE), + ("ru", Language.RUSSIAN), + ("sw", Language.SWAHILI), + ("te", Language.TELUGU), + ("th", Language.THAI), + ("zh", Language.CHINESE), + ] +] +TASKS_TABLE.extend(MGSM_TASKS)