---

piqa_middle

Note

ID: catalog.cards.piqa_middle | Type: TaskCard

{
    "loader": {
        "name": "middle",
        "path": "race",
        "type": "load_hf"
    },
    "preprocess_steps": [
        {
            "fields": {
                "numbering": [
                    "A",
                    "B",
                    "C",
                    "D",
                    "E",
                    "F",
                    "G",
                    "H",
                    "I",
                    "J",
                    "K",
                    "L",
                    "M",
                    "N",
                    "O",
                    "P",
                    "Q",
                    "R",
                    "S",
                    "T",
                    "U",
                    "V",
                    "W",
                    "X",
                    "Y",
                    "Z"
                ],
                "topic": "reading comprehension"
            },
            "type": "add_fields"
        },
        {
            "index_of": "answer",
            "search_in": "numbering",
            "to_field": "index",
            "type": "index_of"
        },
        {
            "field_to_field": {
                "article": "context",
                "index": "_index",
                "numbering": "_numbering",
                "options": "_options",
                "question": "sentence1",
                "topic": "topic"
            },
            "type": "rename_fields"
        },
        {
            "field": "_numbering",
            "index": "_index",
            "to_field": "number",
            "type": "take_by_field"
        },
        {
            "field": "_options",
            "index": "_index",
            "to_field": "answer",
            "type": "take_by_field"
        },
        {
            "fields": [
                "_numbering",
                "_options"
            ],
            "to_field": "choices",
            "type": "zip_field_values"
        },
        {
            "field": "choices/*",
            "process_every_value": true,
            "separator": ". ",
            "to_field": "choices_list",
            "type": "join_str",
            "use_query": true
        },
        {
            "field": "choices_list",
            "index": "_index",
            "to_field": "number_and_answer",
            "type": "take_by_field"
        },
        {
            "field": "choices/*/0",
            "separator": ",",
            "to_field": "numbers",
            "type": "join_str",
            "use_query": true
        },
        {
            "field": "choices_list",
            "separator": " ",
            "to_field": "choices",
            "type": "join_str"
        },
        {
            "field_to_field": {
                "number": "label"
            },
            "type": "rename_fields"
        }
    ],
    "task": {
        "inputs": [
            "choices",
            "sentence1",
            "numbers",
            "topic",
            "context"
        ],
        "metrics": [
            "metrics.accuracy"
        ],
        "outputs": [
            "label"
        ],
        "type": "form_task"
    },
    "templates": {
        "fm-eval": {
            "input_format": "The following are multiple choice questions (with answers) about {topic}.\n\n\n                            Context: {context}\nQuestion: {context}\n{sentence1}\nChoose from {numbers}\nAnswers: {choices}\nAnswer:",
            "output_format": "{label}",
            "type": "input_output_template"
        },
        "helm": {
            "input_format": "The following are multiple choice questions (with answers) about {topic}.\n\n\n                            Context: {context}\nQuestion: {context}\n{sentence1}.\nAnswers: {choices}.\nAnswer:",
            "output_format": "{label}",
            "type": "input_output_template"
        },
        "lm_eval_harness": {
            "input_format": "Context: {context}\nQuestion: {context}\n{sentence1}.\nChoices:\n{choices}.\nAnswer:",
            "output_format": "{label}",
            "type": "input_output_template"
        },
        "original": {
            "input_format": "The following are multiple choice questions (with answers) about {topic}.\n\n                            {context}\n{sentence1}.\nAnswers: {choices}.\nAnswer:",
            "output_format": "{label}",
            "type": "input_output_template"
        }
    },
    "type": "task_card"
}