---

pop_qa

Note

ID: catalog.cards.pop_qa | Type: TaskCard

{
    "loader": {
        "path": "akariasai/PopQA",
        "type": "load_hf"
    },
    "preprocess_steps": [
        {
            "page_size": 14267,
            "type": "shuffle"
        },
        {
            "_argv": [
                "possible_answers"
            ],
            "function": "json.loads",
            "to_field": "possible_answers",
            "type": "apply"
        }
    ],
    "task": {
        "inputs": [
            "question",
            "prop",
            "subj"
        ],
        "metrics": [
            "metrics.accuracy"
        ],
        "outputs": [
            "possible_answers"
        ],
        "type": "form_task"
    },
    "templates": {
        "items": [
            {
                "input_format": "Answer to the following question. There is no need to explain the reasoning at all. Simply state just the answer in few words. No need for full answer. No need to repeat the question or words from the question. The answer text should be partial and contain only {prop}. Do not use full sentence. \nQuestion: {question}\nThe {prop} of {subj} is:",
                "postprocessors": [
                    "processors.take_first_non_empty_line",
                    "processors.lower_case"
                ],
                "references_field": "possible_answers",
                "type": "multi_reference_template"
            }
        ],
        "type": "templates_list"
    },
    "type": "task_card"
}