unitxt.benchmark module

class unitxt.benchmark.BaseBenchmark(__tags__: Dict[str, str] = {}, data_classification_policy: List[str] = None, caching: bool = None, format: Format = None, num_demos: int = None, system_prompt: SystemPrompt = None, loader_limit: int = None)

Bases: SourceOperator

class unitxt.benchmark.Benchmark(__tags__: ~typing.Dict[str, str] = {}, data_classification_policy: ~typing.List[str] = None, caching: bool = None, format: ~unitxt.formats.Format = None, num_demos: int = None, system_prompt: ~unitxt.system_prompts.SystemPrompt = None, loader_limit: int = None, subsets: ~typing.Dict[str, ~unitxt.standard.StandardRecipe | ~unitxt.benchmark.BaseBenchmark], max_total_samples: int = None, max_samples_per_subset: int = None)

Bases: BaseBenchmark