Index

Symbols

--apply_chat_template command line option --batch_size command line option --cache_dir command line option --chat_template_kwargs command line option --disable_hf_cache command line option --gen_kwargs command line option --limit command line option --log_samples command line option --model command line option --model_args command line option --num_fewshots command line option --output_file_prefix command line option --output_path command line option

--split command line option --tasks command line option --trust_remote_code command line option --verbosity command line option -a command line option -b command line option -L command line option -m command line option -o command line option -s command line option -t command line option -v command line option

A

abstract_factory() (in module unitxt.metrics) abstract_field() (in module unitxt.metrics) abstract_fields() (in module unitxt.dataclass) AbstractCatalog (class in unitxt.artifact) AbstractField (class in unitxt.dataclass) AbstractFieldError Accuracy (class in unitxt.metrics) AccuracyFast (class in unitxt.metrics) acquire() (unitxt.inference.AsyncTokenBucket method) activate() (unitxt.generator_utils.ReusableGenerator method) add_link_to_catalog() (in module unitxt.catalog) add_to_catalog() (in module unitxt.catalog) AddConstant (class in unitxt.operators) AddDemosPool (class in unitxt.standard) AddID (class in unitxt.operators) AddIncrementalId (class in unitxt.operators) additional_info() (in module unitxt.error_utils) AddPrefix (class in unitxt.processors) AggregationReduction (class in unitxt.metrics) all_metrics (unitxt.metrics.SQLExecutionAccuracy attribute) (unitxt.metrics.SQLExecutionLogicAccuracy attribute) (unitxt.metrics.SQLNonExecutionAccuracy attribute) ANLS (class in unitxt.metrics) ANSWER_RELEVANCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) ANSWER_RELEVANCE_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) anthropic_models (unitxt.metrics.LlamaIndexLLMMetric attribute) Apply (class in unitxt.operators) apply_capital_new_line_notation() (in module unitxt.formats) apply_metric() (in module unitxt.test_utils.metrics) apply_operator() (in module unitxt.test_utils.operators) ApplyMetric (class in unitxt.operators) ApplyOperatorsField (class in unitxt.operators) ApplyRandomTemplate (class in unitxt.templates) ApplySingleTemplate (class in unitxt.templates) ApplyStreamOperatorsField (class in unitxt.operators)

ApplyTemplate (class in unitxt.templates) args (unitxt.type_utils.NormalizedType attribute) Artifact (class in unitxt.artifact) ArtifactFetcherMixin (class in unitxt.operators) ArtifactLink (class in unitxt.artifact) ArtifactList (class in unitxt.artifact) artifacts_json_cache() (in module unitxt.utils) as_dataset() (unitxt.dataset.Dataset method) as_remote_metric() (in module unitxt.eval_utils) as_streaming_dataset() (unitxt.dataset.Dataset method) asdict() (in module unitxt.dataclass) AssignDemosToInstance (class in unitxt.splitters) async_get_or_set() (unitxt.text2sql_utils.Cache method) async_memoize() (unitxt.text2sql_utils.Cache method) AsyncTokenBucket (class in unitxt.inference) at_k (unitxt.metrics.RerankRecall attribute) Audio (class in unitxt.types) augmentable_inputs (unitxt.task.Task attribute) augmented_type (unitxt.image_operators.ImageAugmentor attribute) (unitxt.struct_data_operators.DuplicateTableColumns attribute) (unitxt.struct_data_operators.DuplicateTableRows attribute) (unitxt.struct_data_operators.InsertEmptyTableRows attribute) (unitxt.struct_data_operators.MaskColumnsNames attribute) (unitxt.struct_data_operators.ShuffleColumnsNames attribute) (unitxt.struct_data_operators.ShuffleTableColumns attribute) (unitxt.struct_data_operators.ShuffleTableRows attribute) (unitxt.struct_data_operators.TransposeTable attribute) augmented_type() (unitxt.augmentors.TextAugmentor method) Augmentor (class in unitxt.augmentors) AugmentPrefixSuffix (class in unitxt.augmentors) AugmentWhitespace (class in unitxt.augmentors) available_risks (unitxt.metrics.GraniteGuardianBase attribute) average_item_scores() (unitxt.metrics.MetricWithConfidenceInterval static method) averages (unitxt.metrics.F1Fast attribute) AzureOpenAIInferenceEngine (class in unitxt.inference)

B

Balance (class in unitxt.operators) BaseBenchmark (class in unitxt.benchmark) BaseFormat (class in unitxt.formats) BaseFusion (class in unitxt.fusion) batch_size (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) (unitxt.operators.CollateInstances attribute) batched() (in module unitxt.inference) before_process_multi_stream() (unitxt.llm_as_judge.LLMJudge method) (unitxt.llm_as_judge.LLMJudgeDirect method) (unitxt.llm_as_judge.LLMJudgePairwise method)

Benchmark (class in unitxt.benchmark) BertScore (class in unitxt.metrics) BinaryAccuracy (class in unitxt.metrics) BinaryMaxAccuracy (class in unitxt.metrics) BinaryMaxF1 (class in unitxt.metrics) build_stream_routing() (in module unitxt.split_utils) BulkInstanceMetric (class in unitxt.metrics) BytesToString (class in unitxt.operators)

C

Cache (class in unitxt.text2sql_utils) camel_to_snake_case() (in module unitxt.text_utils) Capitalize (class in unitxt.processors) Cast (class in unitxt.operators) cast_to_type() (in module unitxt.settings_utils) CastFields (class in unitxt.operators) Catalog (class in unitxt.catalog) Catalogs (class in unitxt.artifact) CharEditDistance (class in unitxt.metrics) CharEditDistanceAccuracy (class in unitxt.metrics) chat_kwargs_dict (unitxt.formats.HFSystemFormat attribute) (unitxt.inference.HFInferenceEngineBase attribute) ChatAPIFormat (class in unitxt.formats) check_augmentable() (in module unitxt.ui.load_catalog_data) check_operator() (in module unitxt.test_utils.operators) check_operator_exception() (in module unitxt.test_utils.operators) check_scores() (in module unitxt.test_utils.metrics) Chunk (class in unitxt.collections_operators) ci_score_names (unitxt.metrics.NormalizedSacrebleu attribute) (unitxt.metrics.Pearsonr attribute) (unitxt.metrics.SafetyMetric attribute) (unitxt.metrics.Spearmanr attribute) ci_scores (unitxt.metrics.Accuracy attribute) (unitxt.metrics.BinaryAccuracy attribute) (unitxt.metrics.BinaryMaxF1 attribute) (unitxt.metrics.CharEditDistance attribute) (unitxt.metrics.CharEditDistanceAccuracy attribute) (unitxt.metrics.F1Binary attribute) (unitxt.metrics.FinQAEval attribute) (unitxt.metrics.MAP attribute) (unitxt.metrics.Meteor attribute) (unitxt.metrics.MRR attribute) (unitxt.metrics.Rouge attribute) (unitxt.metrics.RougeHF attribute) (unitxt.metrics.SQLExecutionAccuracy attribute) (unitxt.metrics.SQLExecutionLogicAccuracy attribute) (unitxt.metrics.SQLNonExecutionAccuracy attribute) (unitxt.metrics.StringContainmentOld attribute) (unitxt.metrics.StringContainmentRatio attribute) (unitxt.metrics.TokenOverlap attribute) (unitxt.metrics.UnsortedListExactMatch attribute) class_fields() (in module unitxt.dataclass) classes (unitxt.processors.ExtractVerbalJudgementBadGood attribute) (unitxt.processors.ExtractVerbalJudgment attribute) clean_results() (unitxt.llm_as_judge.LLMJudge method) clear() (unitxt.utils.LRUCache method) cli_load_dataset() (in module unitxt.evaluate_cli) CloseTextSampler (class in unitxt.splitters) COHERENCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) CollateInstances (class in unitxt.operators) CollateInstancesByField (class in unitxt.operators) collect_clause() (in module unitxt.text2sql_utils) Collection (class in unitxt.collections) column_indices (unitxt.struct_data_operators.DuplicateTableColumns attribute) column_names (unitxt.loaders.LoadIOB attribute) command line option --apply_chat_template --batch_size --cache_dir --chat_template_kwargs --disable_hf_cache --gen_kwargs --limit --log_samples --model --model_args --num_fewshots --output_file_prefix --output_path --split --tasks --trust_remote_code --verbosity -a -b -L -m -o -s -t -v

compare_dfs_bird_eval_logic() (in module unitxt.text2sql_utils) compare_dfs_ignore_colnames_ordered_rows() (in module unitxt.text2sql_utils) compare_dfs_ignore_colnames_subset() (in module unitxt.text2sql_utils) compare_dfs_ignore_colnames_unordered_rows() (in module unitxt.text2sql_utils) compare_result_dfs() (in module unitxt.text2sql_utils) compare_versions() (in module unitxt.deprecation_utils) COMPLIANCE_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) compute() (unitxt.llm_as_judge.LLMJudgeDirect method) (unitxt.llm_as_judge.LLMJudgePairwise method) (unitxt.metrics.ANLS method) (unitxt.metrics.GlobalMetric method) (unitxt.metrics.LlamaIndexCorrectness method) (unitxt.metrics.Perplexity method) (unitxt.metrics.WebsrcSquadF1 method) compute_global_confidence_intervals() (unitxt.metrics.MetricWithConfidenceInterval method) ComputeExpressionMixin (class in unitxt.operators) CONCISENESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) condition_to_func (unitxt.operators.FilterByCondition attribute) ConfidenceIntervalMixin (class in unitxt.metrics) configure_unitxt_settings() (in module unitxt.evaluate_cli) CONSISTENCY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) Constants (class in unitxt.settings_utils) ConstantSizeSample (class in unitxt.splitters) construct_dict_as_python_lines() (in module unitxt.text_utils) construct_dict_as_yaml_lines() (in module unitxt.text_utils) ConstructTableFromRowsCols (class in unitxt.struct_data_operators) consume_assignment() (in module unitxt.parsing_utils) consume_name() (in module unitxt.parsing_utils) consume_name_val() (in module unitxt.parsing_utils) consume_overwrites() (in module unitxt.parsing_utils) consume_query() (in module unitxt.parsing_utils) consume_term() (in module unitxt.parsing_utils) context() (unitxt.settings_utils.Settings method) context_fields (unitxt.llm_as_judge.LLMJudge attribute) CONTEXT_RELEVANCE_CONTEXT (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) Conversation (class in unitxt.types) CONVERSATIONAL (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) ConversationSerializer (class in unitxt.serializers) convert_tool_call() (unitxt.metrics.ReflectionToolCallingMixin static method) convert_tools_inventory() (unitxt.metrics.ReflectionToolCallingMixin static method) convert_union_type() (in module unitxt.type_utils) ConvertTableColNamesToSequential (class in unitxt.struct_data_operators) ConvertToBoolean (class in unitxt.processors) Copy (class in unitxt.operators) CopyingReusableGenerator (class in unitxt.generator_utils) CORRECTNESS_BASED_ON_GROUND_TRUTH (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) CorrelationMetric (class in unitxt.metrics) count_files_recursively() (in module unitxt.catalog) count_tags() (in module unitxt.catalog) create_dataset() (in module unitxt.api) create_temporary_dict() (in module unitxt.ui.load_catalog_data) CreateCriteriaFromDict (class in unitxt.llm_as_judge_operators) CreateCriteriaFromJson (class in unitxt.llm_as_judge_operators) CreateCriteriaFromString (class in unitxt.llm_as_judge_operators) CreateCriteriaWithOptionsFromDict (class in unitxt.llm_as_judge_operators) CreateCriteriaWithOptionsFromJson (class in unitxt.llm_as_judge_operators) CreateDemosPool (class in unitxt.standard) CreateYesNoCriteriaFromString (class in unitxt.llm_as_judge_operators) CreateYesNoPartiallyCriteriaFromString (class in unitxt.llm_as_judge_operators) credentials (unitxt.inference.OpenAiInferenceEngine attribute) CredentialsOpenAi (class in unitxt.inference) Criteria (class in unitxt.llm_as_judge_constants) CriteriaOption (class in unitxt.llm_as_judge_constants) CriteriaWithOptions (class in unitxt.llm_as_judge_constants) cross_group_reduction (unitxt.metrics.GroupMetric attribute) CrossProviderInferenceEngine (class in unitxt.inference) CustomF1 (class in unitxt.metrics) CustomF1Fuzzy (class in unitxt.metrics)

D

data (unitxt.stream.MultiStream attribute) data_classification_policy (unitxt.inference.OllamaInferenceEngine attribute) (unitxt.inference.OpenAiInferenceEngine attribute) (unitxt.inference.RITSInferenceEngine attribute) (unitxt.inference.TogetherAiInferenceEngine attribute) (unitxt.inference.WMLInferenceEngineBase attribute) (unitxt.loaders.LoadFromIBMCloud attribute) (unitxt.loaders.LoadFromKaggle attribute) (unitxt.loaders.LoadFromSklearn attribute) (unitxt.loaders.TURLColumnTypeAnnotationLoader attribute) (unitxt.metrics.LlamaIndexLLMMetric attribute) (unitxt.metrics.RemoteMetric attribute) data_url_to_image() (in module unitxt.image_operators) DatabaseConnector (class in unitxt.text2sql_utils) Dataclass (class in unitxt.dataclass) DataclassMeta (class in unitxt.dataclass) Dataset (class in unitxt.dataset) DatasetRecipe (class in unitxt.standard) DecodeImage (class in unitxt.image_operators) Deduplicate (class in unitxt.operators) deep_copy() (in module unitxt.utils) default (unitxt.dataclass.Field attribute) default_headers (unitxt.inference.OpenAiInferenceEngine attribute) DefaultListSerializer (class in unitxt.serializers) DefaultPlaceHolder (class in unitxt.operators) DefaultSerializer (class in unitxt.serializers) delete_values() (in module unitxt.dict_utils) DeleteSplits (class in unitxt.stream_operators) DeleteTargetPrefix (class in unitxt.metric_utils) depraction_wrapper() (in module unitxt.deprecation_utils) deprecation() (in module unitxt.deprecation_utils) DeprecationError Detector (class in unitxt.metrics) DeterministicBalancer (class in unitxt.operators)

DialogFieldsData (class in unitxt.templates) DialogPairwiseChoiceTemplate (class in unitxt.templates) DialogSerializer (class in unitxt.serializers) DialogTemplate (class in unitxt.templates) dict_delete() (in module unitxt.dict_utils) dict_diff_string() (in module unitxt.artifact) dict_equal() (in module unitxt.test_utils.metrics) dict_get() (in module unitxt.dict_utils) dict_set() (in module unitxt.dict_utils) DictAsJsonSerializer (class in unitxt.serializers) DictCollection (class in unitxt.collections) Dictify (class in unitxt.collections_operators) DictOfListsToPairs (class in unitxt.processors) DictReduction (class in unitxt.metrics) DictToTuplesList (class in unitxt.collections_operators) DirectCriteriaCatalogEnum (class in unitxt.llm_as_judge_constants) DistributionNotFound DistStub (class in unitxt.utils) DiverseLabelsSampler (class in unitxt.splitters) DivideAllFieldsBy (class in unitxt.operators) Document (class in unitxt.types) Documentation (class in unitxt.error_utils) DocumentSerializer (class in unitxt.serializers) DOMAINS (unitxt.metrics.WebsrcSquadF1 attribute) download_database() (unitxt.text2sql_utils.LocalSQLiteConnector method) DownloadError DownloadOperator (class in unitxt.operators) DumpJson (class in unitxt.struct_data_operators) DuplicateByList (class in unitxt.collections_operators) DuplicateBySubLists (class in unitxt.collections_operators) DuplicateInstances (class in unitxt.operators) DuplicateSplit (class in unitxt.stream_operators) DuplicateTableColumns (class in unitxt.struct_data_operators) DuplicateTableRows (class in unitxt.struct_data_operators) DynamicStream (class in unitxt.stream)

E

eager_failed() (in module unitxt.stream) EFFECTIVENESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) EMAIL_INCLUSIVITY (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) EMPATHY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) empty_predictions_generator() (in module unitxt.metric_utils) EmptyPrediction (class in unitxt.metric_utils) EmptyStreamError EmptySystemPrompt (class in unitxt.system_prompts) enable_explicit_format() (in module unitxt.logging_utils) EncodeImageToString (class in unitxt.image_operators) EncodeLabels (class in unitxt.operators) ENGAGEMENT (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) environment_variable_key_name() (unitxt.settings_utils.Settings method) EnvironmentLocalCatalog (class in unitxt.catalog) error_context() (in module unitxt.error_utils) escape_characters (unitxt.templates.SpanLabelingTemplate attribute) escape_chars() (in module unitxt.templates) eval_forward_ref() (in module unitxt.type_utils) evaluate() (in module unitxt.api) (in module unitxt.eval_utils) EvaluationInput (class in unitxt.metrics) EvaluationResults (class in unitxt.metric_utils) EvaluatorMetadata (class in unitxt.llm_as_judge_constants) EvaluatorNameEnum (class in unitxt.llm_as_judge_constants) EvaluatorTypeEnum (class in unitxt.llm_as_judge_constants) exact_match() (unitxt.metrics.ExactMatchMM static method) ExactMatchMM (class in unitxt.metrics) EXAMPLES_AND_DETAILS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute)

execute_query() (unitxt.text2sql_utils.DatabaseConnector method) (unitxt.text2sql_utils.InMemoryDatabaseConnector method) (unitxt.text2sql_utils.LocalSQLiteConnector method) (unitxt.text2sql_utils.RemoteDatabaseConnector method) execute_query_local() (in module unitxt.text2sql_utils) execute_query_remote() (in module unitxt.text2sql_utils) ExecuteExpression (class in unitxt.operators) Explode (class in unitxt.collections_operators) ExplodeSubLists (class in unitxt.collections_operators) external_api_models (unitxt.metrics.LlamaIndexLLMMetric attribute) external_fields_names() (in module unitxt.dataclass) extract_images() (in module unitxt.image_operators) extract_overwrite_args() (in module unitxt.evaluate_cli) extract_possible_json_str() (in module unitxt.struct_data_operators) extract_scores() (in module unitxt.evaluate_cli) extract_select_columns() (in module unitxt.text2sql_utils) extract_select_info() (in module unitxt.text2sql_utils) extract_sql_from_text() (in module unitxt.text2sql_utils) ExtractArenaHardNumericalJudgment (class in unitxt.processors) ExtractFieldValues (class in unitxt.operators) ExtractHarmRatingJudgement (class in unitxt.processors) ExtractMostCommonFieldValues (class in unitxt.operators) ExtractMtBenchLabelJudgment (class in unitxt.processors) ExtractMtBenchRatingJudgment (class in unitxt.processors) ExtractSafeUnsafeJudgment (class in unitxt.processors) ExtractVerbalJudgementBadGood (class in unitxt.processors) ExtractVerbalJudgment (class in unitxt.processors) ExtractWithRegex (class in unitxt.processors) ExtractZipFile (class in unitxt.operators)

F

F1 (class in unitxt.metrics) F1Binary (class in unitxt.metrics) F1BinaryPosOnly (class in unitxt.metrics) F1Fast (class in unitxt.metrics) F1Macro (class in unitxt.metrics) F1MacroMultiLabel (class in unitxt.metrics) F1Micro (class in unitxt.metrics) F1MicroMultiLabel (class in unitxt.metrics) F1MultiLabel (class in unitxt.metrics) F1Strings (class in unitxt.metrics) F1Weighted (class in unitxt.metrics) FACTUALLY_CONSISTENT (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) FaithfulnessHHEM (class in unitxt.metrics) FaultyStreamError FeatureGroupedShuffle (class in unitxt.operators) fetch() (in module unitxt.dataset_utils) fetch_artifact() (in module unitxt.artifact) Field (class in unitxt.dataclass) field (unitxt.metrics.StringContainmentRatio attribute) FieldOperator (class in unitxt.operators) fields() (in module unitxt.dataclass) fields_names() (in module unitxt.dataclass) fill_metadata() (in module unitxt.api) Fillna (class in unitxt.operators) Filter (class in unitxt.collections_operators) FilterByCondition (class in unitxt.operators) FilterByConditionBasedOnFields (class in unitxt.operators) FilterByExpression (class in unitxt.operators) final_fields() (in module unitxt.dataclass) FinalField (class in unitxt.dataclass) FinalFieldError FinalizeDataset (class in unitxt.schema) FinQAEval (class in unitxt.metrics) FirstCharacter (class in unitxt.processors) FixedFusion (class in unitxt.fusion) FixedGroupAbsvalNormCohensHParaphraseAccuracy (class in unitxt.metrics) FixedGroupAbsvalNormCohensHParaphraseStringContainment (class in unitxt.metrics)

FixedGroupAbsvalNormHedgesGParaphraseAccuracy (class in unitxt.metrics) FixedGroupAbsvalNormHedgesGParaphraseStringContainment (class in unitxt.metrics) FixedGroupMeanAccuracy (class in unitxt.metrics) FixedGroupMeanBaselineAccuracy (class in unitxt.metrics) FixedGroupMeanBaselineStringContainment (class in unitxt.metrics) FixedGroupMeanParaphraseAccuracy (class in unitxt.metrics) FixedGroupMeanParaphraseStringContainment (class in unitxt.metrics) FixedGroupMeanStringContainment (class in unitxt.metrics) FixedGroupNormCohensHParaphraseAccuracy (class in unitxt.metrics) FixedGroupNormCohensHParaphraseStringContainment (class in unitxt.metrics) FixedGroupNormHedgesGParaphraseAccuracy (class in unitxt.metrics) FixedGroupNormHedgesGParaphraseStringContainment (class in unitxt.metrics) FixedGroupPDRParaphraseAccuracy (class in unitxt.metrics) FixedGroupPDRParaphraseStringContainment (class in unitxt.metrics) FixedIndicesSampler (class in unitxt.splitters) FixJsonSchemaOfParameterTypes (class in unitxt.operators) FixWhiteSpace (class in unitxt.processors) Flatten (class in unitxt.collections_operators) flatten() (in module unitxt.collections_operators) flatten_dict() (in module unitxt.utils) FlattenInstances (class in unitxt.operators) FLUENCY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) Format (class in unitxt.formats) format (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) format() (unitxt.logging_utils.SizeLimitedFormatter method) format_type_string() (in module unitxt.type_utils) FormatText (class in unitxt.string_operators) from_dict() (unitxt.artifact.Artifact class method) from_exception() (unitxt.operator.OperatorError class method) from_generators() (unitxt.stream.MultiStream class method) from_iterables() (unitxt.stream.MultiStream class method) from_operator_error() (unitxt.operator.OperatorError class method) FromIterables (class in unitxt.operators) FromPredictionsAndOriginalData (class in unitxt.metric_utils) FUNNY_JOKE (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) FuzzyNer (class in unitxt.metrics)

G

GENERAL_HARM_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) GENERAL_HARM_USER_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) generate_cache_key() (in module unitxt.text2sql_utils) generate_until (unitxt.inference.LMMSEvalInferenceEngine attribute) generators (unitxt.dataset.Dataset property) GeneratorStream (class in unitxt.stream) GenericInferenceEngine (class in unitxt.inference) GenericTypedDict (class in unitxt.type_utils) Get (class in unitxt.collections_operators) get() (unitxt.utils.LRUCache method) get_all_environment_variables() (unitxt.settings_utils.Settings method) get_all_files_in_dir() (in module unitxt.file_utils) get_args() (in module unitxt.type_utils) get_artifact() (unitxt.operators.ArtifactFetcherMixin class method) get_artifact_type() (unitxt.artifact.Artifact class method) get_artifacts_data_classification() (in module unitxt.artifact) get_cache() (in module unitxt.text2sql_utils) get_catalog_dirs() (in module unitxt.ui.load_catalog_data) get_catalog_items() (in module unitxt.ui.load_catalog_data) get_catalog_items_from_dir() (in module unitxt.ui.load_catalog_data) get_catalog_name_and_args() (in module unitxt.artifact) get_closest_artifact_type() (in module unitxt.artifact) get_constants() (in module unitxt.settings_utils) get_contexts() (unitxt.llm_as_judge.LLMJudge method) get_criteria() (unitxt.llm_as_judge.LLMJudge method) get_dataset_artifact() (in module unitxt.dataset_utils) get_db_connector() (in module unitxt.text2sql_utils) get_db_file_path() (unitxt.text2sql_utils.LocalSQLiteConnector method) get_default() (unitxt.dataclass.Field method) get_directory_size() (in module unitxt.test_utils.storage) get_evaluator_metadata() (in module unitxt.llm_as_judge_utils) get_field_default() (in module unitxt.dataclass) get_fields() (in module unitxt.dataclass) get_file_from_item_name() (in module unitxt.ui.load_catalog_data) get_from_catalog() (in module unitxt.catalog) get_generator() (unitxt.stream.MultiStream method) get_group_scores() (unitxt.metrics.InstanceMetric method) get_images_without_text() (in module unitxt.inference) get_index_or_default() (in module unitxt.metrics) get_installed_version() (in module unitxt.hf_utils) get_internal_imports() (in module unitxt.hf_utils) get_likert_scale_criteria() (in module unitxt.llm_as_judge_constants) get_local_catalogs_paths() (in module unitxt.catalog) get_logger() (in module unitxt.logging_utils) get_metric_results_from_prediction_outputs() (unitxt.llm_as_judge_from_template.LLMAsJudgeBase method) get_missing_imports() (in module unitxt.hf_utils) get_model_and_label_id() (in module unitxt.inference) get_model_details() (unitxt.inference.InferenceEngine method) get_options_log_probs() (unitxt.inference.OptionSelectingByLogProbsInferenceEngine method) (unitxt.inference.WMLInferenceEngineBase method) get_or_set() (unitxt.text2sql_utils.Cache method) get_origin() (in module unitxt.type_utils) get_parsed_context() (in module unitxt.llm_as_judge_utils) get_pretty_print_name() (unitxt.artifact.Artifact method)

get_random_generator_based_on_instance() (in module unitxt.splitters) get_raw() (in module unitxt.artifact) get_remote_metrics_endpoint() (in module unitxt.metric_utils) get_remote_metrics_names() (in module unitxt.metric_utils) get_repr_dict() (unitxt.dataclass.Dataclass method) get_schema() (in module unitxt.schema) get_seed() (in module unitxt.random_utils) get_settings() (in module unitxt.settings_utils) get_sql_execution_results() (in module unitxt.text2sql_utils) get_table_schema() (unitxt.text2sql_utils.DatabaseConnector method) (unitxt.text2sql_utils.InMemoryDatabaseConnector method) (unitxt.text2sql_utils.LocalSQLiteConnector method) (unitxt.text2sql_utils.RemoteDatabaseConnector method) get_task_data_dict() (in module unitxt.llm_as_judge_from_template) get_templates() (in module unitxt.ui.load_catalog_data) get_templates_from_dict() (in module unitxt.ui.load_catalog_data) get_templates_from_list() (in module unitxt.ui.load_catalog_data) get_templates_from_str() (in module unitxt.ui.load_catalog_data) get_text_without_images() (in module unitxt.inference) get_token_count() (unitxt.inference.OptionSelectingByLogProbsInferenceEngine method) get_values() (in module unitxt.dict_utils) get_yes_no_criteria() (in module unitxt.llm_as_judge_constants) GetItemByIndex (class in unitxt.operators) GetLength (class in unitxt.collections_operators) GetNumOfTableCells (class in unitxt.struct_data_operators) GetSQL (class in unitxt.processors) GetStringAfter (class in unitxt.processors) GithubCatalog (class in unitxt.catalog) global_scores (unitxt.metric_utils.EvaluationResults property) GlobalMetric (class in unitxt.metrics) GlobalScores (class in unitxt.metric_utils) GRAMMAR_AND_PUNCTUATION (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) GraniteDocumentsFormat (class in unitxt.formats) GraniteGuardianAgenticRisk (class in unitxt.metrics) GraniteGuardianAssistantRisk (class in unitxt.metrics) GraniteGuardianBase (class in unitxt.metrics) GraniteGuardianCustomRisk (class in unitxt.metrics) GraniteGuardianRagRisk (class in unitxt.metrics) GraniteGuardianUserRisk (class in unitxt.metrics) GrayScale (class in unitxt.image_operators) GridLines (class in unitxt.image_operators) GROUNDEDNESS_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) group_by (unitxt.standard.DatasetRecipe attribute) group_str() (in module unitxt.metric_utils) group_str_to_key_value() (in module unitxt.metric_utils) GroupDictWithRegex (class in unitxt.processors) GroupMean (class in unitxt.metrics) GroupMeanAccuracy (class in unitxt.metrics) GroupMeanStringContainment (class in unitxt.metrics) GroupMeanTokenOverlap (class in unitxt.metrics) GroupMetric (class in unitxt.metrics) GroupReduction (class in unitxt.metrics) groups_scores (unitxt.metric_utils.EvaluationResults property) GroupsScores (class in unitxt.metric_utils)

H

HARMFULNESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) has_card_templates (unitxt.standard.DatasetRecipe property) has_custom_demos_pool (unitxt.standard.DatasetRecipe property) has_no_templates (unitxt.standard.DatasetRecipe property) HashImage (class in unitxt.image_operators) hf_compute_args (unitxt.metrics.HuggingfaceBulkMetric attribute) (unitxt.metrics.HuggingfaceInstanceMetric attribute) hf_evaluate_load() (in module unitxt.metrics) hf_get_dataset_splits() (in module unitxt.loaders) hf_load_dataset() (in module unitxt.loaders) hf_metric_fields (unitxt.metrics.RougeHF attribute)

HFAutoModelInferenceEngine (class in unitxt.inference) HFGenerationParamsMixin (class in unitxt.inference) HFInferenceEngineBase (class in unitxt.inference) HFLlavaInferenceEngine (class in unitxt.inference) HFOptionSelectingInferenceEngine (class in unitxt.inference) HFPeftInferenceEngine (class in unitxt.inference) HFPipelineBasedInferenceEngine (class in unitxt.inference) HFSystemFormat (class in unitxt.formats) HuggingfaceBulkMetric (class in unitxt.metrics) HuggingfaceInstanceMetric (class in unitxt.metrics) HuggingfaceMetric (class in unitxt.metrics)

I

Image (class in unitxt.types) image_to_data_url() (in module unitxt.image_operators) ImageAugmentor (class in unitxt.image_operators) ImageDataString (class in unitxt.image_operators) ImageFieldOperator (class in unitxt.image_operators) ImageFileContent (class in unitxt.formats) ImageSerializer (class in unitxt.serializers) ImageUrlContent (class in unitxt.formats) import_module_from_file() (in module unitxt.utils) in_group_reduction (unitxt.metrics.GroupMetric attribute) INCLUSIVITY (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) IndexOf (class in unitxt.operators) infer() (in module unitxt.api) (unitxt.inference.InferenceEngine method) infer_instances() (unitxt.llm_as_judge_from_template.LLMAsJudgeBase method) infer_log_probs() (unitxt.inference.LogProbInferenceEngine method) infer_type() (in module unitxt.type_utils) infer_type_string() (in module unitxt.type_utils) InferDictsToBinaryLogprobs (class in unitxt.processors) inference_model (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) InferenceEngine (class in unitxt.inference) INFORMATION_FROM_REFERENCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) INFORMATION_OUTSIDE_REFERENCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) init_warning() (in module unitxt.deprecation_utils) initialize_inference_engine() (in module unitxt.evaluate_cli) InMemoryDatabaseConnector (class in unitxt.text2sql_utils) InputFormatTemplate (class in unitxt.templates) InputOutputTemplate (class in unitxt.templates) InputOutputTemplateWithCustomTarget (class in unitxt.templates) INSENSITIVITY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) InsertEmptyTableRows (class in unitxt.struct_data_operators) instance (unitxt.artifact.Catalogs attribute) instance_generator() (in module unitxt.operator) instance_result() (in module unitxt.operator) instance_scores (unitxt.metric_utils.EvaluationResults property) InstanceFieldOperator (class in unitxt.operators) InstanceInput (class in unitxt.metric_utils) InstanceMetric (class in unitxt.metrics) InstanceOperator (class in unitxt.operator) InstanceOperatorValidator (class in unitxt.operator) InstanceOperatorWithMultiStreamAccess (class in unitxt.operator) InstanceScores (class in unitxt.metric_utils)

InterleaveListsToDialogOperator (class in unitxt.operators) InternalField (class in unitxt.dataclass) interpret_effect_size() (in module unitxt.metrics) Intersect (class in unitxt.operators) IntersectCorrespondingFields (class in unitxt.operators) IobExtractor (class in unitxt.span_lableing_operators) IRRELEVANT_INFORMATION (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) is_abstract_field() (in module unitxt.dataclass) is_artifact_dict() (unitxt.artifact.Artifact class method) is_artifact_file() (unitxt.artifact.Artifact class method) is_camel_case() (in module unitxt.text_utils) is_class_method() (in module unitxt.dataclass) is_dataclass() (in module unitxt.dataclass) is_final_field() (in module unitxt.dataclass) is_index() (in module unitxt.dict_utils) is_local_catalog_registered() (in module unitxt.register) is_made_of_sub_strings() (in module unitxt.text_utils) is_module_available() (in module unitxt.utils) is_name() (in module unitxt.dict_utils) is_name_legal_for_catalog() (in module unitxt.artifact) is_new_type() (in module unitxt.type_utils) is_original_key() (in module unitxt.metrics) is_package_installed() (in module unitxt.utils) is_possible_field() (in module unitxt.dataclass) is_possible_identifier() (unitxt.artifact.Artifact class method) is_registered_class() (unitxt.artifact.Artifact class method) is_registered_class_name() (unitxt.artifact.Artifact class method) is_registered_type() (unitxt.artifact.Artifact class method) is_snake_case() (in module unitxt.text_utils) is_sqlglot_parsable() (in module unitxt.text2sql_utils) is_sqlparse_parsable() (in module unitxt.text2sql_utils) is_subpath() (in module unitxt.dict_utils) is_type() (in module unitxt.type_utils) is_type_dict() (in module unitxt.type_utils) is_typed_dict() (in module unitxt.type_utils) is_uninitilized() (unitxt.settings_utils.Constants class method) (unitxt.settings_utils.Settings class method) is_wildcard() (in module unitxt.dict_utils) IsCodeMixed (class in unitxt.metrics) isoftype() (in module unitxt.type_utils) issubtype() (in module unitxt.type_utils) ItemPicker (class in unitxt.collections) IterableSource (class in unitxt.operators)

J

JaccardIndex (class in unitxt.metrics) JaccardIndexString (class in unitxt.metrics) JAILBREAK_USER_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) Join (class in unitxt.string_operators) JoinStr (class in unitxt.operators) JoinStreams (class in unitxt.stream_operators)

JoinSubsetsAndGroups (class in unitxt.metric_utils) json_dump() (in module unitxt.utils) JsonOutputFormatTemplate (class in unitxt.templates) JsonOutputTemplate (class in unitxt.templates) JsonSchema (class in unitxt.types) JsonStrToDict (class in unitxt.struct_data_operators) judge_to_generator_fields_mapping (unitxt.llm_as_judge_from_template.TaskBasedLLMasJudge attribute)

K

KendallTauMetric (class in unitxt.metrics) key (unitxt.operators.RecursiveReplace attribute)

KeyValTemplate (class in unitxt.templates) KeyValueExtraction (class in unitxt.metrics) KPA (class in unitxt.metrics)

L

launch() (in module unitxt.assistant) (in module unitxt.ui) launch_without_header() (in module unitxt.ui) LazyLoader (class in unitxt.loaders) LazyLoadMixin (class in unitxt.inference) LengthBalancer (class in unitxt.operators) line_color (unitxt.image_operators.GridLines attribute) lines_defining_obj_in_card() (in module unitxt.text_utils) ListCollection (class in unitxt.collections) ListFieldValues (class in unitxt.operators) ListSerializer (class in unitxt.serializers) ListStream (class in unitxt.stream) ListToEmptyEntitiesTuples (class in unitxt.processors) ListToKeyValPairs (class in unitxt.struct_data_operators) ListWithMetadata (class in unitxt.inference) LiteLLMInferenceEngine (class in unitxt.inference) LiteralEval (class in unitxt.processors) LlamaIndexCorrectness (class in unitxt.metrics) LlamaIndexFaithfulness (class in unitxt.metrics) LlamaIndexLLMMetric (class in unitxt.metrics) LLMAsJudge (class in unitxt.llm_as_judge_from_template) LLMAsJudgeBase (class in unitxt.llm_as_judge_from_template) LLMJudge (class in unitxt.llm_as_judge) LLMJudgeDirect (class in unitxt.llm_as_judge) LLMJudgePairwise (class in unitxt.llm_as_judge) LMMSEvalBaseInferenceEngine (class in unitxt.inference) LMMSEvalInferenceEngine (class in unitxt.inference) LMMSEvalLoglikelihoodInferenceEngine (class in unitxt.inference) load() (unitxt.artifact.Artifact class method) load_cards_data() (in module unitxt.ui.load_catalog_data) load_chat_source() (in module unitxt.schema)

load_dataset() (in module unitxt.api) load_examples_from_dataset_recipe() (in module unitxt.test_utils.card) load_json() (in module unitxt.utils) load_recipe() (in module unitxt.api) LoadCriteria (class in unitxt.llm_as_judge_operators) LoadCriteriaWithOptions (class in unitxt.llm_as_judge_operators) LoadCSV (class in unitxt.loaders) Loader (class in unitxt.loaders) LoadFromAPI (class in unitxt.loaders) LoadFromDictionary (class in unitxt.loaders) LoadFromHFSpace (class in unitxt.loaders) LoadFromIBMCloud (class in unitxt.loaders) LoadFromKaggle (class in unitxt.loaders) LoadFromSklearn (class in unitxt.loaders) LoadHF (class in unitxt.loaders) LoadIOB (class in unitxt.loaders) LoadJson (class in unitxt.struct_data_operators) LoadJsonFile (class in unitxt.loaders) loads_batch() (in module unitxt.schema) loads_instance() (in module unitxt.schema) LoadWithPandas (class in unitxt.loaders) local_catalog_summary() (in module unitxt.catalog) LocalCatalog (class in unitxt.catalog) LocalSQLiteConnector (class in unitxt.text2sql_utils) LOGICAL_VALIDITY_OF_REASONING (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) LogProbInferenceEngine (class in unitxt.inference) LongString (class in unitxt.utils) Lower (class in unitxt.processors) LowerCaseTillPunc (class in unitxt.processors) lru_cache_decorator() (in module unitxt.utils) LRUCache (class in unitxt.utils) ls() (in module unitxt.catalog)

M

main() (in module unitxt.evaluate_cli) main_score (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) MANIPULATIVE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) MAP (class in unitxt.metrics) map() (unitxt.metrics.NormalizedSacrebleu method) map_stream() (unitxt.metrics.ReflectionToolCallingMetric method) map_values (unitxt.operators.RecursiveReplace attribute) MapHTMLTableToJSON (class in unitxt.struct_data_operators) MapInstanceValues (class in unitxt.operators) MapReduceMetric (class in unitxt.metrics) MapReplace (class in unitxt.string_operators) MapTableListsToStdTableJSON (class in unitxt.struct_data_operators) MapValues (class in unitxt.operators) MaskColumnsNames (class in unitxt.struct_data_operators) MatchClosestOption (class in unitxt.processors) MatthewsCorrelation (class in unitxt.metrics) max_demos_size (unitxt.standard.DatasetRecipe property) max_item_scores() (unitxt.metrics.MetricWithConfidenceInterval static method) MaxAccuracy (class in unitxt.metrics) MaxReduction (class in unitxt.metrics) maybe_recover_artifact() (in module unitxt.artifact) maybe_recover_artifacts_structure() (in module unitxt.artifact) mean() (unitxt.metrics.Statistic method) mean_subgroup_score() (in module unitxt.metrics) MeanReduction (class in unitxt.metrics) MeanSquaredError (class in unitxt.metrics) memoize() (unitxt.text2sql_utils.Cache method) merge_dialog_entries() (unitxt.dialog_operators.SerializeOpenAiFormatDialog static method) MergeStreams (class in unitxt.operators) Message (class in unitxt.formats) Meteor (class in unitxt.metrics) MeteorFast (class in unitxt.metrics) Metric (class in unitxt.base_metric) (class in unitxt.metric) MetricBasedNer (class in unitxt.metrics) MetricInferenceEngine (class in unitxt.inference) MetricPipeline (class in unitxt.metrics) MetricRecipe (class in unitxt.metric_utils) MetricRequest (class in unitxt.metric_utils) MetricResponse (class in unitxt.metric_utils) MetricsEnsemble (class in unitxt.metrics) MetricsList (class in unitxt.metrics) MetricWithConfidenceInterval (class in unitxt.metrics) MinimumOneExamplePerLabelRefiner (class in unitxt.operators) MissingArtifactTypeError MissingDefaultError MissingKaggleCredentialsError MissingRequirementsError MissingStreamError mock_logprobs_default_value_factory() (in module unitxt.inference) mock_models (unitxt.metrics.LlamaIndexLLMMetric attribute) MockInferenceEngine (class in unitxt.inference) MockModeMixin (class in unitxt.inference) model_names_dict (unitxt.inference.RITSInferenceEngine attribute) ModelProviderEnum (class in unitxt.llm_as_judge_constants) module unitxt, [1] unitxt.api unitxt.artifact unitxt.assistant unitxt.assistant.assessment unitxt.assistant.assessment.my_pretty_table unitxt.augmentors unitxt.base_metric unitxt.benchmark unitxt.blocks unitxt.card unitxt.catalog unitxt.collections unitxt.collections_operators unitxt.dataclass unitxt.dataset unitxt.dataset_utils unitxt.deprecation_utils unitxt.dialog_operators unitxt.dict_utils unitxt.error_utils unitxt.eval_utils unitxt.evaluate_cli unitxt.file_utils unitxt.formats unitxt.fusion unitxt.generator_utils unitxt.hf_utils unitxt.image_operators unitxt.inference unitxt.instructions unitxt.llm_as_judge unitxt.llm_as_judge_chat_templates unitxt.llm_as_judge_constants unitxt.llm_as_judge_from_template unitxt.llm_as_judge_operators unitxt.llm_as_judge_utils unitxt.loaders unitxt.logging_utils unitxt.metric unitxt.metric_utils unitxt.metrics unitxt.normalizers unitxt.operator unitxt.operators unitxt.parsing_utils unitxt.processors unitxt.random_utils unitxt.recipe unitxt.register unitxt.schema unitxt.serializers unitxt.service unitxt.service.metrics unitxt.settings_utils unitxt.span_lableing_operators unitxt.split_utils unitxt.splitters unitxt.standard unitxt.stream unitxt.stream_operators unitxt.string_operators unitxt.struct_data_operators unitxt.system_prompts unitxt.task unitxt.templates unitxt.test_utils unitxt.test_utils.artifact unitxt.test_utils.card unitxt.test_utils.catalog unitxt.test_utils.metrics unitxt.test_utils.operators unitxt.test_utils.storage unitxt.text2sql_utils unitxt.text_utils unitxt.type_utils unitxt.types unitxt.ui unitxt.ui.load_catalog_data unitxt.ui.settings unitxt.utils unitxt.validate unitxt.version

Move (class in unitxt.operators) MRR (class in unitxt.metrics) MultiDocumentSerializer (class in unitxt.serializers) MultiLabelTemplate (class in unitxt.templates) MultipleChoiceTemplate (class in unitxt.templates) MultipleSourceLoader (class in unitxt.loaders) MultipleToolCallPostProcessor (class in unitxt.struct_data_operators) MultiReferenceTemplate (class in unitxt.templates) MultiStream (class in unitxt.stream) MultiStreamOperator (class in unitxt.operator) MultiTurnMetric (class in unitxt.metrics) MultiTurnTemplate (class in unitxt.templates) MultiTurnToolCallingMetric (class in unitxt.metrics) MultiTypeSerializer (class in unitxt.serializers)

N

nan_max() (in module unitxt.metrics) nan_mean() (in module unitxt.metric_utils) (in module unitxt.metrics) nan_std() (in module unitxt.metrics) NATURALNESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) NDCG (class in unitxt.metrics) NER (class in unitxt.metrics) nested_tuple_to_string() (in module unitxt.text_utils) new_random_generator() (in module unitxt.metrics) (in module unitxt.random_utils) NLTKMixin (class in unitxt.metrics) None (unitxt.operators.ShuffleFieldValues attribute)

NonPositionalField (class in unitxt.dataclass) normalize() (in module unitxt.type_utils) normalize_answer() (in module unitxt.metrics) normalized_cohens_h() (in module unitxt.metrics) normalized_hedges_g() (in module unitxt.metrics) NormalizedSacrebleu (class in unitxt.metrics) NormalizedType (class in unitxt.type_utils) NormalizeListFields (class in unitxt.normalizers) NullAugmentor (class in unitxt.augmentors) NullTemplate (class in unitxt.templates) NumberQuantizingSerializer (class in unitxt.serializers) NumberSerializer (class in unitxt.serializers)

O

object_to_str_without_addresses() (in module unitxt.api) OBJECTIVITY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) Oldify (class in unitxt.image_operators) OllamaInferenceEngine (class in unitxt.inference) openai_models (unitxt.metrics.LlamaIndexLLMMetric attribute) OpenAiInferenceEngine (class in unitxt.inference) OpenAiInferenceEngineParamsMixin (class in unitxt.inference) Operator (class in unitxt.operator)

OperatorError optional_all() (in module unitxt.type_utils) optional_any() (in module unitxt.type_utils) OptionalField (class in unitxt.dataclass) OptionSelectingByLogProbsInferenceEngine (class in unitxt.inference) OptionSelectionStrategyEnum (class in unitxt.llm_as_judge_constants) origin (unitxt.type_utils.NormalizedType attribute) OutputFormatTemplate (class in unitxt.templates) OutputQuantizingTemplate (class in unitxt.templates)

P

PackageRequirementsMixin (class in unitxt.operator) PagedStreamOperator (class in unitxt.operator) PairwiseChoiceTemplate (class in unitxt.templates) PairwiseComparativeRatingTemplate (class in unitxt.templates) PairwiseCriteriaCatalogEnum (class in unitxt.llm_as_judge_constants) parse() (in module unitxt.dataset_utils) parse_key_equals_value_string_to_dict() (in module unitxt.parsing_utils) parse_random_mix_string() (in module unitxt.split_utils) parse_slices_string() (in module unitxt.split_utils) parse_string_types_instead_of_actual_objects() (in module unitxt.base_metric) (in module unitxt.metrics) (in module unitxt.task) parse_type_dict() (in module unitxt.type_utils) parse_type_string() (in module unitxt.type_utils) ParseCSV (class in unitxt.struct_data_operators) Pearsonr (class in unitxt.metrics) peek() (unitxt.stream.DynamicStream method) (unitxt.stream.GeneratorStream method) (unitxt.stream.ListStream method) (unitxt.stream.Stream method) perform_evaluation_step() (unitxt.llm_as_judge.LLMJudge method) performance_drop_rate() (in module unitxt.metrics) Perplexity (class in unitxt.metrics) Perturb (class in unitxt.operators) PillowMixin (class in unitxt.image_operators) PixelNoise (class in unitxt.image_operators) Pop (class in unitxt.collections_operators) post_process() (in module unitxt.api) PostProcess (class in unitxt.processors) postprocessors (unitxt.templates.MultiLabelTemplate attribute) (unitxt.templates.NullTemplate attribute) (unitxt.templates.SpanLabelingJsonTemplate attribute) (unitxt.templates.SpanLabelingTemplate attribute) PostProcessRecipe (class in unitxt.metric_utils) PrecisionBinary (class in unitxt.metrics) PrecisionMacroMultiLabel (class in unitxt.metrics) PrecisionMicroMultiLabel (class in unitxt.metrics) prediction_type (unitxt.base_metric.Metric attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) (unitxt.metrics.Accuracy attribute) (unitxt.metrics.ANLS attribute) (unitxt.metrics.BinaryAccuracy attribute) (unitxt.metrics.BinaryMaxAccuracy attribute) (unitxt.metrics.CharEditDistance attribute) (unitxt.metrics.CorrelationMetric attribute) (unitxt.metrics.CustomF1 attribute) (unitxt.metrics.Detector attribute) (unitxt.metrics.ExactMatchMM attribute) (unitxt.metrics.F1 attribute) (unitxt.metrics.F1Binary attribute) (unitxt.metrics.F1MultiLabel attribute) (unitxt.metrics.F1Strings attribute) (unitxt.metrics.FaithfulnessHHEM attribute) (unitxt.metrics.FinQAEval attribute) (unitxt.metrics.FuzzyNer attribute) (unitxt.metrics.IsCodeMixed attribute) (unitxt.metrics.JaccardIndex attribute) (unitxt.metrics.JaccardIndexString attribute) (unitxt.metrics.KendallTauMetric attribute) (unitxt.metrics.KeyValueExtraction attribute) (unitxt.metrics.KPA attribute) (unitxt.metrics.LlamaIndexLLMMetric attribute) (unitxt.metrics.MatthewsCorrelation attribute) (unitxt.metrics.MeanSquaredError attribute) (unitxt.metrics.Meteor attribute) (unitxt.metrics.MetricBasedNer attribute) (unitxt.metrics.MultiTurnToolCallingMetric attribute) (unitxt.metrics.NDCG attribute) (unitxt.metrics.NER attribute) (unitxt.metrics.NormalizedSacrebleu attribute) (unitxt.metrics.Perplexity attribute) (unitxt.metrics.PredictionLength attribute) (unitxt.metrics.ReflectionToolCallingMetric attribute) (unitxt.metrics.ReflectionToolCallingMetricSyntactic attribute) (unitxt.metrics.RegardMetric attribute) (unitxt.metrics.RelaxedCorrectness attribute) (unitxt.metrics.RetrievalMetric attribute) (unitxt.metrics.RocAuc attribute) (unitxt.metrics.Rouge attribute) (unitxt.metrics.RougeHF attribute) (unitxt.metrics.SafetyMetric attribute) (unitxt.metrics.Squad attribute) (unitxt.metrics.StringContainment attribute) (unitxt.metrics.StringContainmentOld attribute) (unitxt.metrics.StringContainmentRatio attribute) (unitxt.metrics.TokenOverlap attribute) (unitxt.metrics.ToolCallingMetric attribute) (unitxt.metrics.ToolCallKeyValueExtraction attribute) (unitxt.metrics.WebsrcSquadF1 attribute) (unitxt.metrics.Wer attribute)

PredictionLength (class in unitxt.metrics) prefixes (unitxt.augmentors.AugmentPrefixSuffix attribute) prepare() (unitxt.artifact.Artifact method) (unitxt.llm_as_judge.LLMJudge method) (unitxt.llm_as_judge.LLMJudgePairwise method) (unitxt.metrics.LlamaIndexCorrectness method) (unitxt.metrics.LlamaIndexFaithfulness method) (unitxt.standard.DatasetRecipe method) prepare_args() (unitxt.artifact.Artifact method) prepare_engine() (unitxt.inference.InferenceEngine method) prepare_instances() (unitxt.llm_as_judge_from_template.LLMAsJudgeBase method) prepare_kwargs() (in module unitxt.evaluate_cli) prepare_output_paths() (in module unitxt.evaluate_cli) prepend_timestamp_to_path() (in module unitxt.evaluate_cli) print_dict() (in module unitxt.text_utils) print_dict_as_python() (in module unitxt.text_utils) print_dict_as_yaml() (in module unitxt.text_utils) print_generic_table() (in module unitxt.assistant.assessment.my_pretty_table) print_predictions() (in module unitxt.test_utils.card) print_recipe_output() (in module unitxt.test_utils.card) process_and_save_results() (in module unitxt.evaluate_cli) process_data_after_load() (unitxt.artifact.Artifact class method) process_data_before_dump() (unitxt.artifact.Artifact method) process_image() (unitxt.image_operators.GridLines method) (unitxt.image_operators.PixelNoise method) process_instance_value() (in module unitxt.processors) process_value() (unitxt.operators.ReadFile method) (unitxt.processors.GetSQL method) produce() (in module unitxt.api) (unitxt.standard.DatasetRecipe method) PROFANITY_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) PROFANITY_USER_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) PROFESSIONAL_TONE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) ProjectArtifactRegisterer (class in unitxt.register) provider_model_map (unitxt.inference.CrossProviderInferenceEngine attribute) PythonCallProcessor (class in unitxt.struct_data_operators) pytrec_eval_at_k() (in module unitxt.metrics)

Q

QUALITY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute)

R

RagResponse (class in unitxt.types) random() (in module unitxt.templates) random_mix_generator() (in module unitxt.split_utils) random_mix_streams() (in module unitxt.split_utils) RandomForestMetricsEnsemble (class in unitxt.metrics) RandomPicker (class in unitxt.collections) RandomSampler (class in unitxt.splitters) RandomSizeSample (class in unitxt.splitters) rank_indexes() (in module unitxt.llm_as_judge_utils) read_file() (in module unitxt.loaders) ReadFile (class in unitxt.operators) RecallBinary (class in unitxt.metrics) RecallMacroMultiLabel (class in unitxt.metrics) RecallMicroMultiLabel (class in unitxt.metrics) Recipe (class in unitxt.recipe) recursive_copy() (in module unitxt.utils) recursive_deep_copy() (in module unitxt.utils) recursive_key_value_replace() (in module unitxt.operators) recursive_shallow_copy() (in module unitxt.utils) RecursiveCopy (class in unitxt.operators) RecursiveReplace (class in unitxt.operators) reduce() (unitxt.metrics.NormalizedSacrebleu method) reduction (unitxt.metrics.AccuracyFast attribute) (unitxt.metrics.BertScore attribute) (unitxt.metrics.JaccardIndex attribute) (unitxt.metrics.MeteorFast attribute) (unitxt.metrics.MultiTurnToolCallingMetric attribute) (unitxt.metrics.ReflectionToolCallingMetric attribute) (unitxt.metrics.ReflectionToolCallingMetricSyntactic attribute) (unitxt.metrics.StringContainment attribute) (unitxt.metrics.ToolCallingMetric attribute) reduction_map (unitxt.llm_as_judge.LLMJudgeDirect attribute) (unitxt.llm_as_judge.LLMJudgePairwise attribute) (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) (unitxt.metrics.Accuracy attribute) (unitxt.metrics.ANLS attribute) (unitxt.metrics.BinaryAccuracy attribute) (unitxt.metrics.CharEditDistance attribute) (unitxt.metrics.CharEditDistanceAccuracy attribute) (unitxt.metrics.Detector attribute) (unitxt.metrics.ExactMatchMM attribute) (unitxt.metrics.F1Strings attribute) (unitxt.metrics.FaithfulnessHHEM attribute) (unitxt.metrics.FinQAEval attribute) (unitxt.metrics.FixedGroupAbsvalNormCohensHParaphraseAccuracy attribute) (unitxt.metrics.FixedGroupAbsvalNormCohensHParaphraseStringContainment attribute) (unitxt.metrics.FixedGroupAbsvalNormHedgesGParaphraseAccuracy attribute) (unitxt.metrics.FixedGroupAbsvalNormHedgesGParaphraseStringContainment attribute) (unitxt.metrics.FixedGroupMeanAccuracy attribute) (unitxt.metrics.FixedGroupMeanBaselineAccuracy attribute) (unitxt.metrics.FixedGroupMeanBaselineStringContainment attribute) (unitxt.metrics.FixedGroupMeanParaphraseAccuracy attribute) (unitxt.metrics.FixedGroupMeanParaphraseStringContainment attribute) (unitxt.metrics.FixedGroupMeanStringContainment attribute) (unitxt.metrics.FixedGroupNormCohensHParaphraseAccuracy attribute) (unitxt.metrics.FixedGroupNormCohensHParaphraseStringContainment attribute) (unitxt.metrics.FixedGroupNormHedgesGParaphraseAccuracy attribute) (unitxt.metrics.FixedGroupNormHedgesGParaphraseStringContainment attribute) (unitxt.metrics.FixedGroupPDRParaphraseAccuracy attribute) (unitxt.metrics.FixedGroupPDRParaphraseStringContainment attribute) (unitxt.metrics.GraniteGuardianBase attribute) (unitxt.metrics.GroupMeanAccuracy attribute) (unitxt.metrics.GroupMeanStringContainment attribute) (unitxt.metrics.GroupMeanTokenOverlap attribute) (unitxt.metrics.IsCodeMixed attribute) (unitxt.metrics.MAP attribute) (unitxt.metrics.MaxAccuracy attribute) (unitxt.metrics.Meteor attribute) (unitxt.metrics.MetricsEnsemble attribute) (unitxt.metrics.MRR attribute) (unitxt.metrics.Perplexity attribute) (unitxt.metrics.PredictionLength attribute) (unitxt.metrics.Rouge attribute) (unitxt.metrics.RougeHF attribute) (unitxt.metrics.SQLExecutionAccuracy attribute) (unitxt.metrics.SQLExecutionLogicAccuracy attribute) (unitxt.metrics.SQLNonExecutionAccuracy attribute) (unitxt.metrics.StringContainmentOld attribute) (unitxt.metrics.StringContainmentRatio attribute) (unitxt.metrics.TokenOverlap attribute) (unitxt.metrics.UnsortedListExactMatch attribute)

ReductionInstanceMetric (class in unitxt.metrics) REFERENCE_DOCUMENT_FAITHFULNESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) ReflectionToolCallingMetric (class in unitxt.metrics) ReflectionToolCallingMetricSyntactic (class in unitxt.metrics) ReflectionToolCallingMixin (class in unitxt.metrics) RegardMetric (class in unitxt.metrics) RegexParser (class in unitxt.processors) RegexReplace (class in unitxt.string_operators) RegexSplit (class in unitxt.string_operators) register() (unitxt.artifact.Catalogs method) register_all_artifacts() (in module unitxt.artifact) (in module unitxt.register) register_class() (unitxt.artifact.Artifact class method) register_existing_template() (in module unitxt.ui.load_catalog_data) register_inline_template() (in module unitxt.ui.load_catalog_data) register_local_catalog() (in module unitxt.register) register_local_catalog_for_tests() (in module unitxt.test_utils.catalog) register_type() (in module unitxt.type_utils) relaxed_correctness() (unitxt.metrics.RelaxedCorrectness method) RelaxedCorrectness (class in unitxt.metrics) RELEVANCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) RemoteDatabaseConnector (class in unitxt.text2sql_utils) RemoteMetric (class in unitxt.metrics) remove_numerics_and_quoted_texts() (in module unitxt.utils) remove_values (unitxt.operators.RecursiveReplace attribute) RemoveArticles (class in unitxt.processors) RemoveFields (class in unitxt.operators) RemovePunctuations (class in unitxt.processors) RemoveValues (class in unitxt.operators) Rename (class in unitxt.operators) rename_split() (in module unitxt.split_utils) RenameSplits (class in unitxt.splitters) Replace (class in unitxt.string_operators) replace_class_names() (in module unitxt.type_utils) replace_select_clause() (in module unitxt.text2sql_utils) require() (in module unitxt.utils) required_fields() (in module unitxt.dataclass) RequiredField (class in unitxt.dataclass) RequiredFieldError requirements_list (unitxt.loaders.LoadFromHFSpace attribute) RerankRecall (class in unitxt.metrics) resample_from_non_nan() (unitxt.metrics.MetricWithConfidenceInterval method) reset() (unitxt.artifact.Catalogs method) reset_artifacts_json_cache() (in module unitxt.artifact) RetrievalAtK (class in unitxt.metrics) RetrievalMetric (class in unitxt.metrics) retry_connection_with_exponential_backoff() (in module unitxt.utils) ReusableGenerator (class in unitxt.generator_utils) Reward (class in unitxt.metrics) RiskType (class in unitxt.metrics) RITSInferenceEngine (class in unitxt.inference) RocAuc (class in unitxt.metrics) RootMeanReduction (class in unitxt.metrics) RootMeanSquaredError (class in unitxt.metrics) Rouge (class in unitxt.metrics) rouge_types (unitxt.metrics.Rouge attribute) (unitxt.metrics.RougeHF attribute) RougeHF (class in unitxt.metrics) round_floats() (in module unitxt.test_utils.metrics) row_indices (unitxt.struct_data_operators.DuplicateTableRows attribute) run_evaluation() (in module unitxt.evaluate_cli) run_inference() (in module unitxt.evaluate_cli) run_query() (in module unitxt.text2sql_utils) run_with_imap() (in module unitxt.inference)

S

SacreBleuStats (class in unitxt.metrics) safe_eval() (in module unitxt.utils) safe_load_json() (in module unitxt.ui.load_catalog_data) SafetyMetric (class in unitxt.metrics) Sampler (class in unitxt.splitters) save() (unitxt.artifact.Artifact method) save_temporary_template() (in module unitxt.ui.load_catalog_data) save_to_file() (in module unitxt.utils) scaled_fields (unitxt.metrics.Squad attribute) ScaleNumberToZeroOneReturnZeroIfFails (class in unitxt.processors) score (unitxt.metric_utils.GlobalScores property) score_based_confidence_interval() (unitxt.metrics.MetricWithConfidenceInterval method) score_name (unitxt.metric_utils.GlobalScores property) select() (in module unitxt.api) (unitxt.inference.OptionSelectingByLogProbsInferenceEngine method) select_from (unitxt.operators.Perturb attribute) SelectFields (class in unitxt.operators) SentenceBert (class in unitxt.metrics) separate_inside_and_outside_square_brackets() (in module unitxt.parsing_utils) SeparateSplit (class in unitxt.splitters) SequentialMixin (class in unitxt.operator) SequentialOperator (class in unitxt.operator) SequentialOperatorInitializer (class in unitxt.operator) SequentialRecipe (class in unitxt.recipe) SequentialSuccess (class in unitxt.metrics) serialize() (unitxt.artifact.Artifact method) serialized_type (unitxt.serializers.ConversationSerializer attribute) (unitxt.serializers.DictAsJsonSerializer attribute) (unitxt.serializers.DocumentSerializer attribute) (unitxt.serializers.ImageSerializer attribute) (unitxt.serializers.ListSerializer attribute) (unitxt.serializers.SQLDatabaseAsSchemaSerializer attribute) (unitxt.serializers.TableSerializer attribute) (unitxt.serializers.ToolCallSerializer attribute) (unitxt.serializers.ToolsSerializer attribute) serialized_type() (unitxt.serializers.DialogSerializer method) (unitxt.serializers.MultiDocumentSerializer method) (unitxt.serializers.NumberQuantizingSerializer method) (unitxt.serializers.NumberSerializer method) (unitxt.serializers.VideoSerializer method) SerializeDialog (class in unitxt.dialog_operators) SerializeKeyValPairs (class in unitxt.struct_data_operators) SerializeOpenAiFormatDialog (class in unitxt.dialog_operators) Serializer (class in unitxt.serializers) SerializeTable (class in unitxt.struct_data_operators) SerializeTableAsConcatenation (class in unitxt.struct_data_operators) SerializeTableAsDFLoader (class in unitxt.struct_data_operators) SerializeTableAsHTML (class in unitxt.struct_data_operators) SerializeTableAsImage (class in unitxt.struct_data_operators) SerializeTableAsIndexedRowMajor (class in unitxt.struct_data_operators) SerializeTableAsJson (class in unitxt.struct_data_operators) SerializeTableAsMarkdown (class in unitxt.struct_data_operators) SerializeTableRowAsList (class in unitxt.struct_data_operators) SerializeTableRowAsText (class in unitxt.struct_data_operators) SerializeTriples (class in unitxt.struct_data_operators) Set (class in unitxt.operators) set_caching() (unitxt.stream.MultiStream method) set_confidence_interval_calculation() (unitxt.metrics.RemoteMetric method) set_copying() (unitxt.stream.DynamicStream method) (unitxt.stream.GeneratorStream method) (unitxt.stream.ListStream method) (unitxt.stream.MultiStream method) (unitxt.stream.Stream method) set_n_resamples() (unitxt.metrics.RemoteMetric method) set_values() (in module unitxt.dict_utils) set_verbosity() (in module unitxt.logging_utils) Settings (class in unitxt.settings_utils) setup_logging() (in module unitxt.evaluate_cli) setup_parser() (in module unitxt.evaluate_cli) shallow_copy() (in module unitxt.utils) short_hex_hash() (in module unitxt.api) Shuffle (class in unitxt.operators) shuffle_columns() (in module unitxt.struct_data_operators) shuffle_rows() (in module unitxt.struct_data_operators) ShuffleColumnsNames (class in unitxt.struct_data_operators) ShuffleFieldValues (class in unitxt.operators) ShuffleTableColumns (class in unitxt.struct_data_operators) ShuffleTableRows (class in unitxt.struct_data_operators) SideEffectOperator (class in unitxt.operator) SingleStreamOperator (class in unitxt.operator) SingleStreamReducer (class in unitxt.operator) Singleton (class in unitxt.utils) SingleTypeSerializer (class in unitxt.serializers) SizeLimitedFormatter (class in unitxt.logging_utils) Slice (class in unitxt.collections_operators) slice_stream() (in module unitxt.split_utils)

slice_streams() (in module unitxt.split_utils) SliceSplit (class in unitxt.splitters) SOCIAL_BIAS_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) SOCIAL_BIAS_USER_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) SourceOperator (class in unitxt.operator) SourceSequentialOperator (class in unitxt.operator) SpanLabelingBaseTemplate (class in unitxt.templates) SpanLabelingJsonTemplate (class in unitxt.templates) SpanLabelingTemplate (class in unitxt.templates) Spearmanr (class in unitxt.metrics) Split (class in unitxt.string_operators) split (unitxt.stream_operators.DuplicateSplit attribute) split_words() (in module unitxt.text_utils) SplitByNestedGroup (class in unitxt.operators) SplitRandomMix (class in unitxt.splitters) splits (unitxt.loaders.LoadFromSklearn attribute) (unitxt.stream_operators.DeleteSplits attribute) SplitStrip (class in unitxt.processors) SplitSubsetsAndGroups (class in unitxt.metric_utils) Splitter (class in unitxt.splitters) sql_exact_match() (in module unitxt.text2sql_utils) SQLDatabase (class in unitxt.types) SQLDatabaseAsSchemaSerializer (class in unitxt.serializers) SQLExecutionAccuracy (class in unitxt.metrics) SQLExecutionLogicAccuracy (class in unitxt.metrics) SQLExecutionResult (class in unitxt.text2sql_utils) sqlglot_optimized_equivalence() (in module unitxt.text2sql_utils) sqlglot_parsed_queries_equivalent() (in module unitxt.text2sql_utils) SQLNonExecutionAccuracy (class in unitxt.metrics) SQLNonExecutionMetricResult (class in unitxt.text2sql_utils) sqlparse_queries_equivalent() (in module unitxt.text2sql_utils) Squad (class in unitxt.metrics) StanceToProCon (class in unitxt.processors) StandardAPIParamsMixin (class in unitxt.inference) StandardSchema (class in unitxt.validate) Statistic (class in unitxt.metrics) std() (unitxt.metrics.Statistic method) STEP_BY_STEP_BAD_FINAL_ANSWER (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_NON_COHERENT (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_ARITHMETIC (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_BAD_GRAMMAR (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_COHERENCY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_COMMONSENSE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_CONTRADICTION (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_HALLUCINATION (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_MISSING_STEPS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_NON_FACTUAL (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_OVERALL_QUALITY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_REDUNDANCY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) STEP_BY_STEP_REASONING_REPETITION (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) Stream (class in unitxt.stream) stream_name_to_origin_subset_group() (in module unitxt.metric_utils) stream_single() (in module unitxt.operator) StreamingOperator (class in unitxt.operator) StreamInitializerOperator (class in unitxt.operator) StreamOperator (class in unitxt.operator) StreamRefiner (class in unitxt.operators) StringContainment (class in unitxt.metrics) StringContainmentOld (class in unitxt.metrics) StringContainmentRatio (class in unitxt.metrics) StringEquals (class in unitxt.processors) Strip (class in unitxt.string_operators) strip_alias() (in module unitxt.text2sql_utils) strip_system_prompt_and_format_from_inputs (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) StrToFloatFormat (class in unitxt.processors) strtype() (in module unitxt.type_utils) STRUCTURE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) subsets_scores (unitxt.metric_utils.EvaluationResults property) SubsetsScores (class in unitxt.metric_utils) Substring (class in unitxt.processors) suffixes (unitxt.augmentors.AugmentPrefixSuffix attribute) SUMMARIZATION_COHERENCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) SUMMARIZATION_FLUENCY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) SUMMARIZATION_INFORMATIVENESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) SUMMARIZATION_PREFERENCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) SUMMARIZATION_RELEVANCE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) summarize_cli() (in module unitxt.evaluate_cli) summary (unitxt.metric_utils.GlobalScores property) (unitxt.metric_utils.GroupsScores property) (unitxt.metric_utils.InstanceScores property) (unitxt.metric_utils.SubsetsScores property) summary() (in module unitxt.catalog) system_prompt (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) SystemFormat (class in unitxt.formats) SystemPrompt (class in unitxt.system_prompts)

T

Table (class in unitxt.types) TableSerializer (class in unitxt.serializers) take() (unitxt.stream.DynamicStream method) (unitxt.stream.GeneratorStream method) (unitxt.stream.ListStream method) (unitxt.stream.Stream method) TakeByField (class in unitxt.operators) TakeFirstNonEmptyLine (class in unitxt.processors) TakeFirstWord (class in unitxt.processors) TakeLastNonEmptyLine (class in unitxt.processors) Task (class in unitxt.task) task (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) TaskBasedLLMasJudge (class in unitxt.llm_as_judge_from_template) TaskCard (class in unitxt.card) TaskInputsAugmentor (class in unitxt.augmentors) temp_catalog() (in module unitxt.test_utils.catalog) TEMPERATURE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) (unitxt.llm_as_judge_constants.PairwiseCriteriaCatalogEnum attribute) Template (class in unitxt.templates) template (unitxt.llm_as_judge_from_template.LLMAsJudge attribute) (unitxt.llm_as_judge_from_template.LLMAsJudgeBase attribute) TemplateFormatKeyError TemplatesDict (class in unitxt.templates) TemplatesList (class in unitxt.templates) test_adding_to_catalog() (in module unitxt.test_utils.card) test_artfifact_saving_and_loading() (in module unitxt.test_utils.artifact) test_card() (in module unitxt.test_utils.card) test_correct_predictions() (in module unitxt.test_utils.card) test_evaluate() (in module unitxt.test_utils.metrics) test_loading_from_catalog() (in module unitxt.test_utils.card) test_metric() (in module unitxt.test_utils.metrics) test_metrics_exist() (in module unitxt.test_utils.card) test_wrong_predictions() (in module unitxt.test_utils.card) TextAugmentor (class in unitxt.augmentors) TextContent (class in unitxt.formats) TextGenerationInferenceOutput (class in unitxt.inference) TextTurn (class in unitxt.types) TextualInstruction (class in unitxt.instructions) TextualSystemPrompt (class in unitxt.system_prompts) to_dataset() (unitxt.stream.MultiStream method) (unitxt.stream.Stream method) to_df() (unitxt.metric_utils.GlobalScores method) (unitxt.metric_utils.InstanceScores method) to_dict() (in module unitxt.dataclass) (unitxt.dataclass.Dataclass method) to_float_or_default() (in module unitxt.type_utils)

to_iterable_dataset() (unitxt.stream.MultiStream method) (unitxt.stream.Stream method) to_json() (unitxt.artifact.Artifact method) to_markdown() (unitxt.metric_utils.InstanceScores method) to_pretty_string() (in module unitxt.text_utils) to_split (unitxt.stream_operators.DuplicateSplit attribute) to_tools() (unitxt.inference.WMLInferenceEngineChat method) to_type_dict() (in module unitxt.type_utils) to_type_string() (in module unitxt.type_utils) to_yaml() (unitxt.artifact.Artifact method) ToDialog (class in unitxt.dialog_operators) TogetherAiInferenceEngine (class in unitxt.inference) TogetherAiInferenceEngineParamsMixin (class in unitxt.inference) ToImage (class in unitxt.image_operators) TokenOverlap (class in unitxt.metrics) TokensSlice (class in unitxt.string_operators) TokensSplit (class in unitxt.string_operators) ToListByComma (class in unitxt.processors) ToListByCommaSpace (class in unitxt.processors) Tool (class in unitxt.types) ToolCall (class in unitxt.types) ToolCallContext (class in unitxt.types) ToolCallingMetric (class in unitxt.metrics) ToolCallKeyValueExtraction (class in unitxt.metrics) ToolCallPostProcessor (class in unitxt.struct_data_operators) ToolCallSerializer (class in unitxt.serializers) ToolCallTurn (class in unitxt.types) ToolOutputTurn (class in unitxt.types) ToolsSerializer (class in unitxt.serializers) TorchDeviceMixin (class in unitxt.inference) ToRGB (class in unitxt.image_operators) ToString (class in unitxt.processors) ToStringStripped (class in unitxt.processors) TOXICITY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) ToYesOrNone (class in unitxt.processors) transform_dialog_to_standard_format() (unitxt.dialog_operators.SerializeOpenAiFormatDialog method) TRANSLATION_QUALITY (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) TransposeTable (class in unitxt.struct_data_operators) truncate_cell() (in module unitxt.struct_data_operators) TruncateTableCells (class in unitxt.struct_data_operators) TruncateTableRows (class in unitxt.struct_data_operators) TRUTHFULNESS (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) try_parse_json() (in module unitxt.evaluate_cli) TURLColumnTypeAnnotationLoader (class in unitxt.loaders) TypeDependentAugmentor (class in unitxt.augmentors) TypeMismatchError

U

Undefined (class in unitxt.dataclass) UNETHICAL_BEHAVIOR_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) UNETHICAL_BEHAVIOR_USER_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) UnexpectedArgumentError UnexpectedHttpCodeError unitxt module, [1] unitxt.api module unitxt.artifact module unitxt.assistant module unitxt.assistant.assessment module unitxt.assistant.assessment.my_pretty_table module unitxt.augmentors module unitxt.base_metric module unitxt.benchmark module unitxt.blocks module unitxt.card module unitxt.catalog module unitxt.collections module unitxt.collections_operators module unitxt.dataclass module unitxt.dataset module unitxt.dataset_utils module unitxt.deprecation_utils module unitxt.dialog_operators module unitxt.dict_utils module unitxt.error_utils module unitxt.eval_utils module unitxt.evaluate_cli module unitxt.file_utils module unitxt.formats module unitxt.fusion module unitxt.generator_utils module unitxt.hf_utils module unitxt.image_operators module unitxt.inference module unitxt.instructions module unitxt.llm_as_judge module unitxt.llm_as_judge_chat_templates module unitxt.llm_as_judge_constants module unitxt.llm_as_judge_from_template module unitxt.llm_as_judge_operators module unitxt.llm_as_judge_utils module unitxt.loaders module unitxt.logging_utils module unitxt.metric module unitxt.metric_utils module unitxt.metrics module unitxt.normalizers module unitxt.operator module

unitxt.operators module unitxt.parsing_utils module unitxt.processors module unitxt.random_utils module unitxt.recipe module unitxt.register module unitxt.schema module unitxt.serializers module unitxt.service module unitxt.service.metrics module unitxt.settings_utils module unitxt.span_lableing_operators module unitxt.split_utils module unitxt.splitters module unitxt.standard module unitxt.stream module unitxt.stream_operators module unitxt.string_operators module unitxt.struct_data_operators module unitxt.system_prompts module unitxt.task module unitxt.templates module unitxt.test_utils module unitxt.test_utils.artifact module unitxt.test_utils.card module unitxt.test_utils.catalog module unitxt.test_utils.metrics module unitxt.test_utils.operators module unitxt.test_utils.storage module unitxt.text2sql_utils module unitxt.text_utils module unitxt.type_utils module unitxt.types module unitxt.ui module unitxt.ui.load_catalog_data module unitxt.ui.settings module unitxt.utils module unitxt.validate module unitxt.version module UnitxtArtifactNotFoundError UnitxtError UnitxtUnverifiedCodeError UnitxtVersionsConflictError UnitxtWarning (class in unitxt.error_utils) UnrecognizedArtifactTypeError unregister() (unitxt.artifact.Catalogs method) unregister_local_catalog() (in module unitxt.register) UnsortedListExactMatch (class in unitxt.metrics) UnsupportedTypeError update_init_signature() (unitxt.dataclass.DataclassMeta method) update_scores_of_stream_instances() (in module unitxt.operators) UpdateStream (class in unitxt.metrics) Upper (class in unitxt.processors) use_demos (unitxt.standard.DatasetRecipe property)

V

validate_openai_dialog_format() (unitxt.dialog_operators.SerializeOpenAiFormatDialog static method) validate_query_and_break_to_components() (in module unitxt.dict_utils) validate_subgroup_types() (in module unitxt.metrics) ValidateSchema (class in unitxt.validate) ValidateStandardSchema (class in unitxt.validate) Validator (class in unitxt.validate) verbosed_fetch_artifact() (in module unitxt.artifact) verify() (unitxt.artifact.Artifact method) verify_artifact_dict() (unitxt.artifact.Artifact class method) verify_data_classification_policy() (unitxt.artifact.Artifact method) verify_instance() (unitxt.artifact.Artifact method)

verify_legal_catalog_name() (in module unitxt.artifact) verify_messages() (unitxt.inference.WMLInferenceEngineChat static method) verify_required_schema() (in module unitxt.type_utils) verify_versions_compatibility() (in module unitxt.hf_utils) VersionConflict VideoSerializer (class in unitxt.serializers) VIOLENCE_ASSISTANT_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) VIOLENCE_USER_MESSAGE (unitxt.llm_as_judge_constants.DirectCriteriaCatalogEnum attribute) VLLMInferenceEngine (class in unitxt.inference) VLLMParamsMixin (class in unitxt.inference) VLLMRemoteInferenceEngine (class in unitxt.inference)

W

warn_on_call() (in module unitxt.deprecation_utils) WebsrcSquadF1 (class in unitxt.metrics) WeightedFusion (class in unitxt.fusion) WeightedWinRateCorrelation (class in unitxt.metrics) Wer (class in unitxt.metrics) WikipediaFetcher (class in unitxt.operators) wml_params (unitxt.metrics.GraniteGuardianBase attribute)

WMLChatParamsMixin (class in unitxt.inference) WMLGenerationParamsMixin (class in unitxt.inference) WMLInferenceEngineBase (class in unitxt.inference) WMLInferenceEngineChat (class in unitxt.inference) WMLInferenceEngineGeneration (class in unitxt.inference) Wrap (class in unitxt.collections_operators) wrap_inner_metric_pipeline_metric() (unitxt.metrics.RemoteMetric static method)

Y

YesNoTemplate (class in unitxt.templates)

YesNoToInt (class in unitxt.processors) YesToOneElseZero (class in unitxt.processors)

Z

Zip (class in unitxt.collections_operators)

ZipFieldValues (class in unitxt.operators) ZipLongest (class in unitxt.collections_operators)