U
    <Af                     @   sB   d dl Z d dlmZmZmZmZ er0d dlmZ G dd dZdS )    N)TYPE_CHECKINGAnyDictUnion)SentenceTransformerc                
   @   s   e Zd ZdZdd Zddeeeeee	eef f ddd	Z
e	eef ee	eef d
ddZde	eef ddddZeedddZdS )SentenceEvaluatorzl
    Base class for all evaluators

    Extend this class and implement __call__ for custom evaluators.
    c                 C   s   d| _ d| _dS )a  
        Base class for all evaluators. Notably, this class introduces the ``greater_is_better`` and ``primary_metric``
        attributes. The former is a boolean indicating whether a higher evaluation score is better, which is used
        for choosing the best checkpoint if ``load_best_model_at_end`` is set to ``True`` in the training arguments.

        The latter is a string indicating the primary metric for the evaluator. This has to be defined whenever
        the evaluator returns a dictionary of metrics, and the primary metric is the key pointing to the primary
        metric, i.e. the one that is used for model selection and/or logging.
        TN)Zgreater_is_betterprimary_metric)self r
   V/tmp/pip-unpacked-wheel-i7fohqg6/sentence_transformers/evaluation/SentenceEvaluator.py__init__   s    
zSentenceEvaluator.__init__Nr   )modeloutput_pathepochstepsreturnc                 C   s   dS )a  
        This is called during training to evaluate the model.
        It returns a score for the evaluation with a higher score indicating a better result.

        Args:
            model: the model to evaluate
            output_path: path where predictions and metrics are written
                to
            epoch: the epoch where the evaluation takes place. This is
                used for the file prefixes. If this is -1, then we
                assume evaluation on test data.
            steps: the steps in the current epoch at time of the
                evaluation. This is used for the file prefixes. If this
                is -1, then we assume evaluation at the end of the
                epoch.

        Returns:
            Either a score for the evaluation with a higher score
            indicating a better result, or a dictionary with scores. If
            the latter is chosen, then `evaluator.primary_metric` must
            be defined
        Nr
   )r	   r   r   r   r   r
   r
   r   __call__   s    zSentenceEvaluator.__call__)metricsnamer   c                    sL    s|S  fdd|  D }t| drH| j d sH d | j | _|S )Nc                    s   i | ]\}} d  | |qS )_r
   ).0keyvaluer   r
   r   
<dictcomp>:   s     
 z<SentenceEvaluator.prefix_name_to_metrics.<locals>.<dictcomp>r   r   )itemshasattrr   
startswith)r	   r   r   r
   r   r   prefix_name_to_metrics7   s    z(SentenceEvaluator.prefix_name_to_metrics)r   r   r   c                 C   s   |j | | d S )N)Zmodel_card_dataZset_evaluation_metrics)r	   r   r   r
   r
   r    store_metrics_in_model_card_data?   s    z2SentenceEvaluator.store_metrics_in_model_card_data)r   c                 C   sF   | j j}z|d}|d| }W n tk
r6   Y nX tdd|S )z
        Returns a human-readable description of the evaluator: BinaryClassificationEvaluator -> Binary Classification

        1. Remove "Evaluator" from the class name
        2. Add a space before every capital letter
        Z	EvaluatorNz([a-z])([A-Z])z\g<1> \g<2>)	__class____name__index
IndexErrorresub)r	   
class_namer#   r
   r
   r   descriptionB   s    
zSentenceEvaluator.description)Nr   r   )r"   
__module____qualname____doc__r   strintr   floatr   r   r   r   r    propertyr(   r
   r
   r
   r   r      s            "r   )	r%   typingr   r   r   r   Z)sentence_transformers.SentenceTransformerr   r   r
   r
   r
   r   <module>   s   