U 4Af1@sdZddlmZddlZddlZddlmZddlmZm Z m Z ddl Z ddl ZddlmZddlmZmZmZmZmZmZdd lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/d d l0m1Z1e.2e3Z4dZ5dZ6Gddde"j7j8Z9Gddde"j7j8Z:Gddde"j7j8Z;Gddde"j7j8ZGddde"j7j8Z?Gddde"j7j8Z@Gd d!d!e"j7j8ZAGd"d#d#e"j7j8ZBGd$d%d%e"j7j8ZCGd&d'd'eZDe#Gd(d)d)e"j7j8ZEeGd*d+d+e*ZFd,ZGd-ZHe,d.eGGd/d0d0eDZIe,d1eGGd2d3d3eDZJGd4d5d5e"j7j8ZKe,d6eGGd7d8d8eDeZLGd9d:d:e"j7j8ZMe,d;eGGdeGGd?d@d@eDeZOe,dAeGGdBdCdCeDe ZPe,dDeGGdEdFdFeDeZQdS)GzTF Electra model.) annotationsN) dataclass)OptionalTupleUnion)get_tf_activation)+TFBaseModelOutputWithPastAndCrossAttentionsTFMaskedLMOutputTFMultipleChoiceModelOutputTFQuestionAnsweringModelOutputTFSequenceClassifierOutputTFTokenClassifierOutput) TFMaskedLanguageModelingLossTFModelInputTypeTFMultipleChoiceLossTFPreTrainedModelTFQuestionAnsweringLossTFSequenceClassificationLossTFSequenceSummaryTFTokenClassificationLossget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax) ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) ElectraConfigz"google/electra-small-discriminatorr%c s^eZdZddfdd Zdddddd Zddddddd d d d d ddZdddZZS)TFElectraSelfAttentionr%configc stjf||j|jdkr8td|jd|jd|j|_t|j|j|_|j|j|_t |j|_ t j j |jt|jdd|_t j j |jt|jdd|_t j j |jt|jdd|_t j j|jd |_|j|_||_dS) NrzThe hidden size (z6) is not a multiple of the number of attention heads ()queryunitskernel_initializernamekeyvalueZrate)super__init__ hidden_sizenum_attention_heads ValueErrorintattention_head_size all_head_sizemathsqrtsqrt_att_head_sizerlayersDenserinitializer_ranger*r/r0DropoutZattention_probs_dropout_probdropout is_decoderr(selfr(kwargs __class__S/tmp/pip-unpacked-wheel-zw5xktn0/transformers/models/electra/modeling_tf_electra.pyr3Fs6zTFElectraSelfAttention.__init__ tf.Tensorr7)tensor batch_sizereturncCs0tj||d|j|jfd}tj|ddddgdS)NrKshaperr$rperm)tfreshaper5r8 transpose)rDrKrLrHrHrItranspose_for_scoresbsz+TFElectraSelfAttention.transpose_for_scoresFTuple[tf.Tensor]bool hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionstrainingrMc Cst|d} |j|d} |dk } | rB|dk rB|d} |d} |}n| rt||j|d| } ||j|d| } |}n|dk r||j|d| } ||j|d| } tj|d| gdd} tj|d| gdd} n(||j|d| } ||j|d| } || | }|jr| | f}tj|| dd}tj |j |j d}t ||}|dk rZt ||}t|d d }|j||d }|dk rt||}t|| }tj|dddd gd }tj|| d |jfd}|r||fn|f}|jr||f}|S)Nrinputsr$rQaxisT) transpose_bdtyperN)logitsrfrdrbrrRrO)rr*rWr/r0rTconcatrBmatmulcastr<ridivideaddrrAmultiplyrVrUr9)rDr[r\r]r^r_r`rarbrLZmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layerZattention_scoresZdkZattention_probsattention_outputoutputsrHrHrIcallisJ            zTFElectraSelfAttention.callNc Cs|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXt|dddk rt|j j|j dd|jjgW5QRXt|dddk rt|j j|j dd|jjgW5QRXdS)NTr*r/r0) builtgetattrrT name_scoper*r.buildr(r4r/r0rD input_shaperHrHrIrxs  zTFElectraSelfAttention.build)F)N)__name__ __module__ __qualname__r3rWrtrx __classcell__rHrHrFrIr&Es  Qr&csBeZdZddfdd Zddddddd d Zdd d ZZS)TFElectraSelfOutputr%r'c sZtjf|tjj|jt|jdd|_tjj |j dd|_ tjj |j d|_||_dSNdenser+ LayerNormepsilonr.r1r2r3rr=r>r4rr?rLayerNormalizationlayer_norm_epsrr@hidden_dropout_probrAr(rCrFrHrIr3szTFElectraSelfOutput.__init__FrJrYr[ input_tensorrbrMcCs.|j|d}|j||d}|j||d}|SNrcrkrrArrDr[rrbrHrHrIrts zTFElectraSelfOutput.callNc Cs|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXt|dddk rt|j j|j dd|jjgW5QRXdSNTrr) rurvrTrwrr.rxr(r4rryrHrHrIrxs zTFElectraSelfOutput.build)F)Nr{r|r}r3rtrxr~rHrHrFrIrs rc sTeZdZddfdd ZddZddddddd d d d d d d ZdddZZS)TFElectraAttentionr%r'c s.tjf|t|dd|_t|dd|_dS)NrDr.output)r2r3r&self_attentionr dense_outputrCrFrHrIr3szTFElectraAttention.__init__cCstdSNNotImplementedError)rDZheadsrHrHrI prune_headsszTFElectraAttention.prune_headsFrJrXrY) rr\r]r^r_r`rarbrMc CsD|j||||||||d} |j| d||d} | f| dd} | S)Nr[r\r]r^r_r`rarbrr[rrbr$)rr) rDrr\r]r^r_r`rarbZ self_outputsrrrsrHrHrIrts"  zTFElectraAttention.callNc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXdS)NTrr)rurvrTrwrr.rxrryrHrHrIrxszTFElectraAttention.build)F)N)r{r|r}r3rrtrxr~rHrHrFrIrs  rcs<eZdZddfdd ZdddddZd d d ZZS) TFElectraIntermediater%r'c sVtjf|tjj|jt|jdd|_t |j t rDt |j |_ n|j |_ ||_dS)Nrr+)r2r3rr=r>intermediate_sizerr?r isinstance hidden_actstrrintermediate_act_fnr(rCrFrHrIr3s zTFElectraIntermediate.__init__rJr[rMcCs|j|d}||}|S)Nrc)rr)rDr[rHrHrIrt+s  zTFElectraIntermediate.callNc CsT|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXdSNTr rurvrTrwrr.rxr(r4ryrHrHrIrx1s zTFElectraIntermediate.build)NrrHrHrFrIrs rcsBeZdZddfdd Zddddddd d Zdd d ZZS)TFElectraOutputr%r'c sZtjf|tjj|jt|jdd|_tjj |j dd|_ tjj |j d|_||_dSrrrCrFrHrIr3<szTFElectraOutput.__init__FrJrYrcCs.|j|d}|j||d}|j||d}|SrrrrHrHrIrtFs zTFElectraOutput.callNc Cs|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXt|dddk rt|j j|j dd|jj gW5QRXdSr) rurvrTrwrr.rxr(rrr4ryrHrHrIrxMs zTFElectraOutput.build)F)NrrHrHrFrIr;s rc sLeZdZddfdd Zdddddddd d d d d d ZdddZZS)TFElectraLayerr%r'c sttjf|t|dd|_|j|_|j|_|jrT|jsFt|dt|dd|_t|dd|_ t |dd|_ dS)N attentionrz> should be used as a decoder model if cross attention is addedcrossattention intermediater) r2r3rrrBadd_cross_attentionr6rrrr bert_outputrCrFrHrIr3[szTFElectraLayer.__init__FrJtf.Tensor | NoneTuple[tf.Tensor] | NonerYrXrZc  Cs$|dk r|ddnd} |j|||dd| ||d} | d} |jrV| dd} | d} n | dd} d}|jr|dk rt|dstd|d|dk r|d dnd}|j| |||||||d}|d} | |dd} |d}| |} |j| d }|j|| |d }|f| } |jr | | f} | S) NrQ)rr\r]r^r_r`rarbrr$rNrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r[r)rrBhasattrr6rrr)rDr[r\r]r^r_r`rarbZself_attn_past_key_valueZself_attention_outputsrrrsZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputsZintermediate_outputZ layer_outputrHrHrIrths^           zTFElectraLayer.callNc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXt|dddk rt|jj|jdW5QRXt|dddk rt|j j|j dW5QRXdS)NTrrrr) rurvrTrwrr.rxrrrryrHrHrIrxszTFElectraLayer.build)F)NrrHrHrFrIrZs GrcsReZdZddfdd Zdddddddd d d d d d d d dZdddZZS)TFElectraEncoderr%r'c s2tjf||_fddtjD|_dS)Ncsg|]}td|dqS)zlayer_._r)r).0ir'rHrI sz-TFElectraEncoder.__init__..)r2r3r(rangenum_hidden_layerslayerrCrFr'rIr3szTFElectraEncoder.__init__FrJrzTuple[Tuple[tf.Tensor]] | NoneOptional[bool]rYDUnion[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]]) r[r\r]r^r_past_key_values use_cacheraoutput_hidden_states return_dictrbrMc  Cs| rdnd} |rdnd} |r(|jjr(dnd}|r4dnd}t|jD]\}}| rX| |f} |dk rh||nd}|||||||||| d}|d}|r||df7}|rB| |df} |jjrB|dk rB||df}qB| r| |f} | stdd|| | |fDSt||| | |d S) NrHrrrNr$rQcss|]}|dk r|VqdSrrH)rvrHrHrI sz(TFElectraEncoder.call..)Zlast_hidden_staterr[ attentionsZcross_attentions)r(r enumeratertupler )rDr[r\r]r^r_rrrarrrbZall_hidden_statesZall_attentionsZall_cross_attentionsZnext_decoder_cacherZ layer_moduler`Z layer_outputsrHrHrIrtsL       zTFElectraEncoder.callNc CsR|jr dSd|_t|dddk rN|jD]&}t|j|dW5QRXq&dS)NTr)rurvrrTrwr.rx)rDrzrrHrHrIrxs zTFElectraEncoder.build)F)NrrHrHrFrIrs&>rcs<eZdZddfdd ZdddddZd d d ZZS) TFElectraPoolerr%r'c s6tjf|tjj|jt|jddd|_||_ dS)Ntanhr)r,r- activationr.) r2r3rr=r>r4rr?rr(rCrFrHrIr3szTFElectraPooler.__init__rJrcCs |dddf}|j|d}|S)Nrrc)r)rDr[Zfirst_token_tensorZ pooled_outputrHrHrIrts zTFElectraPooler.callNc CsT|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXdSrrryrHrHrIrx%s zTFElectraPooler.build)NrrHrHrFrIrs rcsJeZdZdZddfdd ZdddZdd d d d d d d ddZZS)TFElectraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.r%r'c sVtjf|||_|j|_|j|_|j|_tjj|j dd|_ tjj |j d|_ dS)Nrrr1)r2r3r(embedding_sizemax_position_embeddingsr?rr=rrrr@rrArCrFrHrIr32szTFElectraEmbeddings.__init__Nc Cstd(|jd|jj|jgt|jd|_W5QRXtd(|jd|jj |jgt|jd|_ W5QRXtd&|jd|j |jgt|jd|_ W5QRX|j rdSd|_ t|dddk rt|jj|jdd|jjgW5QRXdS) NZword_embeddingsweight)r.rP initializertoken_type_embeddings embeddingsposition_embeddingsTr)rTrw add_weightr( vocab_sizerrr?rZtype_vocab_sizerrrrurvrr.rxryrHrHrIrx<s0      zTFElectraEmbeddings.buildrFrJrY) input_ids position_idstoken_type_ids inputs_embedsrbrMc Cs|dkr|dkrtd|dk r>t||jjtj|j|d}t|dd}|dkrdtj|dd}|dkrtj tj ||d|ddd }tj|j |d}tj|j |d} ||| } |j | d } |j| |d } | S) z Applies embedding based on inputs tensor. Returns: final_embeddings (`tf.Tensor`): output embedding tensor. Nz5Need to provide either `input_ids` or `input_embeds`.)paramsindicesrNrZdimsr0r$)startlimitrercrk)r6rr(rrTZgatherrrfillZ expand_dimsrrrrrA) rDrrrrpast_key_values_lengthrbrzZposition_embedsZtoken_type_embedsZfinal_embeddingsrHrHrIrtZs&  zTFElectraEmbeddings.call)N)NNNNrF)r{r|r}__doc__r3rxrtr~rHrHrFrIr/s rcs0eZdZfddZd ddZd ddZZS) !TFElectraDiscriminatorPredictionsc s>tjf|tjj|jdd|_tjjddd|_||_dS)Nrrr$dense_prediction) r2r3rr=r>r4rrr(rCrFrHrIr3sz*TFElectraDiscriminatorPredictions.__init__FcCs0||}t|jj|}t||d}|S)NrN)rrr(rrTsqueezer)rDdiscriminator_hidden_statesrbr[rjrHrHrIrts z&TFElectraDiscriminatorPredictions.callNc Cs|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXt|dddk rt|j j|j dd|jjgW5QRXdS)NTrr) rurvrTrwrr.rxr(r4rryrHrHrIrxs z'TFElectraDiscriminatorPredictions.build)F)NrrHrHrFrIrs  rcs0eZdZfddZd ddZd ddZZS) TFElectraGeneratorPredictionsc s@tjf|tjj|jdd|_tjj|jdd|_ ||_ dS)Nrrrr) r2r3rr=rrrr>rrr(rCrFrHrIr3sz&TFElectraGeneratorPredictions.__init__FcCs$||}td|}||}|S)Ngelu)rrr)rDgenerator_hidden_statesrbr[rHrHrIrts   z"TFElectraGeneratorPredictions.callNc Cs|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXt|dddk rt|j j|j dd|jj gW5QRXdS)NTrr) rurvrTrwrr.rxr(rrr4ryrHrHrIrxs z#TFElectraGeneratorPredictions.build)F)NrrHrHrFrIrs  rc@s$eZdZdZeZdZdgZdgZdS)TFElectraPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. electrazgenerator_lm_head.weightrAN) r{r|r}rr% config_classZbase_model_prefixZ"_keys_to_ignore_on_load_unexpectedZ_keys_to_ignore_on_load_missingrHrHrHrIrs rcseZdZeZfddZddZddZddZdd d Z d d Z e dddddddddddddddddddZ dddZ ZS)TFElectraMainLayerc s\tjf|||_|j|_t|dd|_|j|jkrJtj j |jdd|_ t |dd|_ dS)Nrrembeddings_projectencoder)r2r3r(rBrrrr4rr=r>rrrrCrFrHrIr3s zTFElectraMainLayer.__init__cCs|jSr)rrDrHrHrIget_input_embeddingssz'TFElectraMainLayer.get_input_embeddingscCs||j_t|d|j_dSNr)rrrrrDr0rHrHrIset_input_embeddingssz'TFElectraMainLayer.set_input_embeddingscCstdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel Nr)rDZheads_to_prunerHrHrI _prune_headsszTFElectraMainLayer._prune_headsrc CsX|\}}|dkr&tj|||fdd}t|}||}|jrt|} tt| ddddf||df| ddddf} tj| |jd} | |dddddf} t| }t | |dd|d|df} |dkr| dddd| dddf} nt ||ddd|df} tj| |d} tj d|d} tj d|d} t t | | | } | S)Nr$rrhrrQ?) rTrrrBrZ less_equalZtilernrirUZconstantrqsubtract)rDr\rzrirrL seq_lengthZattention_mask_shapeZmask_seq_lengthZseq_idsZ causal_maskextended_attention_maskZone_cstZten_thousand_cstrHrHrIget_extended_attention_masks: $ z.TFElectraMainLayer.get_extended_attention_maskcCs |dk rtndg|jj}|Sr)rr(r)rDr]rHrHrI get_head_masksz TFElectraMainLayer.get_head_maskNFTFModelInputType | Nonenp.ndarray | tf.Tensor | None4Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]rrrr\rrr]rr^r_rrrarrrbrMc Cs|jjs d} |dk r&|dk r&tdn4|dk r8t|}n"|dk rRt|dd}ntd|\}}| dkrd}dgt|jj} nt| ddd}|dkrtj|||fdd}|dkrtj|dd}|j ||||||d }| |||j |}|jrv|dk rvtj ||j d }tt|}|d krF|dddddddf}|d krh|ddddddf}d |d}nd}| |}t|dr|j||d}|j|||||| | | | | |d }|S)NFzDYou cannot specify both input_ids and inputs_embeds at the same timerNz5You have to specify either input_ids or inputs_embedsrrr$r)rrrrrrbrhrrQrrrrb) r[r\r]r^r_rrrarrrb)r(rBr6rlenrrrTrrrrirnrrr)rDrr\rrr]rr^r_rrrarrrbrzrLrrr[rZnum_dims_encoder_attention_maskZencoder_extended_attention_maskrHrHrIrtst       zTFElectraMainLayer.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXt|dddk rt|jj|jdd|j j gW5QRXdS)NTrrr) rurvrTrwrr.rxrrr(rryrHrHrIrxszTFElectraMainLayer.build)r)NNNNNNNNNNNNNF)N)r{r|r}r%rr3rrrrrrrtrxr~rHrHrFrIrs0  1.arc@s6eZdZUdZdZded<dZded<dZded<dS)TFElectraForPreTrainingOutputa Output type of [`TFElectraForPreTraining`]. Args: loss (*optional*, returned when `labels` is provided, `tf.Tensor` of shape `(1,)`): Total loss of the ELECTRA objective. logits (`tf.Tensor` of shape `(batch_size, sequence_length)`): Prediction scores of the head (scores for each token before SoftMax). hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. NrJrjrr[r)r{r|r}rrj__annotations__r[rrHrHrHrIrs   ra} This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`ElectraConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). a]The bare Electra Model transformer outputting raw hidden-states without any specific head on top. Identical to the BERT model except that it uses an additional linear layer between the embedding layer and the encoder if the hidden size and embedding size are different. Both the generator and discriminator checkpoints may be loaded into this model.csreZdZfddZeeedee e e ddddddddddd d d d d d d d d dZ dddZ ZS)TFElectraModelcs&tj|f||t|dd|_dS)Nrr)r2r3rrrDr(rdrErFrHrIr3szTFElectraModel.__init__batch_size, sequence_length checkpoint output_typerNFrrrrrrcCs*|j|||||||| | || | | |d}|S)a encoder_hidden_states (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`) contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Set to `False` during training, `True` during generation )rr\rrr]r^r_rrrrarrrb)r)rDrr\rrr]rr^r_rrrarrrbrsrHrHrIrts",zTFElectraModel.callc CsJ|jr dSd|_t|dddk rFt|jj|jdW5QRXdS)NTr)rurvrTrwrr.rxryrHrHrIrxRs zTFElectraModel.build)NNNNNNNNNNNNNF)N)r{r|r}r3rr!ELECTRA_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrtrxr~rHrHrFrIrs0  28raH Electra model with a binary classification head on top as used during pretraining for identifying generated tokens. Even though both the discriminator and generator may be loaded into this model, the discriminator is the only model of the two to have the correct classification head to be used for this model. csheZdZfddZeeedee e ddddddddd d d d d d d d Z dddZ Z S)TFElectraForPreTrainingc s0tj|f|t|dd|_t|dd|_dS)Nrrdiscriminator_predictions)r2r3rrrrrCrFrHrIr3esz TFElectraForPreTraining.__init__r)rrNFrrrz6Union[TFElectraForPreTrainingOutput, Tuple[tf.Tensor]]) rr\rrr]rrarrrbrMc  CsX|j||||||||| | d } | d} || } | sF| f| ddSt| | j| jdS)a! Returns: Examples: ```python >>> import tensorflow as tf >>> from transformers import AutoTokenizer, TFElectraForPreTraining >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-small-discriminator") >>> model = TFElectraForPreTraining.from_pretrained("google/electra-small-discriminator") >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 >>> outputs = model(input_ids) >>> scores = outputs[0] ``` rr\rrr]rrarrrbrr$N)rjr[r)rrrr[r)rDrr\rrr]rrarrrbrdiscriminator_sequence_outputrjrHrHrIrtks*  zTFElectraForPreTraining.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXdS)NTrr)rurvrTrwrr.rxrryrHrHrIrxszTFElectraForPreTraining.build) NNNNNNNNNF)N)r{r|r}r3rr!rrr#rrrtrxr~rHrHrFrIr[s    *4rcsPeZdZfddZfddZddZddZd d Zd d Zd dZ Z S)TFElectraMaskedLMHeadc s&tjf|||_|j|_||_dSr)r2r3r(rinput_embeddings)rDr(r rErFrHrIr3szTFElectraMaskedLMHead.__init__cs*|j|jjfdddd|_t|dS)NzerosTbias)rPrZ trainabler.)rr(rr r2rxryrFrHrIrxszTFElectraMaskedLMHead.buildcCs|jSr)r rrHrHrIget_output_embeddingssz+TFElectraMaskedLMHead.get_output_embeddingscCs||j_t|d|j_dSr)r rrrrrHrHrIset_output_embeddingssz+TFElectraMaskedLMHead.set_output_embeddingscCs d|jiS)Nr )r rrHrHrIget_biasszTFElectraMaskedLMHead.get_biascCs"|d|_t|dd|j_dS)Nr r)r rr(rrrHrHrIset_biass zTFElectraMaskedLMHead.set_biascCsdt|dd}tj|d|jgd}tj||jjdd}tj|d||jjgd}tj j ||j d}|S)N)rKr$rNrOT)abrg)r0r ) rrTrUrrmr rr(rnnZbias_addr )rDr[rrHrHrIrts zTFElectraMaskedLMHead.call) r{r|r}r3rxrrrrrtr~rHrHrFrIr s  r z Electra model with a language modeling head on top. Even though both the discriminator and generator may be loaded into this model, the generator is the only model of the two to have been trained for the masked language modeling task. cseZdZfddZddZddZeee de de e d d d d dddddddddddddd ddZ dddZZS)TFElectraForMaskedLMc sltj|f|||_t|dd|_t|dd|_t|jt rLt |j|_ n|j|_ t ||jj dd|_dS)Nrrgenerator_predictionsgenerator_lm_head)r2r3r(rrrrrrrrrr rrrCrFrHrIr3s zTFElectraForMaskedLM.__init__cCs|jSr)rrrHrHrI get_lm_headsz TFElectraForMaskedLM.get_lm_headcCstdt|jd|jjS)NzMThe method get_prefix_bias_name is deprecated. Please use `get_bias` instead./)warningswarn FutureWarningr.rrrHrHrIget_prefix_bias_names z)TFElectraForMaskedLM.get_prefix_bias_namerzgoogle/electra-small-generatorz[MASK]z'paris'gQ?)rrrmaskexpected_output expected_lossNFrrrz)Union[TFMaskedLMOutput, Tuple[tf.Tensor]] rr\rrr]rrarrlabelsrbrMc  Cs|j||||||||| | d } | d} |j| | d}|j|| d}| dkrNdn || |}| s|f| dd}|dk r|f|S|St||| j| jdS)a labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` rrrNr$lossrjr[r)rrrhf_compute_lossr r[r)rDrr\rrr]rrarrr"rbrZgenerator_sequence_outputZprediction_scoresr$rrHrHrIrts2 zTFElectraForMaskedLM.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXt|dddk rt|jj|jdW5QRXdS)NTrrr) rurvrTrwrr.rxrrryrHrHrIrx-szTFElectraForMaskedLM.build) NNNNNNNNNNF)N)r{r|r}r3rrrr!rrrr rrtrxr~rHrHrFrIrs4   ,1rcs2eZdZdZfddZddZd ddZZS) TFElectraClassificationHeadz-Head for sentence-level classification tasks.c sttjf|tjj|jt|jdd|_|j dk r:|j n|j }tj ||_ tjj|jt|jdd|_||_dS)Nrr-r.out_proj)r2r3rr=r>r4rr?rclassifier_dropoutZ%classifhidden_dropout_probier_dropoutrr@rA num_labelsr(r(rDr(rEr)rFrHrIr3?s"z$TFElectraClassificationHead.__init__cKsN|dddddf}||}||}td|}||}||}|S)Nrr)rArrr()rDrdrExrHrHrIrtPs     z TFElectraClassificationHead.callNc Cs|jr dSd|_t|dddk rPt|jj|jdd|jjgW5QRXt|dddk rt|j j|j dd|jjgW5QRXdS)NTrr() rurvrTrwrr.rxr(r4r(ryrHrHrIrxZs z!TFElectraClassificationHead.build)N)r{r|r}rr3rtrxr~rHrHrFrIr&<s  r&z ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cspeZdZfddZeeedede e ddddd d d d d d d d d d d d d ddZ dddZ Z S)"TFElectraForSequenceClassificationcs<tj|f|||j|_t|dd|_t|dd|_dS)Nrr classifier)r2r3r*rrr&r.rrFrHrIr3nsz+TFElectraForSequenceClassification.__init__rz$bhadresh-savani/electra-base-emotionz'joy'gQ?rrrrr NFrrrz3Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]r!c  Cs|j||||||||| | d } || d} | dkr8dn || | }| sp| f| dd}|dk rl|f|S|St|| | j| jdS)a labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). rrNr$r#)rr.r%r r[r)rDrr\rrr]rrarrr"rbrsrjr$rrHrHrIrtts. z'TFElectraForSequenceClassification.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXdSNTrr.)rurvrTrwrr.rxr.ryrHrHrIrxsz(TFElectraForSequenceClassification.build) NNNNNNNNNNF)N)r{r|r}r3rr!rrrr rrtrxr~rHrHrFrIr-fs.   ,/r-z ELECTRA Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks. csleZdZfddZeeedee e e ddddddddd d d dd d d d d Z dddZ ZS)TFElectraForMultipleChoicecsXtj|f||t|dd|_t||jdd|_tjj dt |jdd|_ ||_ dS)Nrrsequence_summary)r?r.r$r.r') r2r3rrrr?r2rr=r>rr.r(rrFrHrIr3sz#TFElectraForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthrNFrrrz4Union[TFMultipleChoiceModelOutput, Tuple[tf.Tensor]]r!c  Cst|dk r"t|d} t|d} nt|d} t|d} |dk rRt|d| fnd}|dk rnt|d| fnd}|dk rt|d| fnd}|dk rt|d| fnd}|dk rt|d| t|dfnd}|j||||||||| | d }||d}||}t|d| f}| dkr$dn || |}| s`|f|dd}|dk r\|f|S|St|||j|j dS) a5 labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr$rQrNrrrr#) rrTrUrr2r.r%r r[r)rDrr\rrr]rrarrr"rbZ num_choicesrZflat_input_idsZflat_attention_maskZflat_token_type_idsZflat_position_idsZflat_inputs_embedsrsrjZreshaped_logitsr$rrHrHrIrtsL     zTFElectraForMultipleChoice.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXt|dddk rt|jj|jdd|j j gW5QRXdS)NTrr2r.) rurvrTrwrr.rxr2r.r(r4ryrHrHrIrxsz TFElectraForMultipleChoice.build) NNNNNNNNNNF)N)r{r|r}r3rr!rrrrr rrtrxr~rHrHrFrIr1s*  ,Ar1z Electra model with a token classification head on top. Both the discriminator and generator may be loaded into this model. cspeZdZfddZeeedede e ddddd d d d d d d d d d d d d ddZ dddZ Z S)TFElectraForTokenClassificationc shtj|f|t|dd|_|jdk r.|jn|j}tj||_ tjj |j t |j dd|_||_dS)Nrrr.r')r2r3rrr)rrr=r@rAr>r*rr?r.r(r+rFrHrIr3,sz(TFElectraForTokenClassification.__init__rzDbhadresh-savani/electra-base-discriminator-finetuned-conll03-englishzK['B-LOC', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'B-LOC', 'O', 'B-LOC', 'I-LOC']g)\(?r/NFrrrz0Union[TFTokenClassifierOutput, Tuple[tf.Tensor]]r!c  Cs|j||||||||| | d } | d} || } || }| dkrFdn || |}| s~|f| dd}|dk rz|f|S|St||| j| jdS)z labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. rrNr$r#)rrAr.r%rr[r)rDrr\rrr]rrarrr"rbrr rjr$rrHrHrIrt9s2   z$TFElectraForTokenClassification.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk rt|jj|jdd|jj gW5QRXdSr0) rurvrTrwrr.rxr.r(r4ryrHrHrIrxqsz%TFElectraForTokenClassification.build) NNNNNNNNNNF)N)r{r|r}r3rr!rrrrrrtrxr~rHrHrFrIr3#s.   ,/r3z Electra Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). csveZdZfddZeeedede e ddddd dd d d d d d dddd d ddd ddZ dddZ Z S)TFElectraForQuestionAnsweringcsPtj|f|||j|_t|dd|_tjj|jt|j dd|_ ||_ dS)Nrr qa_outputsr') r2r3r*rrrr=r>rr?r5r(rrFrHrIr3sz&TFElectraForQuestionAnswering.__init__rz#bhadresh-savani/electra-base-squad2 z'a nice puppet'gQ@)rrrZqa_target_start_indexZqa_target_end_indexrr NFrrrz7Union[TFQuestionAnsweringModelOutput, Tuple[tf.Tensor]]) rr\rrr]rrarrstart_positions end_positionsrbrMc  Cs|j||||||||| | d } | d}||}tj|ddd\}}tj|dd}tj|dd}d}| dk r| dk rd| i}| |d<||||f}| s||f| d d}|dk r|f|S|St|||| j| jd S) a start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. rrrQrNreNZstart_positionZ end_positionr$)r$ start_logits end_logitsr[r) rr5rTsplitrr%r r[r)rDrr\rrr]rrarrr8r9rbrr rjr:r;r$r"rrHrHrIrtsH$   z"TFElectraForQuestionAnswering.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk rt|jj|jdd|jj gW5QRXdS)NTrr5) rurvrTrwrr.rxr5r(r4ryrHrHrIrxsz#TFElectraForQuestionAnswering.build) NNNNNNNNNNNF)N)r{r|r}r3rr!rrrr rrtrxr~rHrHrFrIr4}s4   .Ar4)Rr __future__rr:rZ dataclassesrtypingrrrZnumpynpZ tensorflowrTZactivations_tfrZmodeling_tf_outputsr r r r r rZmodeling_tf_utilsrrrrrrrrrrrrZtf_utilsrrrutilsrrr r!r"r#Zconfiguration_electrar%Z get_loggerr{loggerrrr=ZLayerr&rrrrrrrrrrrrrZELECTRA_START_DOCSTRINGrrrr rr&r-r1r3r4rHrHrHrIs    8   4hOT I*0N J% `*KdR