U 4Af3@szdZddlZddlmZmZmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZmZmZdd lmZdd lmZmZdd lmZmZmZm Z d d l!m"Z"d dl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)e *e+Z,dZ-dZ.Gdddej/Z0Gdddej/Z1Gdddej/Z2Gdddej/Z3Gdddej/Z4Gdddej/Z5Gdddej/Z6Gdd d ej/Z7Gd!d"d"ej/Z8Gd#d$d$eZ9d%Z:d&Z;ed'e:Gd(d)d)e9Zed/e:Gd0d1d1e9Z?ed2e:Gd3d4d4e9Z@ed5e:Gd6d7d7e9ZAGd8d9d9ej/ZBed:e:Gd;d<dZDdS)@zPyTorch I-BERT model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)gelu))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) IBertConfig)IntGELU IntLayerNorm IntSoftmaxQuantActQuantEmbedding QuantLinearzkssteven/ibert-roberta-basercs2eZdZdZfddZd ddZdd ZZS) IBertEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. cs t|j|_d|_d|_d|_d|_d|_t|j |j |j |j|jd|_ t|j |j |j|jd|_|jdt|jdd d t|d d |_|j |_t|j|j |j|j|jd|_t|j|jd |_t|j|jd |_t|j |j|j|j|jd|_t|j|jd |_t !|j"|_#dS)N ) padding_idx weight_bit quant_mode)r'r( position_ids)rF) persistentposition_embedding_typeabsoluter(epsZ output_bitr( force_dequant)$super__init__r(Z embedding_bitZembedding_act_bitact_bit ln_input_bit ln_output_bitr vocab_size hidden_sizeZ pad_token_idword_embeddingsZtype_vocab_sizetoken_type_embeddingsZregister_buffertorcharangeZmax_position_embeddingsexpandgetattrr,r&position_embeddingsrembeddings_act1Zembeddings_act2rlayer_norm_epsr1 LayerNormoutput_activationrDropouthidden_dropout_probdropoutselfconfig __class__L/tmp/pip-unpacked-wheel-zw5xktn0/transformers/models/ibert/modeling_ibert.pyr38sZ  zIBertEmbeddings.__init__NrcCs|dkr2|dk r(t||j||j}n ||}|dk rD|}n|dd}|dkrrtj|tj|j jd}|dkr| |\}}nd}| |\}} |j |||| d\} } |j dkr||\} } |j | | | | d\} } || | \} } || } || | \} } | | fS)Nr*dtypedeviceidentityZidentity_scaling_factorr-)"create_position_ids_from_input_idsr&torP&create_position_ids_from_inputs_embedssizer;zeroslongr)r9r:r@r,r?rBrFrC)rH input_idstoken_type_idsr) inputs_embedspast_key_values_length input_shapeZinputs_embeds_scaling_factorr:Z$token_type_embeddings_scaling_factor embeddingsZembeddings_scaling_factorr?Z"position_embeddings_scaling_factorrLrLrMforwardlsJ      zIBertEmbeddings.forwardcCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr*rrNr)rVr;r<r&rXrPZ unsqueezer=)rHr[r]Zsequence_lengthr)rLrLrMrUs  z6IBertEmbeddings.create_position_ids_from_inputs_embeds)NNNNr)__name__ __module__ __qualname____doc__r3r_rU __classcell__rLrLrJrMr!3s 5 /r!cs.eZdZfddZddZd ddZZS) IBertSelfAttentionc sxt|j|jdkr>t|ds>td|jd|jd|j|_d|_d|_d|_ |j|_t |j|j|_ |j|j |_ t |j|j d|j|j|jdd |_t |j|j d|j|j|jdd |_t |j|j d|j|j|jdd |_t|j |jd |_t|j |jd |_t|j |jd |_t|j |jd |_t|j|_t|d d |_|jd kr^td t|j |j|jd|_dS)NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r"r%Tbiasr'bias_bitr(Z per_channelr.r,r-zDI-BERT only supports 'absolute' for `config.position_embedding_type`r(r1)r2r3r8num_attention_headshasattr ValueErrorr(r'rir4intattention_head_size all_head_sizer querykeyvaluerquery_activationkey_activationvalue_activationrCrrDZattention_probs_dropout_probrFr>r,rr1softmaxrGrJrLrMr3s`     zIBertSelfAttention.__init__cCs6|dd|j|jf}|j|}|ddddS)Nr*rrr )rVrkroviewpermute)rHxZ new_x_shaperLrLrMtranspose_for_scoress z'IBertSelfAttention.transpose_for_scoresNFcCs|||\}}|||\}} |||\} } |||\} } ||| \}}|| | \}}|| } ||}||}t| | dd}t |j }||}|j r| ||}nd}|dk r||}|||\}}||}|dk r||}t||}|dk r||}nd}|dddd}|dd|jf}|j|}|||\}}|rp||fn|f}|r||fn|f}||fS)Nr*rrxrr )rqrrrsrtrurvr|r;matmulZ transposemathsqrtror(rwrFrz contiguousrVrpryrC)rH hidden_stateshidden_states_scaling_factorattention_mask head_maskoutput_attentionsZmixed_query_layerZ mixed_query_layer_scaling_factorZmixed_key_layerZmixed_key_layer_scaling_factorZmixed_value_layerZ mixed_value_layer_scaling_factorZ query_layerZquery_layer_scaling_factorZ key_layerZkey_layer_scaling_factorZ value_layerZvalue_layer_scaling_factorZattention_scoresZscaleZattention_scores_scaling_factorZattention_probsZattention_probs_scaling_factorZ context_layerZcontext_layer_scaling_factorZnew_context_layer_shapeoutputsZoutput_scaling_factorrLrLrMr_s^           zIBertSelfAttention.forward)NNF)r`rarbr3r|r_rdrLrLrJrMres  : recs$eZdZfddZddZZS)IBertSelfOutputc st|j|_d|_d|_d|_d|_d|_t|j |j d|j|j|jdd|_ t |j|jd|_ t |j |j|j|j|jd|_t |j|jd|_t|j|_dSNr"r%r$Trgr.r/)r2r3r(r4r'rir5r6r r8denser ln_input_actrrAr1rBrCrrDrErFrGrJrLrMr3<s4  zIBertSelfOutput.__init__cCsX|||\}}||}|j||||d\}}|||\}}|||\}}||fSNrQrrFrrBrCrHrrZ input_tensorZinput_tensor_scaling_factorrLrLrMr_Ys  zIBertSelfOutput.forwardr`rarbr3r_rdrLrLrJrMr;s rcs.eZdZfddZddZd ddZZS) IBertAttentioncs2t|j|_t||_t||_t|_dSN) r2r3r(rerHroutputset pruned_headsrGrJrLrMr3ks    zIBertAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)NrrZdim)lenrrHrkrorrrqrrrsrrrpunion)rHheadsindexrLrLrM prune_headsrs zIBertAttention.prune_headsNFc Cs^||||||\}}||d|d||\}} |f|dd} | f|dd} | | fS)Nrr)rHr) rHrrrrrZ self_outputsZself_outputs_scaling_factorattention_outputattention_output_scaling_factorrZoutputs_scaling_factorrLrLrMr_s zIBertAttention.forward)NNF)r`rarbr3rr_rdrLrLrJrMrjs  rcs$eZdZfddZddZZS)IBertIntermediatec st|j|_d|_d|_d|_t|j|jd|j|j|jdd|_ |j dkrXt dt |j|j d|_t|j|jd|_dS) Nr"r%Trgr z3I-BERT only supports 'gelu' for `config.hidden_act`rjr.)r2r3r(r4r'rir r8intermediate_sizerZ hidden_actrmrr1intermediate_act_fnrrCrGrJrLrMr3s$  zIBertIntermediate.__init__cCs8|||\}}|||\}}|||\}}||fSr)rrrC)rHrrrLrLrMr_szIBertIntermediate.forwardrrLrLrJrMrs rcs$eZdZfddZddZZS) IBertOutputc st|j|_d|_d|_d|_d|_d|_t|j |j d|j|j|jdd|_ t |j|jd|_ t|j |j|j|j|jd|_t |j|jd|_t|j|_dSr)r2r3r(r4r'rir5r6r rr8rrrrrAr1rBrCrrDrErFrGrJrLrMr3s4  zIBertOutput.__init__cCsX|||\}}||}|j||||d\}}|||\}}|||\}}||fSrrrrLrLrMr_s  zIBertOutput.forwardrrLrLrJrMrs rcs.eZdZfddZd ddZddZZS) IBertLayercsdt|j|_d|_d|_t||_t||_t ||_ t |j|jd|_ t |j|jd|_ dS)Nr"rr.)r2r3r(r4Z seq_len_dimr attentionr intermediaterrrpre_intermediate_actpre_output_actrGrJrLrMr3s    zIBertLayer.__init__NFc CsR|j|||||d\}}|d}|d} |dd} ||| \} } | f| } | S)N)rrr)rfeed_forward_chunk) rHrrrrrZself_attention_outputsZ%self_attention_outputs_scaling_factorrrr layer_outputlayer_output_scaling_factorrLrLrMr_s    zIBertLayer.forwardcCsL|||\}}|||\}}|||\}}|||||\}}||fSr)rrrr)rHrrZintermediate_outputZ"intermediate_output_scaling_factorrrrLrLrMrs&zIBertLayer.feed_forward_chunk)NNF)r`rarbr3r_rrdrLrLrJrMrs   rcs&eZdZfddZdddZZS) IBertEncodercs<t|_j|_tfddtjD|_dS)Ncsg|] }tqSrL)r).0_rIrLrM +sz)IBertEncoder.__init__..) r2r3rIr(rZ ModuleListrangenum_hidden_layerslayerrGrJrrMr3's zIBertEncoder.__init__NFTcCs|rdnd}|rdnd} d} d} t|jD]T\} } |r@||f}|dk rP|| nd}| |||||}|d}|r*| |df} q*|r||f}|stdd|| || | fDSt|| || | dS)NrLrrcss|]}|dk r|VqdSrrL)rvrLrLrM Rsz'IBertEncoder.forward..)last_hidden_statepast_key_valuesr attentionscross_attentions) enumeratertupler )rHrrrrroutput_hidden_states return_dictZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZ layer_moduleZlayer_head_maskZ layer_outputsrLrLrMr_-sH     zIBertEncoder.forward)NNFFTrrLrLrJrMr&s rcs$eZdZfddZddZZS) IBertPoolercs2t|j|_t|j|j|_t|_dSr) r2r3r(rLinearr8rZTanh activationrGrJrLrMr3gs zIBertPooler.__init__cCs(|dddf}||}||}|SNr)rr)rHrZfirst_token_tensor pooled_outputrLrLrMr_ms  zIBertPooler.forwardrrLrLrJrMrfs rc@s*eZdZdZeZdZddZdddZdS) IBertPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. ibertcCst|ttjfr>|jjjd|jjd|j dk r|j j nnt|t tj fr|jjjd|jjd|j dk r|jj|j  n*t|ttjfr|j j |jjddS)zInitialize the weightsg)ZmeanZstdNg?) isinstancer rrZweightdataZnormal_rIZinitializer_rangerhZzero_rZ Embeddingr&rrBZfill_)rHmodulerLrLrM _init_weightss   z"IBertPreTrainedModel._init_weightsNcCs tddS)Nz6`resize_token_embeddings` is not supported for I-BERT.)NotImplementedError)rHZnew_num_tokensrLrLrMresize_token_embeddingssz,IBertPreTrainedModel.resize_token_embeddings)N) r`rarbrcr config_classZbase_model_prefixrrrLrLrLrMrvs ra? This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`IBertConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a5 Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z`The bare I-BERT Model transformer outputting raw hidden-states without any specific head on top.cseZdZdZdfdd ZddZddZd d Zee d e e e ed deejeejeejeejeejeejeeeeeeee eejfd ddZZS) IBertModela The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. TcsLt|||_|j|_t||_t||_|r:t|nd|_ | dSr) r2r3rIr(r!r^rencoderrpooler post_init)rHrIadd_pooling_layerrJrLrMr3s   zIBertModel.__init__cCs|jjSrr^r9rHrLrLrMget_input_embeddingsszIBertModel.get_input_embeddingscCs ||j_dSrr)rHrsrLrLrMset_input_embeddingsszIBertModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)rHZheads_to_prunerrrLrLrM _prune_headsszIBertModel._prune_headsbatch_size, sequence_length checkpoint output_typerN) rYrrZr)rr[rrrreturnc  Cs|dk r |n|jj}|dk r |n|jj}| dk r4| n|jj} |dk rV|dk rVtdn@|dk rt||||} n"|dk r|dd} ntd| \} } |dk r|jn|j} |dkrtj | | f| d}|dkrtj | tj | d}| || }| ||jj}|j||||d\}}|j||||||| d}|d}|jdk rN||nd}| sl||f|d dSt|||j|j|j|jd S) NzDYou cannot specify both input_ids and inputs_embeds at the same timer*z5You have to specify either input_ids or inputs_embeds)rPrN)rYr)rZr[)rrrrrrr)rZ pooler_outputrrrr)rIrruse_return_dictrmZ%warn_if_padding_and_no_attention_maskrVrPr;ZonesrWrXZget_extended_attention_maskZ get_head_maskrr^rrr rrrr)rHrYrrZr)rr[rrrr]Z batch_sizeZ seq_lengthrPZextended_attention_maskZembedding_outputZembedding_output_scaling_factorZencoder_outputssequence_outputrrLrLrMr_s^      zIBertModel.forward)T) NNNNNNNNN)r`rarbrcr3rrrrIBERT_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrr; LongTensor FloatTensorboolrrr_rdrLrLrJrMrs@   rz4I-BERT Model with a `language modeling` head on top.cseZdZddgZfddZddZddZee d e e e e d d deejeejeejeejeejeejeejeeeeeeee eejfd ddZZS)IBertForMaskedLMzlm_head.decoder.biaszlm_head.decoder.weightcs0t|t|dd|_t||_|dSNF)r)r2r3rr IBertLMHeadlm_headrrGrJrLrMr3Vs  zIBertForMaskedLM.__init__cCs|jjSr)rdecoderrrLrLrMget_output_embeddings_sz&IBertForMaskedLM.get_output_embeddingscCs||j_|j|j_dSr)rrrh)rHZnew_embeddingsrLrLrMset_output_embeddingsbsz&IBertForMaskedLM.set_output_embeddingsrz)rrrmaskN rYrrZr)rr[labelsrrrrc  Cs| dk r | n|jj} |j|||||||| | d } | d} || } d}|dk rpt}|| d|jj|d}| s| f| dd}|dk r|f|S|St|| | j| j dS)a( labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): Used to hide legacy arguments that have been deprecated. NrrZr)rr[rrrrr*rxlosslogitsrr) rIrrrrryr7r rr)rHrYrrZr)rr[rrrrrrZprediction_scoresZmasked_lm_lossloss_fctrrLrLrMr_fs6  zIBertForMaskedLM.forward) NNNNNNNNNN)r`rarbZ_tied_weights_keysr3rrrrrrrr rrr;rrrrrr_rdrLrLrJrMrRsD  rcs6eZdZdZfddZddZdddd ZZS) rz)I-BERT Head for masked language modeling.csdtt|j|j|_tj|j|jd|_t|j|j |_ t t |j |_|j|j _dS)N)r0)r2r3rrr8rrBrA layer_normr7r Parameterr;rWrhrGrJrLrMr3s  zIBertLMHead.__init__cKs*||}t|}||}||}|Sr)rr rr)rHfeatureskwargsr{rLrLrMr_s    zIBertLMHead.forwardN)rcCs*|jjjjdkr|j|j_n |jj|_dS)Nmeta)rrhrPtyperrLrLrM _tie_weightss zIBertLMHead._tie_weights)r`rarbrcr3r_rrdrLrLrJrMrs  rz I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cseZdZfddZeedeee e dd e e j e e je e j e e j e e je e je e j e ee ee eee ee jfd ddZZS) IBertForSequenceClassificationcs8t||j|_t|dd|_t||_|dSr)r2r3 num_labelsrrIBertClassificationHead classifierrrGrJrLrMr3s   z'IBertForSequenceClassification.__init__rrNrc  Csr| dk r | n|jj} |j|||||||| | d } | d} || } d}|dk r.|jjdkr|jdkrpd|j_n4|jdkr|jtjks|jtj krd|j_nd|j_|jjdkrt }|jdkr|| | }n || |}nN|jjdkrt }|| d|j| d}n|jjdkr.t}|| |}| s^| f| d d}|dk rZ|f|S|St|| | j| jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NrrrZ regressionZsingle_label_classificationZmulti_label_classificationr*rxr)rIrrrZ problem_typerrOr;rXrnrsqueezerryrrrrrHrYrrZr)rr[rrrrrrrrrrrLrLrMr_sT      "     z&IBertForSequenceClassification.forward) NNNNNNNNNN)r`rarbr3rrrrrrrrr;rrrrrr_rdrLrLrJrMrs< rz I-BERT Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks. cseZdZfddZeedeee e dd e e j e e j e e je e j e e j e e je e je ee ee eee ee jfd ddZZS) IBertForMultipleChoicecs@t|t||_t|j|_t|j d|_ | dS)Nr) r2r3rrrrDrErFrr8rrrGrJrLrMr3's   zIBertForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthrN) rYrZrrr)rr[rrrrc  Cst| dk r | n|jj} |dk r&|jdn|jd} |dk rJ|d|dnd} |dk rh|d|dnd} |dk r|d|dnd}|dk r|d|dnd}|dk r|d|d|dnd}|j| | |||||| | d }|d}||}||}|d| }d}|dk r0t}|||}| s`|f|dd}|dk r\|f|S|St |||j |j dS)aJ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nrr*r})r)rZrrr[rrrrxr) rIrshaperyrVrrFrrrrr)rHrYrZrrr)rr[rrrZ num_choicesZflat_input_idsZflat_position_idsZflat_token_type_idsZflat_attention_maskZflat_inputs_embedsrrrZreshaped_logitsrrrrLrLrMr_1sL      zIBertForMultipleChoice.forward) NNNNNNNNNN)r`rarbr3rrrrrrrrr;rrrrrr_rdrLrLrJrMrs< rz I-BERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. cseZdZfddZeedeee e dd e e j e e je e j e e j e e je e je e j e ee ee eee ee jfd ddZZS) IBertForTokenClassificationcsNt||j|_t|dd|_t|j|_t |j |j|_ | dSr) r2r3rrrrrDrErFrr8rrrGrJrLrMr3s  z$IBertForTokenClassification.__init__rrNrc  Cs| dk r | n|jj} |j|||||||| | d } | d} || } || } d}|dk rxt}|| d|j|d}| s| f| dd}|dk r|f|S|St|| | j | j dS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nrrr*rxr) rIrrrFrrryrrrrrrLrLrMr_s8   z#IBertForTokenClassification.forward) NNNNNNNNNN)r`rarbr3rrrrrrrrr;rrrrrr_rdrLrLrJrMrys< rcs(eZdZdZfddZddZZS)rz-Head for sentence-level classification tasks.cs@tt|j|j|_t|j|_t|j|j |_ dSr) r2r3rrr8rrDrErFrout_projrGrJrLrMr3s z IBertClassificationHead.__init__cKsL|dddddf}||}||}t|}||}||}|Sr)rFrr;tanhr)rHrrrrLrLrMr_s     zIBertClassificationHead.forward)r`rarbrcr3r_rdrLrLrJrMrs rz I-BERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). cseZdZfddZeedeee e dd e e j e e je e j e e j e e je e je e j e e j e ee ee eee ee jfd ddZZS) IBertForQuestionAnsweringcs@t||j|_t|dd|_t|j|j|_| dSr) r2r3rrrrrr8 qa_outputsrrGrJrLrMr3s  z"IBertForQuestionAnswering.__init__rrN) rYrrZr)rr[start_positions end_positionsrrrrc  CsP| dk r | n|jj} |j||||||| | | d } | d} || }|jddd\}}|d}|d}d}|dk r|dk rt|dkr|d}t|dkr|d}|d}| d|}| d|}t |d}|||}|||}||d}| s:||f| dd}|dk r6|f|S|St |||| j | j d S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Nrrrr*r)Z ignore_indexrx)r start_logits end_logitsrr)rIrrrsplitrrrrVclamprrrr)rHrYrrZr)rr[rrrrrrrrrr Z total_lossZ ignored_indexrZ start_lossZend_lossrrLrLrMr_sP           z!IBertForQuestionAnswering.forward) NNNNNNNNNNN)r`rarbr3rrrrrrrrr;rrrrrr_rdrLrLrJrMrs@ rcCs6||}tj|dd|||}||S)aM Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's *utils.make_positions*. Args: input_ids (`torch.LongTensor`): Indices of input sequence tokens in the vocabulary. Returns: torch.Tensor rr)nernr;ZcumsumZtype_asrX)rYr&r\rZincremental_indicesrLrLrMrS=s rS)r)Ercrtypingrrrr;Ztorch.utils.checkpointrZtorch.nnrrrZ activationsr Zmodeling_outputsr r r rrrrZmodeling_utilsrZ pytorch_utilsrrutilsrrrrZconfiguration_ibertrZ quant_modulesrrrrrr Z get_loggerr`loggerrrModuler!rerrrrrrrrZIBERT_START_DOCSTRINGrrrrrrrrrrSrLrLrLrMsv  $    z/1"/:@2yPVSG\