U 4Afg{@shdZddlmZmZmZddlZddlZddlmZddlm Z ddl m Z ddl m Z mZdd lmZdd lmZmZmZmZd d lmZeeZd ZdZeeejdddZejejdddZ ejejejejdddZ!Gdddej"Z#Gdddej"Z$Gdddej"Z%GdddeZ&d Z'd!Z(ed"e'Gd#d$d$e&Z)ed%e'Gd&d'd'e&Z*dS)(zPyTorch CodeGen model.)OptionalTupleUnionN)nn)CrossEntropyLoss)ACT2FN)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) CodeGenConfigzSalesforce/codegen-2B-monor)num_posdimreturncCs`ddtjd|dtjd|}tdtj|tjd|}tjt|t|fddS) N?i'rdtypez i , j -> i jrr)torcharangeZint64Zeinsumfloatcatsincos)rrZinv_freqZ sinusoid_inpr P/tmp/pip-unpacked-wheel-zw5xktn0/transformers/models/codegen/modeling_codegen.pycreate_sinusoidal_positions&s "r")xrcCsb|dddddddddf}|dddddddddf}tj| |fdd}|dS)Nrrr)rstackflatten)r#x1Zx2r r r!rotate_every_two-s""r))tensorrrrcCs`t|dddddddfdd}t|dddddddfdd}||t||S)Nrr)rZrepeat_interleaver))r*rrr r r!apply_rotary_pos_emb5s&&r+cseZdZfddZddZddZddd Zdeej ee ej eej eej eej ee ee ee ej e ej fee ej e ej e ej d fffd d dZZS)CodeGenAttentioncs"t|j}|jdttj||ftjddd||ddt |j |_ t |j |_|j|_|j|_|j|j|_|j|j|jkrtd|jd|jdttj|jtjdt|_t j|j|jd dd |_t j|j|jdd |_|j|_|jp|j}t|||_dS) N causal_maskrrF) persistentzEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r)bias) super__init__Zmax_position_embeddingsZregister_bufferrZtrilZonesboolviewrDropoutZ attn_pdrop attn_dropout resid_pdrop resid_dropoutZ hidden_size embed_dimnum_attention_headshead_dim ValueErrorsqrtr*float32toZget_default_dtype scale_attnLinearqkv_projout_proj rotary_dimr"embed_positions)selfconfigZ max_positionsZ pos_embd_dim __class__r r!r1<s6 $zCodeGenAttention.__init__cCsJ||jdd|||f}||jddd|jdd}|S)Nr$r%)r$)reshapeshape)rEr#Zn_headZdim_headmp_numZreshapedr r r! _split_heads[s &zCodeGenAttention._split_headscCst|jdkr&|ddddd}n8t|jdkrJ|dddd}ntdt|j|dd ||f}||S) zM Merges attn_head_size dim and num_attn_heads dim into n_ctx rrrrz3Input tensor rank should be one of [4, 5], but is: Nr%)lenrJpermute contiguousr;sizer3)rEr*r9Zattn_head_sizeZ new_shaper r r! _merge_heads`szCodeGenAttention._merge_headsNc Cs|d|d}}|jdddd|||d|f}|tj}|tj}t||dd} | |j} t| j j } tj | | j d| j } t || | } |dk r| |} tjdd| } | |j } || } |dk r| |} t| |} | | fS)Nr%r$rr)rRr-r>rr=matmulZ transposer?finforminr*devicewhererZSoftmaxr5) rEquerykeyvalueattention_mask head_maskZ query_lengthZ key_lengthr- attn_weightsZ mask_value attn_outputr r r!_attnms$ &      zCodeGenAttention._attnF. hidden_states layer_pastr\ position_idsr] use_cacheoutput_attentionsrcCs||}d} ||jdd| df} |j|j| } tj| | dd\} } }|j| |j|j| d} |j||j|j| d}|j| |j|j| d} | dddd} |j }|j |j kr| |j }||_ ||}tj||jdddd\}}|j dk r|ddddddd|j f}|dddddd|j df}| ddddddd|j f}| dddddd|j df}t |||}t |||}tj||gdd}tj||gdd} nt |||}t | ||} |dddd}| dddd} |dk r4|d}|d}tj||fd d}tj|| fd d} |d krP| |j| f}nd}|| || ||\}}|||j|j}||}||}||f}|r||f7}|S) NrNr$r)rKrrrrr%T)rArIrJr:r9rsplitrLrPrDrWr>rCr+rrr`rSrBr7)rErbrcr\rdr]rerfZqkvrKZ qkv_splitZ local_dimrYr[rZrDZsincosrrZk_rotZk_passZq_rotZq_passZpast_keyZ past_valueZpresentr_r^outputsr r r!forwardsX    """"         zCodeGenAttention.forward)NN)NNNNFF)__name__ __module__ __qualname__r1rLrSr`rr FloatTensorrTensor LongTensorr2rri __classcell__r r rGr!r,;s4  , "r,cs4eZdZfddZeejejdddZZS) CodeGenMLPcsJt|j}t|||_t|||_t|j|_ t |j |_ dSN) r0r1n_embdrr@fc_infc_outrZactivation_functionactr4r6dropout)rEZintermediate_sizerFr8rGr r!r1s   zCodeGenMLP.__init__)rbrcCs,||}||}||}||}|Srr)rtrvrurw)rErbr r r!ris     zCodeGenMLP.forward) rjrkrlr1rrrmrirpr r rGr!rqs rqcseZdZfddZd eejeeejeejeej eejee ee e eejeeejeejdfffdddZ Z S) CodeGenBlockcsRt|jdk r|jnd|j}tj|j|jd|_t||_ t |||_ dS)NrNZeps) r0r1Zn_innerrsr LayerNormlayer_norm_epsilonln_1r,attnrqmlp)rErFZ inner_dimrGr r!r1s   zCodeGenBlock.__init__NF.rac Csv|}||}|j|||||||d} | d} | dd} ||} | | |}|r`|f| } n|f| dd} | S)Nrbrcr\rdr]rerfrr)r|r}r~) rErbrcr\rdr]rerfZresidualZ attn_outputsr_rhZfeed_forward_hidden_statesr r r!ris&       zCodeGenBlock.forward)NNNNFF)rjrkrlr1rrrmrrnror2rrirpr r rGr!rxs"  (rxcs>eZdZdZeZdZdZdgZdZ fddZ dd Z Z S) CodeGenPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerTrxpast_key_valuescstj||dSrr)r0r1)rEinputskwargsrGr r!r13szCodeGenPreTrainedModel.__init__cCst|tjfr<|jjjd|jjd|jdk r|jj nft|tj r||jjjd|jjd|j dk r|jj|j  n&t|tj r|jj |jj ddS)zInitialize the weights.g)ZmeanZstdNr) isinstancerr@ZweightdataZnormal_rFZinitializer_ranger/Zzero_ EmbeddingZ padding_idxrzZfill_)rEmoduler r r! _init_weights6s     z$CodeGenPreTrainedModel._init_weights) rjrkrl__doc__r config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementr1rrpr r rGr!r's raJ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`CodeGenConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a: Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zaThe bare CodeGen Model transformer outputting raw hidden-states without any specific head on top.cseZdZfddZddZddZeede e e e dd e eje eeeje eje eje eje eje eje ee ee ee eeee fd d d ZZS) CodeGenModelcstj|_j|_tj|j|_tj |_ t fddt j D|_tj|jjd|_tjjj|_d|_|dS)Ncsg|] }tqSr )rx).0_rFr r! sz)CodeGenModel.__init__..ryF)r0r1rsr8 vocab_sizerrwter4Z embd_pdropdropZ ModuleListrangen_layerhrzr{ln_frVrCZn_ctxr9gradient_checkpointing post_initrErFrGrr!r1s  zCodeGenModel.__init__cCs|jSrrrrEr r r!get_input_embeddingssz!CodeGenModel.get_input_embeddingscCs ||_dSrrrrEZnew_embeddingsr r r!set_input_embeddingssz!CodeGenModel.set_input_embeddingsbatch_size, sequence_length checkpoint output_typerN) input_idsrr\token_type_idsrdr] inputs_embedsrerfoutput_hidden_states return_dictrc  Cs|| dk r | n|jj} | dk r | n|jj} |dk r4|n|jj}| dk rH| n|jj} |dk rj|dk rjtdnd|dk r||||} |d| d}|j d} n,|dk r|dd} |j d} ntd|dk r|j n|j }|dk r|d| d}|dkrd}t dgt |j }n|ddd}|dkr`tj|| d|tj|d}|d}|dk r| dkr|td|| d}|ddddddf}|j|jd}d |t|jj}|||jj}|dkr||}|}|dk r||}||}||}| |df}|jrF|jrF|rFtd d }|rPd nd}| r^d nd}| rld nd}tt|j |D]\}\}}| r||f}|jr|jr| |j!|d|||||| }n||||||||| d }|d}|dkr||df}| r|||rdndf}q|"|}||}| rL||f}| slt dd||||fDSt#||||dS)NzDYou cannot specify both input_ids and inputs_embeds at the same timer$rz5You have to specify either input_ids or inputs_embedsr%)rrWz$batch_size has to be defined and > 0rrzh`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting `use_cache=False`...Fr rTrrcss|]}|dk r|VqdSrrr )rvr r r! 1sz'CodeGenModel.forward..)Zlast_hidden_staterrb attentions)$rFrfrreuse_return_dictr;Z%warn_if_padding_and_no_attention_maskrRr3rJrWtuplerOrrrlongZ unsqueezer>rrUrVZ get_head_maskrrrrZtrainingloggerZ warning_once enumeratezipZ_gradient_checkpointing_func__call__rr )rErrr\rrdr]rrerfrrZ input_shapeZ batch_sizerW past_lengthrbZtoken_type_embedsZ output_shapeZpresentsZall_self_attentionsZall_hidden_statesiblockrcrhr r r!ris                      zCodeGenModel.forward) NNNNNNNNNNN)rjrkrlr1rrrCODEGEN_INPUTS_DOCSTRINGformatr _CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrrorrnrmr2rrirpr r rGr!rsD   rzM The CodeGen Model transformer with a language modeling head on top. cseZdZdgZfddZddZddZdd d Zee d e e e ed deejeeeejeejeejeejeejeejeejeeeeeeeeeee fd ddZeeeejejeeejdddZZS)CodeGenForCausalLMzlm_head.weightcs4t|t||_t|j|j|_| dSrr) r0r1rrrr@rsrlm_headrrrGr r!r1Ds  zCodeGenForCausalLM.__init__cCs|jSrrrrr r r!get_output_embeddingsLsz(CodeGenForCausalLM.get_output_embeddingscCs ||_dSrrrrr r r!set_output_embeddingsOsz(CodeGenForCausalLM.set_output_embeddingsNc Ks0|dd}|r||ddjd}|jd|kr6|}n|jdd}|dd|df}|dk r||dd|jd df}|dd}|dd} |dk r| dkr|dd} | |dkd|r| dd|jd df} |dk r|dkrd|i} n d |i} | ||d | ||d | S) Nrrrrr\rdr$rrre)rrerdr\r)getrJrZcumsumZ masked_fill_rQupdate) rErrrrrrZremove_prefix_lengthr\rdZ model_inputsr r r!prepare_inputs_for_generationRs8      z0CodeGenForCausalLM.prepare_inputs_for_generationrr) rrr\rrdr]rlabelsrerfrrrc  Cs| dk r | n|jj} |j|||||||| | | | d } | d}||tj}d}|dk r||j}|dddddf}|dddf}t }|| d| d| d}||j }| s|f| dd}|dk r|f|S|St ||| j| j| jdS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` N) rr\rrdr]rrerfrrr.r$r)lossZlogitsrrbr)rFrrrr>rr=rWrQrr3rRrr rrbr)rErrr\rrdr]rrrerfrrZtransformer_outputsrbZ lm_logitsrZ shift_logitsZ shift_labelsZloss_fctoutputr r r!ri~sD   zCodeGenForCausalLM.forward)rbeam_idxrcstfdd|DS)a This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. c3s$|]}tfdd|DVqdS)c3s"|]}|d|jVqdS)rN)Z index_selectr>rW)rZ past_staterr r!rsz>CodeGenForCausalLM._reorder_cache...Nr)rrcrr r!rsz4CodeGenForCausalLM._reorder_cache..r)rrr rr!_reorder_caches z!CodeGenForCausalLM._reorder_cache)NN) NNNNNNNNNNNN)rjrkrlZ_tied_weights_keysr1rrrrrrr rr rrrrorrnrmr2rri staticmethodrrpr r rGr!r;sV  ,  D  r)+rtypingrrrrZtorch.utils.checkpointrZtorch.nnrZ activationsrZmodeling_outputsr r Zmodeling_utilsr utilsr r rrZconfiguration_codegenrZ get_loggerrjrrrintrnr"r)r+Moduler,rqrxrZCODEGEN_START_DOCSTRINGrrrr r r r!sD      -,  24