U :sz+WordEmbeddings.tokenize..cSsg|] }t|qSr)len)r)tokensrrr r-;srr )Zdtype)r!r"sentence_lengths)maxr.appendrZtensorlong) rr'r,Ztokenized_textsr0max_lenr!Zattention_masksr/paddingoutputrr+r r(9szWordEmbeddings.tokenize)returncCs|jS)N)rrrrr get_word_embedding_dimensionMsz+WordEmbeddings.get_word_embedding_dimensionT) output_pathsafe_serializationc Cs|ttj|dd}tj||ddW5QRX|rRt|tj|dnt |tj|d|j |dS)Nwordembedding_config.jsonw)indentmodel.safetensorspytorch_model.bin) openospathjoinjsondumpget_config_dictsave_safetensors_fileZ state_dictrsaver)rr:r;ZfOutrrr rJPs zWordEmbeddings.savecCst|j|j|jdS)N)tokenizer_classrr)rrrrr8rrr rHZszWordEmbeddings.get_config_dict) input_pathc Csttj|dd}t|}W5QRXt|d}||}tjtj|drjttj|d}nt jtj|dt dd}|d}t |||d d }|S) Nr<rrKr@rAcpu)Z map_locationzemb_layer.weightrrrr) rBrCrDrErFloadr existsload_safetensors_filerZdevicer )rLfInconfigrKrweightsrmodelrrr rPas  zWordEmbeddings.load N)embeddings_file_pathritem_separatormax_vocab_sizec Cstd|tj|s\td|d|ksrr Z PADDING_TOKENz\ERROR: A line in the embeddings file had more or less dimensions than expected. Skip token.cSsg|] }t|qSr)float)r)numrrr r-sz1WordEmbeddings.from_text_file..rO)loggerinfoformatrCrDrQ ValueErrorrendswithgziprBrrstripsplitr.r2rzeroserrorarrayrZ set_vocabr )rXrrYrrZurlrZvocabZ embeddingsrSiteratorlinerkwordZvectorrrr from_text_filersX         zWordEmbeddings.from_text_file)Fr)T)__name__ __module__ __qualname__r boolintrr&rstrr(r9rJrH staticmethodrPr rsrrrr r s2   r )rirFloggingrCtypingrZnumpyrrZsafetensors.torchrrRrrIrrZsentence_transformers.utilrrr rr r getLoggerrtrdrr rrrr s