o  h@s*dZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z ddl m Z mZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZmZee Z!dZ"d Z#d Z$Gd d d Z%Gd dde%Z&ddZ'ddZ(ddZ)ddZ*ddZ+de,e-de-fddZ.eeGdddeZ/dS)z Tokenization classes for python tokenizers. For fast tokenizers (provided by HuggingFace's tokenizers library) see tokenization_utils_fast.py N) OrderedDict)AnyOptionalUnionoverload) ENCODE_KWARGS_DOCSTRING'ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRINGINIT_TOKENIZER_DOCSTRING AddedToken BatchEncoding EncodedInputEncodedInputPairPreTokenizedInputPreTokenizedInputPairPreTrainedTokenizerBase TextInput TextInputPairTruncationStrategy)PaddingStrategy TensorTypeadd_end_docstringsloggingzspecial_tokens_map.jsonzadded_tokens.jsonztokenizer_config.jsonc@sLeZdZdZddZddZdefddZd ed eefd d Z d dZ dS)Triez Trie in Python. Creates a Trie out of a list of words. The trie is used to split on `added_tokens` in one pass Loose reference https://en.wikipedia.org/wiki/Trie cGs"i|_t|_d|_|j|dS)N)dataset_tokens_termination_charupdateselfargsr#s/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/transformers/tokenization_utils.py__init__:sz Trie.__init__cGst|D]}||qdS)z Updates the Trie with new tokens provided as arguments. Args: *args: Variable number of words to be added to the Trie. N)tupleadd)r!r"tokenr#r#r$r@s  z Trie.updatewordcCsJ|sdS|j||j}|D]}||i||<||}qd||j<dS)u Passes over every char (utf-8 char) on word and recursively adds it to the internal `data` trie representation. The special key `""` in `self._termination_char` is used to represent termination. This function is idempotent, adding twice the same word will leave the trie unchanged Example: ```python >>> trie = Trie() >>> trie.add("Hello 友達") >>> trie.data {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}} >>> trie.add("Hello") >>> trie.data {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}} ``` Nr)rr'r setdefaultr)r!r)refcharr#r#r$r'Js  zTrie.addtextreturncCst}dg}d}t|D]\}}|r||krq t}d}|D]\} } d| vr|D]V\} } | | kr6nM| | krC|d} |d}n|} |}| t|krQ|| nd}d| vr]| } | }| }|| vr| |} | d7} d| vrs| } | }| }| t|krzn|| }|| vsaq,|| ||d}n|| vr| |} | || <q || q |ri}n|D]} || =q||kr||jvr|j|||<q |D]\} } d| vrt|}|| ||nq|||S)a\ Will look for the words added to the trie within `text`. Output is the original string splitted along the boundaries of the words found. This trie will match the longest possible word first ! Example: ```python >>> trie = Trie() >>> trie.split("[CLS] This is a extra_id_100") ["[CLS] This is a extra_id_100"] >>> trie.add("[CLS]") >>> trie.add("extra_id_1") >>> trie.add("extra_id_100") >>> trie.split("[CLS] This is a extra_id_100") ["[CLS]", " This is a ", "extra_id_100"] ``` rFrrNT) r enumerateritemslenappendr'rcut_text)r!r-statesoffsetsskipcurrent current_char to_removeresetstart trie_pointer lookstartlooktrie_pointerlookahead_indexend next_charr#r#r$splitist!           z Trie.splitcCsX|t|g}d}|D]}||krtdq ||krq |||||}q |S)NrzbThere was a bug in Trie algorithm in tokenization. Attempting to recover. Please report it anyway.)r2r1loggererror)r!r-r5tokensr;r@r#r#r$r3sz Trie.cut_textN) __name__ __module__ __qualname____doc__r%rstrr'listrBr3r#r#r#r$r4s  rcsNeZdZfddZdefddZdedefdd Zd edefd d Z Z S) ExtensionsTriecstj|dSN)superr%r  __class__r#r$r%szExtensionsTrie.__init__prefixcs&|}||}fdd|DS)aC Generates all extensions of a given prefix token in the Trie. Example: ```python >>> trie = Trie() >>> trie.add("apple") >>> trie.add("app") >>> trie.add("application") >>> trie.extensions("app") ['app', 'apple', 'application'] ``` cg|]}|qSr#r#.0r(rQr#r$ 0z-ExtensionsTrie.extensions..) _get_node_collect_tokens)r!rQ prefix_noderetr#rUr$ extensionss  zExtensionsTrie.extensionsr(r.cCs*|j}|D] }||vr|S||}q|S)a Retrieves the node corresponding to the given token in the Trie. Args: token (str): The token for which the corresponding node needs to be retrieved. Returns: dict: The node in the Trie corresponding to the given token. )r)r!r(noder,r#r#r$rX2s  zExtensionsTrie._get_noder]csX|j|vr |jgng}|D]\}|jkr)||}|fdd|Dq|S)a Generates all tokens in the Trie starting from a given node. Args: node (dict): The node in the Trie from which tokens need to be generated. Returns: list: List of tokens generated from the given node. crRr#r#)rTsubtokenr(r#r$rVRrWz2ExtensionsTrie._collect_tokens..)rr0rYextend)r!r]rE subtrie_head subtokensr#r_r$rYDs   zExtensionsTrie._collect_tokens) rFrGrHr%rJr\dictrXrKrY __classcell__r#r#rOr$rLs  rLcCs>|dks|dks|dks|dkrdSt|}|dkrdSdS)z0Checks whether `char` is a whitespace character.    TZsF) unicodedatacategoryr,catr#r#r$_is_whitespaceVs  rncCs8|dks |dks |dkrdSt|}|drdSdS)z-Checks whether `char` is a control character.rfrgrhFCT)rjrk startswithrlr#r#r$ _is_controlbs   rqcCsht|}|dkr |dks$|dkr|dks$|dkr|dks$|dkr&|dkr&d St|}|d r2d Sd S) z1Checks whether `char` is a punctuation character.!/:@[`{~TPF)ordrjrkrp)r,cprmr#r#r$_is_punctuationns@  r}cC$|d}tt|t|Bt|BS)zcChecks whether the last character in text is one of a punctuation, control or whitespace character.boolrqr}rn)r- last_charr#r#r$_is_end_of_word}rcCr~)zdChecks whether the first character in text is one of a punctuation, control or whitespace character.rr)r- first_charr#r#r$_is_start_of_wordrr token_list new_tokencCs8t||}|t|kr|||krdS|||dS)zm Inserts one token to an ordered list if it does not already exist. Note: token_list must be sorted. N)bisect bisect_leftr1insert)rr insertion_idxr#r#r$!_insert_one_token_to_ordered_lists rc'seZdZdZfddZedefddZedefddZ ede e effd d Z ede ee ffd d Zejd e eee e ffde ee ffdd Zde e effddZddZddZd`deee ee fdedefddZgfdee fddZd`dedefddZd edee fd!d"Zd#d$Zd%ee ee fdeeeeffd&d'Zd(d)Zd*d+Zd,d-ej e!j"d,d.dd,d,d,d,d,ddddd-fd eee#e$fd/eeee#e$fd0ed1ed2e!d3eed4ed5ed6eed7ee d8eee e%fd9eed:eed;eded?ede&f&d@dAZ'd-ej e!j"d,d.dd,d,d,d,d,ddddd-dfdBeeeee(ee#ee)ee$ee*fd0ed1ed2e!d3eed4ed5ed6eed7ee d8eee e%fd9eed:eed;eded?edCede&f&dDdEZ+e,e-e.d-ej e!j"d,d.d,d,d,d,d,dddd-dfdFeee)e/eed,ffd0ed1ed2e!d3eed4ed6eed7ee d8ee d9eed:eed;eded?edCede&f"dGdHZ0 d`d e d5ede/e e e e1fffdIdJZ2 dadKedLeedMedeeffdNdO Z3e4d`dPedQede fdRdSZ5e4d`dPeedQedee fdTdSZ5 d`dPeeeefdQedee ee ffdUdSZ5dVede fdWdXZ6d%ee de fdYdZZ7  , -dbd[eeeefdQed\eed]ede f d^d_Z8Z9S)cPreTrainedTokenizera Base class for all slow tokenizers. Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`]. Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary. This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). c s|t_tds i_j|diddjD_tj d i|j fddj Dddd _ dS) N_added_tokens_decoderadded_tokens_decodercSi|]\}}|j|qSr#contentrTvkr#r#r$ z0PreTrainedTokenizer.__init__..csg|] }|jvr|qSr#_added_tokens_encoderrSr!r#r$rVsz0PreTrainedTokenizer.__init__..T)special_tokensFr#) r tokens_triehasattrrrpopr0rrNr% _add_tokensall_special_tokens_extended_decode_use_source_tokenizer)r!kwargsrOrr$r%s  zPreTrainedTokenizer.__init__r.cCsdSNFr#rr#r#r$is_fastszPreTrainedTokenizer.is_fastcCt)zP `int`: Size of the base vocabulary (without the added tokens). NotImplementedErrorrr#r#r$ vocab_sizeszPreTrainedTokenizer.vocab_sizecCs ddt|jdddDS)z Returns the sorted mapping from string to index. The added tokens encoder is cached for performance optimisation in `self._added_tokens_encoder` for the slow tokenizers. cSrr#rrr#r#r$rrz.cS|dSNrr#itemr#r#r$z:PreTrainedTokenizer.added_tokens_encoder..key)sortedrr0rr#r#r$added_tokens_encoders z(PreTrainedTokenizer.added_tokens_encodercCstt|jdddS)z Returns the added tokens in the vocabulary as a dictionary of index to AddedToken. Returns: `Dict[str, int]`: The added tokens. cSrrr#rr#r#r$rrz:PreTrainedTokenizer.added_tokens_decoder..r)rcrrr0rr#r#r$rsz(PreTrainedTokenizer.added_tokens_decodervaluec Cs|D]9\}}t|ttfrt|ts(td|j|jfdttttfft|tr1t|n||j|<||j t|<q| dS)Nz;The provided `added_tokens_decoder` has an element of type z, should be a dict of ) r0 isinstancerJr int TypeErrorrPrrr_update_total_vocab_size)r!rindexr(r#r#r$rs" cC|jS)aX Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from the fast call because for now we always add the tokens even if they are already in the vocabulary. This is something we should change. Returns: `Dict[str, int]`: The added tokens. rrr#r#r$get_added_vocabs z#PreTrainedTokenizer.get_added_vocabcCr)zD Size of the full vocabulary with the added tokens. )total_vocab_sizerr#r#r$__len__szPreTrainedTokenizer.__len__cCst||_dS)a! Update the size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because otherwise if there is a hole in the vocab, we will add tokenizers at a wrong index. This operation is slow and is only updated when adding tokens. N)r1 get_vocabrrr#r#r$rsz,PreTrainedTokenizer._update_total_vocab_sizeF new_tokensrc Cszd}|dur|S|}t|}|D]}t|ttfs*td|dt|dt|dkr1qt|trN||jvr This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop. Args: pair (`bool`, *optional*, defaults to `False`): Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. Returns: `int`: Number of special tokens added to sequences. N)r1 build_inputs_with_special_tokens)r!r token_ids_0 token_ids_1r#r#r$num_special_tokens_to_addVsz-PreTrainedTokenizer.num_special_tokens_to_addr-cKs$|d|j}|j|fi|\}}|rtd|dt|drM|jrMdd|jD}|dd|j D7}dd |d d }t |d d |}|rUg}|g}n |j }|j|}t|D]\}} | |vr|j|j | d} |dkr||dnd} |t|dkr||dnd} t| tr| jr| r| ||d<| jr| r| ||d<| jr| r| ddkr||d| 7<d||<qd| jr| r| ddkr| ||d||d<d||<qdt| dt| qdg} |D]} | sq| |vr| | q| || q| S)a$ Converts a string into a sequence of tokens, using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Takes care of added tokens. Args: text (`str`): The sequence to be encoded. **kwargs (additional keyword arguments): Passed along to the model-specific `prepare_for_tokenization` preprocessing method. Returns: `List[str]`: The list of tokens. split_special_tokenszKeyword arguments z not recognized.rcSsg|]}t|qSr#)reescaperTs_tokr#r#r$rVsz0PreTrainedTokenizer.tokenize..cSs$g|]}|js|jrt|jqSr#)rrrrrrr#r#r$rVs (|z)|z(.+?)cSs|dp |dS)Nrr)groupsr)mr#r#r$rsz.PreTrainedTokenizer.tokenize..Nrrrrerzy cannot be tokenized because it was not properly added to the tokenizer. This means that it is not an `AddedToken` but a )rrprepare_for_tokenizationrCwarningrrrrrjoinrsubrkeysrrBr/getr1rr rr single_word ValueErrorrr2r` _tokenize)r!r-rrescaped_special_tokspatternno_split_tokenrEir( tok_extendedleftrighttokenized_textr#r#r$tokenizems^         zPreTrainedTokenizer.tokenizecKr)a Converts a string into a sequence of tokens (string), using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Do NOT take care of added tokens. r)r!r-rr#r#r$rszPreTrainedTokenizer._tokenizerEcCsB|durdSt|tr||Sg}|D] }|||q|S)aT Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the vocabulary. Args: tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s). Returns: `int` or `List[int]`: The token id or list of token ids. N)rrJ#_convert_token_to_id_with_added_vocr2)r!rEidsr(r#r#r$convert_tokens_to_idss   z)PreTrainedTokenizer.convert_tokens_to_idscCs*|durdS||jvr|j|S||SrM)r_convert_token_to_idr!r(r#r#r$rs    z7PreTrainedTokenizer._convert_token_to_id_with_added_voccCrrMrrr#r#r$rz(PreTrainedTokenizer._convert_token_to_idNTr text_pairadd_special_tokenspadding_strategytruncation_strategy max_lengthstrideis_split_into_wordspad_to_multiple_of padding_sidereturn_tensorsreturn_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthrc sfdd}|rtd||}|dur||nd}j|fid|d|d|jd|jd|d |d | d | d | d dd| d| d|d|d|d|S)Ncst|trj|fi}|St|ttfrBt|dkrBt|dtrBr=ttjfdd|D}|S|St|ttfrXt|dkrXt|dt rX|Srbt d|dt d|d)Nrc3&|]}j|fddiVqdSrTNrrTtrr!r#r$ $zJPreTrainedTokenizer._encode_plus..get_input_ids..zInput z] is not valid. Should be a string or a list/tuple of strings when `is_split_into_words=True`.zW is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers. rrJrrrKr&r1 itertoolschainrrr-rErrr!r#r$ get_input_idss&  (  (  z7PreTrainedTokenizer._encode_plus..get_input_idsa return_offset_mapping is not available when using Python tokenizers. To use this feature, change your tokenizer to one deriving from transformers.PreTrainedTokenizerFast. More information on available tokenizers at https://github.com/huggingface/transformers/pull/2674pair_idsrpadding truncationrrrrrprepend_batch_axisTr rr r r r)rprepare_for_modelr)r!r-rrrrrrrrrrrr r r r r rrr first_ids second_idsr#rr$ _encode_plussT     z PreTrainedTokenizer._encode_plusbatch_text_or_text_pairsrc sfdd}|rtdg}|D]9}t|ttfs!|d}}nr2t|dttfs2|d}}n|\}}||}|durB||nd}|||fqj|f||||||| | | | ||| ||d}t|S)Ncst|trj|fi}|St|ttfrBt|dkrBt|dtrBr=ttjfdd|D}|S|St|ttfrXt|dkrXt|dt rX|St d)Nrc3rrrrrr#r$r[rzPPreTrainedTokenizer._batch_encode_plus..get_input_ids..z\Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.rrrr#r$rTs  (  (z=PreTrainedTokenizer._batch_encode_plus..get_input_idszreturn_offset_mapping is not available when using Python tokenizers. To use this feature, change your tokenizer to one deriving from transformers.PreTrainedTokenizerFast.r)rrrrrrrr rr r r rrr)rrrKr&r2_batch_prepare_for_modelr )r!r$rrrrrrrrrrr r r r r rrrr input_idsids_or_pair_idsrrr!r" batch_outputsr#rr$_batch_encode_plus7sF  z&PreTrainedTokenizer._batch_encode_plusbatch_ids_pairscCsi}|D]W\}}|j||fid|dtjjd|jd|d|ddddd d d | d | d | d|dddd d|d|}|D]\}}||vrSg||<|||qGq|j||j|||| d}t|| d}|S)a Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates sequences if overflowing while taking into account the special tokens and manages a moving window (with user defined stride) for overflowing tokens Args: batch_ids_pairs: list of tokenized input ids or input ids pairs rrrrrrNrr Frr r r rrrr)rrrrr ) tensor_type)r r DO_NOT_PADrr0r2padr )r!r*rrrrrrrrrr r r r rrr(r!r"outputsrrr#r#r$r%sj       z,PreTrainedTokenizer._batch_prepare_for_modelcKs||fS)a Performs any necessary transformations before tokenization. This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the `kwargs` at the end of the encoding process to be sure all the arguments have been used. Args: text (`str`): The text to prepare. is_split_into_words (`bool`, *optional*, defaults to `False`): Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) which it will tokenize. This is useful for NER or token classification. kwargs (`Dict[str, Any]`, *optional*): Keyword arguments to use for the tokenization. Returns: `Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs. r#)r!r-rrr#r#r$rsz,PreTrainedTokenizer.prepare_for_tokenizationrralready_has_special_tokenscsD|r|dur tdtj||ddSdg|rt|ndt|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`List[int]`): List of ids of the first sequence. token_ids_1 (`List[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.T)rrr/r)rrNget_special_tokens_maskr1)r!rrr/rOr#r$r0sz+PreTrainedTokenizer.get_special_tokens_maskrskip_special_tokenscCdSrMr#r!rr1r#r#r$convert_ids_to_tokens rz)PreTrainedTokenizer.convert_ids_to_tokenscCr2rMr#r3r#r#r$r4rcCst|tr||jvr|j|jS||Sg}|D]%}t|}|r'||jvr'q||jvr6||j|jq|||q|S)a Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens. Args: ids (`int` or `List[int]`): The token id (or token ids) to convert to tokens. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Returns: `str` or `List[str]`: The decoded token(s). )rrrr_convert_id_to_tokenall_special_idsr2)r!rr1rErr#r#r$r4s     rcCrrMr)r!rr#r#r$r52rz(PreTrainedTokenizer._convert_id_to_tokencCs d|S)Nre)r)r!rEr#r#r$convert_tokens_to_string5s z,PreTrainedTokenizer.convert_tokens_to_string token_idsclean_up_tokenization_spacesspaces_between_special_tokensc s|dd_j||d}t|tr|g}tjtjfddj DB}g}g} |D]-} |r<| jvrGsz.PreTrainedTokenizer._decode..rrer)rrr4rrJrrrrrr7r1r2rr9clean_up_tokenization)r!r8r1r9r:rfiltered_tokenslegacy_added_tokens sub_textscurrent_sub_textr(stringr- clean_textr#rr$_decode8sB          zPreTrainedTokenizer._decode)Fr)FNT):rFrGrHrIr%propertyrrrrrcrJrr rsetterrrrrrKrrrrrrrrrrrr,rDO_NOT_TRUNCATErr rr r#rrrr)rrr r&r%rrr0rr4r5r7rDrdr#r#rOr$rs  ,  (NP*        Z     Y      F "  r)0rIrrrrj collectionsrtypingrrrrtokenization_utils_baserr r r r r rrrrrrrutilsrrrr get_loggerrFrCSPECIAL_TOKENS_MAP_FILEADDED_TOKENS_FILETOKENIZER_CONFIG_FILErrLrnrqr}rrrKrJrrr#r#r#r$s0  < h;