o
    'Zh6                     @   s`   d dl mZmZ dZdZdZdZdZdd Zd	d
 Z	dd Z
dd Zdd Zee	e
eedZdS )    )data_gym_to_mergeable_bpe_ranksload_tiktoken_bpez<|endoftext|>z<|fim_prefix|>z<|fim_middle|>z<|fim_suffix|>z<|endofprompt|>c                  C   s    t ddd} ddd| tdidS )	NzIhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpezLhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json)Zvocab_bpe_fileZencoder_json_filegpt2Q  J's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+P  nameZexplicit_n_vocabpat_strmergeable_ranksspecial_tokens)r   	ENDOFTEXTr    r   Q/var/www/html/lang_env/lib/python3.10/site-packages/tiktoken_ext/openai_public.pyr   
   s   r   c                  C      t d} ddd| tdidS )NzGhttps://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken	r50k_baser   r   r   r   r   r   r   r   r   r   r         r   c                  C   r   )NGhttps://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken	p50k_basei  r   r   r   r   r   r   r   r   r   %   r   r   c                  C   s*   t d} tdtdtdtdi}dd| |dS )	Nr   r   r   ij  ik  	p50k_editr   r	   r
   r   r   )r   r   
FIM_PREFIX
FIM_MIDDLE
FIM_SUFFIXr   r   r   r   r   r   2   s   r   c               
   C   s.   t d} tdtdtdtdtdi}dd| |d	S )
NzIhttps://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktokeni i i i i cl100k_basezs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+r   )r   r   r   r   r   ENDOFPROMPTr   r   r   r   r   ?   s   r   )r   r   r   r   r   N)Ztiktoken.loadr   r   r   r   r   r   r   r   r   r   r   r   ZENCODING_CONSTRUCTORSr   r   r   r   <module>   s"    
