o Zh{/@sddlmZddlZddlZddlmZmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZmZmZmZmZmZddlmZmZeeZedd d ZGd d d eeZGd d d eZ Gddde!e Z"eddGdddZ#dddZ$dS)) annotationsN)ABCabstractmethod) dataclass)Enum) AbstractSetAnyCallable CollectionIterableListLiteralOptionalSequenceTypeTypeVarUnion)BaseDocumentTransformerDocumentTS TextSplitter)boundc@seZdZdZddedddfdBddZedCddZ dDdEddZdFd"d#Z dGd'd(Z dHd+d,Z e dId0d1Z e d2ded3fdJd=d>ZdKd@dAZdS)Lrz)Interface for splitting text into chunks.iFT chunk_sizeint chunk_overlaplength_functionCallable[[str], int]keep_separator$Union[bool, Literal['start', 'end']]add_start_indexboolstrip_whitespacereturnNonecCsF||krtd|d|d||_||_||_||_||_||_dS)adCreate a new TextSplitter. Args: chunk_size: Maximum size of chunks to return chunk_overlap: Overlap in characters between chunks length_function: Function that measures the length of given chunks keep_separator: Whether to keep the separator and where to place it in each corresponding chunk (True='start') add_start_index: If `True`, includes chunk's start index in metadata strip_whitespace: If `True`, strips whitespace from the start and end of every document zGot a larger chunk overlap (z) than chunk size (z), should be smaller.N) ValueError _chunk_size_chunk_overlap_length_functionZ_keep_separator_add_start_index_strip_whitespace)selfrrrrr r"r,T/var/www/html/lang_env/lib/python3.10/site-packages/langchain_text_splitters/base.py__init__!s zTextSplitter.__init__textstr List[str]cCsdS)z$Split text into multiple components.Nr,)r+r/r,r,r- split_textBszTextSplitter.split_textNtexts metadatasOptional[List[dict]]List[Document]c Cs|pigt|}g}t|D]=\}}d}d}||D]/} t||} |jr@|||j} || td| }|| d<t| }t | | d} | | qq|S)z&Create documents from a list of texts.r start_index) page_contentmetadata) len enumerater2copydeepcopyr)r'findmaxrappend) r+r3r4Z _metadatas documentsir/indexZprevious_chunk_lenchunkr9offsetZnew_docr,r,r-create_documentsFs    zTextSplitter.create_documentsrAIterable[Document]cCs:gg}}|D]}||j||jq|j||dS)zSplit documents.)r4)r@r8r9rF)r+rAr3r4docr,r,r-split_documentsZs  zTextSplitter.split_documentsdocs separator Optional[str]cCs(||}|jr |}|dkrdS|S)N)joinr*strip)r+rJrKr/r,r,r- _join_docsbs zTextSplitter._join_docssplits Iterable[str]c Csz||}g}g}d}|D]}||}||t|dkr|nd|jkr||jkr6td|d|jt|dkr|||} | durK|| ||jkse||t|dkr[|nd|jkr|dkr|||dt|dkrt|nd8}|dd}||jkse||t|dkr|nd|jkr|dkse||||t|dkr|nd7}q |||} | dur|| |S)NrzCreated a chunk of size z%, which is longer than the specified )r(r:r&loggerwarningrPr@r') r+rQrKZ separator_lenrJZ current_doctotald_lenrHr,r,r- _merge_splitsksN            zTextSplitter._merge_splits tokenizerrkwargsc sZzddlm}t|stdd fdd }Wn ty#td w|dd |i|S)z>Text splitter that uses HuggingFace tokenizer to count length.r)PreTrainedTokenizerBasezATokenizer received was not an instance of PreTrainedTokenizerBaser/r0r#rcst|SNr:encoder/rZr,r-_huggingface_tokenizer_lengthszNTextSplitter.from_huggingface_tokenizer.._huggingface_tokenizer_lengthz`Could not import transformers python package. Please install it with `pip install transformers`.rNr/r0r#rr,)Z transformersr\ isinstancer% ImportError)clsrZr[r\rbr,rar-from_huggingface_tokenizers   z'TextSplitter.from_huggingface_tokenizergpt2allrfType[TS] encoding_name model_nameallowed_special'Union[Literal['all'], AbstractSet[str]]disallowed_special&Union[Literal['all'], Collection[str]]rc  szddl}Wn tytdw|dur||n||d fdd }t|tr<||d }i||}|d d |i|S)z9Text splitter that uses tiktoken encoder to count length.rNzCould not import tiktoken python package. This is needed in order to calculate max_tokens_for_prompt. Please install it with `pip install tiktoken`.r/r0r#rcstj|dSN)rmror^r`rmroencr,r-_tiktoken_encodersz=TextSplitter.from_tiktoken_encoder.._tiktoken_encoder)rkrlrmrorrcr,)tiktokenreencoding_for_model get_encoding issubclassTokenTextSplitter) rfrkrlrmror[rurt extra_kwargsr,rrr-from_tiktoken_encoders&      z"TextSplitter.from_tiktoken_encoderSequence[Document]cKs|t|S)z2Transform sequence of documents by splitting them.)rIlist)r+rAr[r,r,r-transform_documentssz TextSplitter.transform_documents)rrrrrrrrr r!r"r!r#r$r/r0r#r1r])r3r1r4r5r#r6)rArGr#r6)rJr1rKr0r#rL)rQrRrKr0r#r1)rZrr[rr#r)rfrjrkr0rlrLrmrnrorpr[rr#r)rAr|r[rr#r|)__name__ __module__ __qualname____doc__r:r.rr2rFrIrPrY classmethodrgsetr{r~r,r,r,r-rs2 !    * +cs8eZdZdZddedfdfdd ZdddZZS)ryz/Splitting text to tokens using model tokenizer.rhNrirkr0rlrLrmrnrorpr[rr#r$c sjtjdi|zddl}Wn tytdw|dur%||}n||}||_||_||_dS)zCreate a new TextSplitter.rNzCould not import tiktoken python package. This is needed in order to for TokenTextSplitter. Please install it with `pip install tiktoken`.r,) superr.rurervrw _tokenizer_allowed_special_disallowed_special)r+rkrlrmror[rurs __class__r,r-r.s      zTokenTextSplitter.__init__r/r1cs2d fdd }tjjjj|d}t||dS) aSplits the input text into smaller chunks based on tokenization. This method uses a custom tokenizer configuration to encode the input text into tokens, processes the tokens in chunks of a specified size with overlap, and decodes them back into text chunks. The splitting is performed using the `split_text_on_tokens` function. Args: text (str): The input text to be split into smaller chunks. Returns: List[str]: A list of text chunks, where each chunk is derived from a portion of the input text based on the tokenization and chunking rules. _textr0r# List[int]csjj|jjdSrq)rr_rr)rr+r,r-_encode s z-TokenTextSplitter.split_text.._encode)rtokens_per_chunkdecoder_)r/rZN)rr0r#r) Tokenizerr'r&rrsplit_text_on_tokens)r+r/rrZr,rr-r2s zTokenTextSplitter.split_text) rkr0rlrLrmrnrorpr[rr#r$r)rrrrrr.r2 __classcell__r,r,rr-rysryc@sxeZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdS)Languagez"Enum of the programming languages.cppgojavakotlinjstsphpprotopythonrstrubyrustscalaswiftmarkdownlatexhtmlZsolcsharpcobolcluaperlhaskellelixir powershellN)rrrrZCPPZGOZJAVAZKOTLINZJSrPHPPROTOPYTHONZRSTZRUBYZRUSTZSCALAZSWIFTMARKDOWNZLATEXHTMLZSOLZCSHARPCOBOLCZLUAZPERLZHASKELLZELIXIRZ POWERSHELLr,r,r,r-rs8rT)frozenc@s8eZdZUdZded< ded< ded< ded<d S) rzTokenizer data class.rrrzCallable[[List[int]], str]rzCallable[[str], List[int]]r_N)rrrr__annotations__r,r,r,r-r;s rr/r0rZr#r1cCsg}||}d}t||jt|}|||}|t|krN||||t|kr0 |S||j|j7}t||jt|}|||}|t|ks|S)z6Split incoming text and return chunks using tokenizer.r)r_minrr:r@rr)r/rZrQZ input_idsZ start_idxZcur_idxZ chunk_idsr,r,r-rIs      r)r/r0rZrr#r1)% __future__rr<loggingabcrr dataclassesrenumrtypingrrr r r r r rrrrrZlangchain_core.documentsrr getLoggerrrTrrryr0rrrr,r,r,r-s"   8  @?