o Zhg@sddlmZddlZddlZddlmZmZmZmZm Z m Z m Z m Z m Z mZmZmZddlZddlZddlmZddlmZmZmZddlmZmZmZmZmZddlm Z e!e"Z#dddZ$GdddeeZ%dS)) annotationsN) AnyDictIterableListLiteralMappingOptionalSequenceSetTupleUnioncast) Embeddings)from_envget_pydantic_field_namessecret_from_env) BaseModel ConfigDictField SecretStrmodel_validator)Self num_textsinttokensList[Union[List[int], str]]batched_embeddingsList[List[float]]indices List[int] skip_emptyboolreturnList[Optional[List[float]]]c s"ddt|D}ddt|Dtt|D]%|r%t|dkr%q||||t|qg}t|D]J|}t|dkrV|dqDt|dkrd||dqDtfddt|D}tdd|Dd |fd d|DqD|S) NcSg|]}gqSr&.0_r&r&W/var/www/html/lang_env/lib/python3.10/site-packages/langchain_openai/embeddings/base.py 'z7_process_batched_chunked_embeddings..cSr%r&r&r'r&r&r*r+,r,rcs,g|]}tddt|DqS)css|] \}}||VqdSNr&)r(valweightr&r&r* Ks  zA_process_batched_chunked_embeddings...)sumzip)r( embedding)inum_tokens_in_batch total_weightr&r*r+Js css|]}|dVqdS)Nr&r(r/r&r&r*r1Uz6_process_batched_chunked_embeddings..g?csg|]}|qSr&r&r9) magnituder&r*r+Vs)rangelenappendr2r3) rrrrr!results embeddings_resultZaverager&)r5r;r6r7r*#_process_batched_chunked_embeddingss.       rBc@seZdZUdZedddZded<edddZded<dZd ed <dZ d ed < eZ d ed<ee dddddZ d ed< ede ddddZ d ed< ee ddddZd ed<ee ddddZd ed<dZded< ed ed!dddZd"ed#< ed$e d%d&gdddZd ed'< dZd(ed)<dZd*ed+<d,Zded-< d.Zded/< edd0d1Zd2ed3< dZded4<dZd5ed6< dZd ed7< d8Zd5ed9< eedZd:ed;< d8Zd5ed<< dZ d=ed><dZ!d?ed@<dAZ"dedB< dCZ#dedD< dZ$dEedF< dZ%dEedG< dZ&d5edH< e'dIddJdKZ(e)dLdMe*dodPdQZ+e)dRdMdpdTdUZ,e-dqdVdWZ.drd[d\Z/dd]dsd`daZ0dd]dsdbdcZ1 dtdudedfZ2 dtdudgdhZ3dvdkdlZ4dvdmdnZ5dS)wOpenAIEmbeddingsu OpenAI embedding model integration. Setup: Install ``langchain_openai`` and set environment variable ``OPENAI_API_KEY``. .. code-block:: bash pip install -U langchain_openai export OPENAI_API_KEY="your-api-key" Key init args — embedding params: model: str Name of OpenAI model to use. dimensions: Optional[int] = None The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. Key init args — client params: api_key: Optional[SecretStr] = None OpenAI API key. organization: Optional[str] = None OpenAI organization ID. If not passed in will be read from env var OPENAI_ORG_ID. max_retries: int = 2 Maximum number of retries to make when generating. request_timeout: Optional[Union[float, Tuple[float, float], Any]] = None Timeout for requests to OpenAI completion API See full list of supported init args and their descriptions in the params section. Instantiate: .. code-block:: python from langchain_openai import OpenAIEmbeddings embed = OpenAIEmbeddings( model="text-embedding-3-large" # With the `text-embedding-3` class # of models, you can specify the size # of the embeddings you want returned. # dimensions=1024 ) Embed single text: .. code-block:: python input_text = "The meaning of life is 42" vector = embeddings.embed_query("hello") print(vector[:3]) .. code-block:: python [-0.024603435769677162, -0.007543657906353474, 0.0039630369283258915] Embed multiple texts: .. code-block:: python vectors = embeddings.embed_documents(["hello", "goodbye"]) # Showing only the first 3 coordinates print(len(vectors)) print(vectors[0][:3]) .. code-block:: python 2 [-0.024603435769677162, -0.007543657906353474, 0.0039630369283258915] Async: .. code-block:: python await embed.aembed_query(input_text) print(vector[:3]) # multiple: # await embed.aembed_documents(input_texts) .. code-block:: python [-0.009100092574954033, 0.005071679595857859, -0.0029193938244134188] NT)defaultexcluderclient async_clientztext-embedding-ada-002strmodel Optional[int] dimensionsz Optional[str] deploymentZOPENAI_API_VERSION)rD api_version)default_factoryaliasopenai_api_versionbase_urlZOPENAI_API_BASE)rOrNopenai_api_baseZOPENAI_API_TYPE)rNopenai_api_typeZ OPENAI_PROXY openai_proxyirembedding_ctx_lengthapi_keyZOPENAI_API_KEYzOptional[SecretStr]openai_api_key organizationZ OPENAI_ORG_IDZOPENAI_ORGANIZATIONopenai_organizationz%Union[Literal['all'], Set[str], None]allowed_specialz4Union[Literal['all'], Set[str], Sequence[str], None]disallowed_speciali chunk_sizer8 max_retriestimeout)rDrOz0Optional[Union[float, Tuple[float, float], Any]]request_timeoutheadersr"tiktoken_enabledtiktoken_model_nameFshow_progress_barDict[str, Any] model_kwargsr!zUnion[Mapping[str, str], None]default_headersz!Union[Mapping[str, object], None] default_queryretry_min_secondsretry_max_secondszUnion[Any, None] http_clienthttp_async_clientcheck_embedding_ctx_lengthZforbidr&)extraZpopulate_by_nameZprotected_namespacesbefore)modevaluesr#c Cst|}|di}t|D](}||vrtd|d||vr6td|d|d|d||||<q||}|rHtd|d ||d<|S) z>Build extra kwargs from additional params that were passed in.rezFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsrrZall_required_field_namesro field_nameZinvalid_model_kwargsr&r&r* build_extras,   zOpenAIEmbeddings.build_extraafterrc Csl|jdvr td|jr|jnd|j|j|j|j|j|j d}|j r@|j s*|j r@|j }|j }|j }td|d|d||j sz|j ri|j sizddl}Wnty`}ztd |d}~ww|j|j d |_ d |j i}tjd i||j|_ |js|j r|j szddl}Wnty}ztd |d}~ww|j|j d |_ d |j i}tjd i||j|_|S) z?Validate that api key and python package exists in environment.)ZazureZazure_adZazureadzEIf you are using Azure, please use the `AzureOpenAIEmbeddings` class.N)rVrXrQr^r]rfrgzwCannot specify 'openai_proxy' if one of 'http_client'/'http_async_client' is already specified. Received: openai_proxy=z http_client=z http_async_client=rzRCould not import httpx python package. Please install it with `pip install httpx`.)proxyrlr&)rSrurWZget_secret_valuerYrRr_r]rfrgrTrlrmrFhttpx ImportErrorZClientopenaiZOpenAIr@rGZ AsyncClientZ AsyncOpenAI) selfZ client_paramsrTrlrmreZ sync_specificZasync_specificr&r&r*validate_environment*s|        z%OpenAIEmbeddings.validate_environmentcCs(d|ji|j}|jdur|j|d<|S)NrIrK)rIrerK)rparamsr&r&r*_invocation_paramsds  z#OpenAIEmbeddings._invocation_paramstexts List[str]\}} |jd r| d d } |r|j | fi|}n|| }t dt ||j D]} ||| | |j ||qq|jrzddlm}|t dt ||}Wntyt dt ||}Yn wt dt ||}|||fS)a Take the input `texts` and `chunk_size` and return 3 iterables as a tuple: We have `batches`, where batches are sets of individual texts we want responses from the openai api. The length of a single batch is `chunk_size` texts. Each individual text is also split into multiple texts based on the `embedding_ctx_length` parameter (based on number of tokens). This function returns a 3-tuple of the following: _iter: An iterable of the starting index in `tokens` for each *batch* tokens: A list of tokenized texts, where each text has already been split into sub-texts based on the `embedding_ctx_length` parameter. In the case of tiktoken, this is a list of token arrays. In the case of HuggingFace transformers, this is a list of strings. indices: An iterable of the same length as `tokens` that maps each token-array to the index of the original text in `texts`. r) AutoTokenizerzCould not import transformers python package. This is needed for OpenAIEmbeddings to work without `tiktoken`. Please install it with `pip install transformers`. )Zpretrained_model_name_or_pathF)Zadd_special_tokensZ cl100k_basecSsi|] \}}|dur||qSr.r&)r(kvr&r&r* s z.OpenAIEmbeddings._tokenize..)rZr[Z001  )tqdm)rbrIraZ transformersrrruZfrom_pretrained enumerateencoder<r=rUdecoder>tiktokenZencoding_for_modelKeyErrorZ get_encodingrZr[itemsendswithreplaceZencode_ordinaryrcZ tqdm.autor)rrr\rrZ model_namerZ tokenizerr5textZ tokenizedjZ token_chunkZ chunk_textencodingZencoder_kwargstokenr_iterr&r&r* _tokenizeksn              zOpenAIEmbeddings._tokenize)r\enginerc s|pj}||\}}}g}|D])} jjd d|| | |ij} t| ts/| } |dd| dDqt t ||||j } dd fdd fd d | DS)al Generate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. inputcs|]}|dVqdSr4Nr&r(rr&r&r*r1r:z.dataNr# List[float]csFdur!jjdddij}t|ts|}|dddSNrrrr4r&)rFcreater isinstancedict model_dumpZaverage_embedded_cached_empty_embeddingrr&r*empty_embeddings  zBOpenAIEmbeddings._get_len_safe_embeddings..empty_embeddingcsg|] }|dur |nqSr.r&r(rrr&r*r+sz=OpenAIEmbeddings._get_len_safe_embeddings..r&r#r) r\rrFrrrrrextendrBr=r! rrrr\ _chunk_sizerrrrr5responser@r&rrrr*_get_len_safe_embeddingss$    z)OpenAIEmbeddings._get_len_safe_embeddingsc s|pj}||\}}}g}|pj}tdt||D],} jjd d|| | |ijIdH} t| ts>| } | dd| dDqt t||||j } ddfd d fd d | DIdHS)a Asynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. rrNcsrrr&rr&r&r*r1r:z=OpenAIEmbeddings._aget_len_safe_embeddings..rr#rcsNdur%jjdddijIdH}t|ts|}|dddSr)rGrrrrrrrr&r*r#s  zCOpenAIEmbeddings._aget_len_safe_embeddings..empty_embeddingcs&g|]}|dur |nIdHqSr.r&rrr&r*r+.s$z>OpenAIEmbeddings._aget_len_safe_embeddings..r&r) r\rr<r=rGrrrrrrrBr!rr&rr*_aget_len_safe_embeddingss(     z*OpenAIEmbeddings._aget_len_safe_embeddings int | NonecCs|p|j}|js>g}tdt||D])}|jjdd||||i|j}t|ts/|}| dd|dDq|St t |j }|j ||dS) aMCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rrcsrrr&rr&r&r*r1Fr:z3OpenAIEmbeddings.embed_documents..rrNr&)r\rnr<r=rFrrrrrrrHrLrrrr\Z chunk_size_r@r5rrr&r&r*embed_documents0s    z OpenAIEmbeddings.embed_documentscs|p|j}|jsBg}tdt||D],}|jjdd||||i|jIdH}t|ts3|}| dd|dDq|St t |j }|j ||dIdHS) aSCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rrNcsrrr&rr&r&r*r1dr:z4OpenAIEmbeddings.aembed_documents..rrr&)r\rnr<r=rGrrrrrrrHrLrrr&r&r*aembed_documentsNs     z!OpenAIEmbeddings.aembed_documentsrrcCs||gdS)zCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. r)r)rrr&r&r* embed_queryls zOpenAIEmbeddings.embed_querycs||gIdH}|dS)zCall out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. Nr)r)rrr@r&r&r* aembed_queryws zOpenAIEmbeddings.aembed_query)rrrdr#r)r#r)r#rd)rrr\rr#r)rrrrHr\rJr#rr.)rrr\rr#r)rrHr#r)6__name__ __module__ __qualname____doc__rrF__annotations__rGrIrKrLrrPrRrSrTrUrrWrYrZr[r\r]r_r`rarbrcrrer!rfrgrirkrlrmrnrZ model_configr classmethodr}rpropertyrrrrrrrrr&r&r&r*rC[s Q                          9  c23   rC) rrrrrrrr r!r"r#r$)& __future__rloggingrvtypingrrrrrrr r r r r rrrZlangchain_core.embeddingsrZlangchain_core.utilsrrrZpydanticrrrrrZtyping_extensionsr getLoggerrloggerrBrCr&r&r&r*s 8    =