o Zhs@s6ddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZmZmZddlZddlmZddlmZddlmZmZmZddlmZmZmZm Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)e*e+Z,d$ddZ-d%ddZ.d&d'ddZ/d(ddZ0d(ddZ1edd d!d"Gd#d d eeZ2dS))) annotationsN) AnyCallableDictListLiteralMappingOptionalSequenceSetTupleUnioncast) deprecated) Embeddings)get_from_dict_or_envget_pydantic_field_namespre_init) BaseModel ConfigDictFieldmodel_validator) AsyncRetryingbefore_sleep_logretryretry_if_exception_typestop_after_attemptwait_exponential) is_openai_v1 embeddingsOpenAIEmbeddingsreturnCallable[[Any], Any]cCsnddl}tdt|jtd|j|jdt|jj t|jj Bt|jj Bt|jj Bt|jj BtttjdS)NrT multiplierminmaxreraisestopwaitr before_sleep)openairr max_retriesrretry_min_secondsretry_max_secondsrerrorTimeoutAPIErrorAPIConnectionErrorRateLimitErrorServiceUnavailableErrorrloggerloggingWARNING)rr-r:\/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/embeddings/openai.py_create_retry_decorator,s(      r<rcsddl}tdt|jtd|j|jdt|jj t|jj Bt|jj Bt|jj Bt|jj Btttjdd fd d }|S) NrTr#r$r(funcrr!csdfdd }|S) Nargsrkwargsr!rcs42z3dHW}|i|IdHS6td)Nzthis is unreachable)AssertionError)r>r?_)async_retryingr=r:r; wrapped_fbs z7_async_retry_decorator..wrap..wrapped_f)r>rr?rr!rr:)r=rCrB)r=r;wrapasz$_async_retry_decorator..wrap)r=rr!r)r-rrr.rr/r0rr1r2r3r4r5r6rr7r8r9)rr-rEr:rDr;_async_retry_decoratorGs,      rFFresponsedict skip_emptyboolcCs2tdd|dDr|sddl}|jd|S)Ncss |] }t|ddkVqdS) embeddingr#N)len).0dr:r:r; nsz"_check_response..datarz&OpenAI API returned an empty embedding)anyr-r1r3)rGrIr-r:r:r;_check_responsems rRr?c s@tr jjdi|St}|dfdd }|di|S) )Use tenacity to retry the embedding call.r?rr!cs jjdi|}t|jdSN)rIr:)clientcreaterRrIr?rGrr:r;_embed_with_retry{sz+embed_with_retry.._embed_with_retryNr:r?rr!r)rrUrVr<)rr?Zretry_decoratorrYr:rXr;embed_with_retryus r[csJtrjjdi|IdHStdfdd }|di|IdHS) rSNr?rr!cs(jjdi|IdH}t|jdSrT)rUZacreaterRrIrWrXr:r;_async_embed_with_retrysz7async_embed_with_retry.._async_embed_with_retryr:rZ)r async_clientrVrF)rr?r\r:rXr;async_embed_with_retrys r^z0.0.9z1.0z!langchain_openai.OpenAIEmbeddings)ZsinceZremovalZalternative_importc@sHeZdZUdZedddZded<edddZded<dZd ed <eZ d ed <edd dZ d ed< edddZ d ed< dZ d ed<dZ d ed<dZded< edddZd ed< edddZd ed< eZded<dZded<d Zded!< d"Zded#< edd$dZd%ed&< dZded'<dZd(ed)< dZd ed*< d+Zd(ed,< eed-Zd.ed/< d+Zd(ed0< dZd1ed2<dZd3ed4<d5Z ded6< d7Z!ded8< dZ"d9ed:< e#dd;dd?e&d^dBdCZ'e(d_dEdFZ)e*d`dGdHZ+ddIdadOdPZ,ddIdadQdRZ- SdbdcdTdUZ. SdbdcdVdWZ/dddZd[Z0ddd\d]Z1dS)er ajOpenAI embedding models. To use, you should have the ``openai`` python package installed, and the environment variable ``OPENAI_API_KEY`` set with your API key or pass it as a named parameter to the constructor. Example: .. code-block:: python from langchain_community.embeddings import OpenAIEmbeddings openai = OpenAIEmbeddings(openai_api_key="my-api-key") In order to use the library with Microsoft Azure endpoints, you need to set the OPENAI_API_TYPE, OPENAI_API_BASE, OPENAI_API_KEY and OPENAI_API_VERSION. The OPENAI_API_TYPE must be set to 'azure' and the others correspond to the properties of your endpoint. In addition, the deployment name must be passed as the model parameter. Example: .. code-block:: python import os os.environ["OPENAI_API_TYPE"] = "azure" os.environ["OPENAI_API_BASE"] = "https://Build extra kwargs from additional params that were passed in.r~zFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsrZall_required_field_namesr field_nameZinvalid_model_kwargsr:r:r; build_extras,   zOpenAIEmbeddings.build_extrarc Cst|dd|d<|dptd|d<t|dddd|d<t|d d dd|d <|dd vr9d }t|d d|d <nd}t|dd|d|d<|dpRtdpRtd|d<zddl}Wn tyftdwtr|dd vrutd|d|d|d|d|d|d|d|dd}| ds|j d i|j |d<| ds|j d i|j |d<|S| ds|j |d<|S |S)!z?Validate that api key and python package exists in environment.rnZOPENAI_API_KEYrhZOPENAI_API_BASEriZOPENAI_API_TYPE)r_rjZ OPENAI_PROXYZazureZazure_adZazureadz 2023-05-15rtirfZOPENAI_API_VERSIONrpZ OPENAI_ORG_IDZOPENAI_ORGANIZATIONrNTCould not import openai python package. Please install it with `pip install openai`.zfIf you have openai>=1.0.0 installed and are using Azure, please use the `AzureOpenAIEmbeddings` class.rwr.rrr)rmrorgrvr.rrrrUr]r:)rosgetenvr&r- ImportErrorrrrrZOpenAIrZ AsyncOpenAIZ Embedding)rrZdefault_api_versionr-Z client_paramsr:r:r;validate_environment s          z%OpenAIEmbeddings.validate_environmentc Cstr d|ji|j}|S|j|j|j|j|j|j|j|j d|j}|jdvr-|j |d<|j rJzddl }Wn t yAt dw|j |j d|_|S)Nrb)rbrwrxrmroZapi_baseZapi_typerdrenginerr)httphttps)rrbr~rwrxrnrprhrirfrcrjr-rproxy)selfZ openai_argsr-r:r:r;_invocation_paramsms8    z#OpenAIEmbeddings._invocation_params)rttexts List[str]r Optional[int]List[List[float]]cCsg}g}|jp |j}|p|j}|jsazddlm}Wn ty%tdw|j|d} t|D]/\} } | j | dd} t dt | |j D]} | | | |j }| |}|||| qDq0nkzddl}Wn tyrtdwz||}Wntytd d }||}Ynwt|D]7\} } |jd r| d d } |j | |j|jd}t dt ||j D]} ||| | |j || qq|jrzddlm}|t dt ||}Wntyt dt ||}Yn wt dt ||}g}|D])} t|fd|| | |i|j}t|ts|}| dd|dDqddt t |D}ddt t |D}t t |D]*} |j!rVt || dkrVqE||| || ||| t || qEddt t |D}t t |D]B} || }t |dkrt|fddi|j}t|ts|}|ddd}n t"j#|d|| d}|t"j$%|&|| <q|S)al Generate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. r AutoTokenizerzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`. Zpretrained_model_name_or_pathFZadd_special_tokensNCould not import tiktoken python package. This is needed in order to for OpenAIEmbeddings. Please install it with `pip install tiktoken`.5Warning: model not found. Using cl100k_base encoding. cl100k_base001  textrqrs)tqdminputcs|]}|dVqdSrKNr:rMrr:r:r;rOz.rPcSg|]}gqSr:r:rMrAr:r:r; z=OpenAIEmbeddings._get_len_safe_embeddings..cSrr:r:rr:r:r;rrr#cSrr:r:rr:r:r;rrrrKZaxisweights)'rzrbrtry transformersrrfrom_pretrained enumerateencoderangerLrldecodeappendtiktokenencoding_for_modelKeyErrorr7warning get_encodingendswithreplacerqrsr{Z tqdm.autorr[r isinstancerHextendrInpaveragelinalgnormtolist)rrrrttokensindices model_name _chunk_sizer tokenizerir tokenizedj token_chunk chunk_textrencodingrbtokenrZ_iterbatched_embeddingsrGresultsnum_tokens_in_batchr_resultaverage_embeddedrr:r:r;_get_len_safe_embeddingss                 z)OpenAIEmbeddings._get_len_safe_embeddingscs2g}g}|jp |j}|p|j}|jsbzddlm}Wn ty&tdw|j|d} t|D]/\} } | j | dd} t dt | |j D]} | | | |j }| |}|||| qEq1nkzddl}Wn tystdwz||}Wntytd d }||}Ynwt|D]7\} } |jd r| d d } |j | |j|jd}t dt ||j D]} ||| | |j || qqg}|p|j}t dt ||D]+} t|fd|| | |i|jIdH}t|ts|}|dd|dDqddt t |D}ddt t |D}t t |D]} ||| || ||| t || q$ddt t |D}t t |D]E} || }t |dkrt|fddi|jIdH}t|tsv|}|ddd}n tj|d|| d}|tj !|"|| <qQ|S)a Asynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. rrzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`.rFrNrrrrrrrrcsrrr:rr:r:r;rOurz=OpenAIEmbeddings._aget_len_safe_embeddings..rPcSrr:r:rr:r:r;rwrz>OpenAIEmbeddings._aget_len_safe_embeddings..cSrr:r:rr:r:r;rxrcSrr:r:rr:r:r;r}rrrKr)#rzrbrtryrrrrrrrrLrlrrrrrr7rrrrrqrsr^rrrHrrrrrr)rrrrtrrrrrrrrrrrrrrrbrrrGrrrrrrr:r:r;_aget_len_safe_embeddingss                z*OpenAIEmbeddings._aget_len_safe_embeddingsrcCstt|j}|j||dS)aMCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. r)rrarcrrrrtrr:r:r;embed_documentss z OpenAIEmbeddings.embed_documentscs"tt|j}|j||dIdHS)aSCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rN)rrarcrrr:r:r;aembed_documentss z!OpenAIEmbeddings.aembed_documentsr List[float]cCs||gdS)zCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. r)r)rrr:r:r; embed_querys zOpenAIEmbeddings.embed_querycs||gIdH}|dS)zCall out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. Nr)r)rrrr:r:r; aembed_querys zOpenAIEmbeddings.aembed_query)rr}r!r)rrr!r)r!r})rrrrartrr!r)r)rrrtrr!r)rrar!r)2__name__ __module__ __qualname____doc__rrU__annotations__r]rbrcrfrhrirjrlrnrpsetrqrsrtr.rwrxryrzr{rHr~rIrrr/r0rrZ model_configr classmethodrrrpropertyrrrrrrrr:r:r:r;r s )                   L $z   )rr r!r")rr r!r)F)rGrHrIrJr!rH)rr r?rr!r)3 __future__rr8rrtypingrrrrrrr r r r r rnumpyrZlangchain_core._api.deprecationrZlangchain_core.embeddingsrZlangchain_core.utilsrrrZpydanticrrrrtenacityrrrrrrZ langchain_community.utils.openair getLoggerrr7r<rFrRr[r^r r:r:r:r;s0 8       &