o
    Zh)q                     @  s  d Z ddlmZ ddlZddlZddlmZmZmZm	Z	m
Z
mZmZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& erddl'm(Z(m)Z)m*Z*m+Z+ ddl,m(Z- ddl,m)Z. ddl,m*Z/ ddl,m+Z0 ee+e0f Z1ee)e.f Z2ee*j3e/j3f Z4ee(e-f Z5d#ddZ6G dd deZ7G dd dee7Z8G dd  d ee7Z9G d!d" d"ee8Z:dS )$zRetriever wrapper for Google Vertex AI Search.

Set the following environment variables before the tests:
export PROJECT_ID=... - set to your Google Cloud project ID
export DATA_STORE_ID=... - the ID of the search engine to use for the test
    )annotationsN)TYPE_CHECKINGAnyDictListOptionalSequenceUnion)ClientOptions)InvalidArgument)MessageToDict)CallbackManagerForRetrieverRun)Document)Serializableload)BaseRetriever)BaseTool)get_from_dict_or_env)
ConfigDictFieldPrivateAttrmodel_validator)get_client_info)!ConversationalSearchServiceClientSearchRequestSearchResponseSearchServiceClientr   r   )r   r   dumpDict[str, Any]returnr   c                 C  s   t | dgdS )NZlangchain_google_community)Zvalid_namespaces)r   )r     r#   b/var/www/html/lang_env/lib/python3.10/site-packages/langchain_google_community/vertex_ai_search.py_load8      r%   c                      s  e Zd ZU ded< 	 ded< 	 dZded< 	 dZded< 	 dZd	ed
< 	 eddddZded< 	 e	ddZ
ded< 	 eddddZd5 fddZed6ddZed6ddZd7d d!Zed"d#ed8d&d'Zed9d)d*Zd:d.d/Zd;d1d2Zd;d3d4Z  ZS )<_BaseVertexAISearchRetrieverstr
project_iddata_store_idgloballocation_idZdefault_configserving_config_idNr   credentialsr      defaultgeleintengine_data_typeF)r1   bool_betaforbidT)extraarbitrary_types_allowedZvalidate_assignmentkwargsr"   Nonec                   s(   | dd}t jdi | || _d S )NbetaFr#   )popsuper__init__r7   )selfr;   r=   	__class__r#   r$   r@   X   s   
z%_BaseVertexAISearchRetriever.__init__c                 C  s   | j S )z$Get whether beta version is enabled.)r7   rA   r#   r#   r$   r=   ]   s   z!_BaseVertexAISearchRetriever.betac                 C  s   dS )NTr#   rD   r#   r#   r$   is_lc_serializableb   s   z/_BaseVertexAISearchRetriever.is_lc_serializablec                 C  s   t |  ffS N)r%   to_jsonrD   r#   r#   r$   
__reduce__f   r&   z'_BaseVertexAISearchRetriever.__reduce__beforemodevaluesr   c              
   C  s   z| ddrddlm} nddlm} W n ty& } ztd|d}~ww t|dd	|d< zt|d
d}|rBtdt ||d< W n   Y t|dd|d< |S )zValidates the environment.r=   Fr   )discoveryengine_v1beta)discoveryengine_v1zCould not import google-cloud-discoveryengine python package. Please, install vertexaisearch dependency group: poetry install --with vertexaisearchNr)   Z
PROJECT_IDsearch_engine_idZSEARCH_ENGINE_IDzLThe `search_engine_id` parameter is deprecated. Use `data_store_id` instead.r*   ZDATA_STORE_ID)	getZgoogle.cloudrM   rN   ImportErrorr   warningswarnDeprecationWarning)clsrL   rM   rN   excrO   r#   r#   r$   validate_environmenti   s>   z1_BaseVertexAISearchRetriever.validate_environment'ClientOptions'c                 C  s$   t | jdkr| j ddS d dS )Nr+   z-discoveryengine.googleapis.com)Zapi_endpoint)r
   r,   rD   r#   r#   r$   client_options   s   

z+_BaseVertexAISearchRetriever.client_optionsresults%Sequence[DiscoveryEngineSearchResult]List[Document]c              
   C  sP   g }|D ]!}t |jjdd}|tt|di |d |d dd q|S )GConverts a sequence of search results to a list of LangChain documents.TZpreserving_proto_field_namestruct_dataidname)r`   ra   Zpage_contentmetadata)r   document_pbappendr   jsondumpsrP   )rA   rZ   	documentsresultdocument_dictr#   r#   r$   #_convert_structured_search_response   s   z@_BaseVertexAISearchRetriever._convert_structured_search_response
chunk_typec           
   	   C  s   g }|D ]p}t |jjdd}|di }|r||vrq|d |ddd|di }|| D ]C}| }	|d	v rf|	d
  |dd 7  < |dkrf|	|dg |dg d d|v rf|d|	d< |t|dd|	d q0q|S )r]   Tr^   derived_struct_datar`   link )r`   sourcer_   )extractive_answersextractive_segmentsrq   Z
pageNumberrs   previous_segmentsnext_segments)rt   ru   ZrelevanceScoreZrelevance_scorecontentrb   )r   rd   re   rP   copyupdaterf   r   )
rA   rZ   rm   ri   rj   rk   rn   doc_metadatachunkZchunk_metadatar#   r#   r$   %_convert_unstructured_search_response   sD   



zB_BaseVertexAISearchRetriever._convert_unstructured_search_responsec           
   	   C  s   g }|D ]G}t |jjdd}|d}|sq|di }|d |d< |dd|d< ||vr/q|d	kr5d
nd}|| D ]}	|t|	|d|d q;q|s^td| d |dkr^td |S )r]   Tr^   rn   r_   r`   ro   rp   rq   snippetsZsnippetrv   rb   zNo z could be found.rr   zMake sure that your data store is using Advanced Website Indexing.
https://cloud.google.com/generative-ai-app-builder/docs/about-advanced-features#advanced-website-indexing)r   rd   re   rP   rf   r   print)
rA   rZ   rm   ri   rj   rk   rn   ry   Z
text_fieldrz   r#   r#   r$    _convert_website_search_response   s8   
z=_BaseVertexAISearchRetriever._convert_website_search_responser;   r   r"   r<   )r"   r6   )r"   r   )rL   r   r"   r   )r"   rX   )rZ   r[   r"   r\   )rZ   r[   rm   r(   r"   r\   )__name__
__module____qualname____annotations__r,   r-   r.   r   r5   r   r7   r   model_configr@   propertyr=   classmethodrE   rH   r   rW   rY   rl   r{   r~   __classcell__r#   r#   rB   r$   r'   <   sD   
 
%
	
/r'   c                      s  e Zd ZU dZdZded< 	 dZded< 	 dZded< 	 dZded	< 	 e	d
dddZ
ded< 	 e	ddd
dZded< 	 e	ddddZded< 	 dZded< 	 e	ddddZded< 	 e	ddddZded< 	 e	ddddZded< 	 e	ddddZded< 	 dZded< 	 dZded< 	 dZded< 	 e	d d d!dZd"ed#< 	 e Zd$ed%< e Zd&ed'< dA fd,d-ZdBd.d/ZdBd0d1ZdCd4d5ZdDd7d8ZdEd:d;ZdFd?d@Z  Z S )GVertexAISearchRetrievera  `Google Vertex AI Search` retriever.

    This retriever supports both stable (v1) and beta versions of the Discovery Engine.
    Beta features are only available when beta=True.

    For a detailed explanation of the Vertex AI Search concepts
    and configuration parameters, refer to the product documentation.
    https://cloud.google.com/generative-ai-app-builder/docs/enterprise-search-introduction
    NOptional[str]filterorder_bycanonical_filterFr6   get_extractive_answers      d   r0   r4   max_documentsmax_extractive_answer_count
   max_extractive_segment_countreturn_extractive_segment_score   num_previous_segmentsnum_next_segmentsr   r/   query_expansion_conditionspell_correction_modezOptional[Dict[Any, Any]]
boost_speczOptional[Any]custom_embeddingcustom_embedding_field_pathg        g      ?zOptional[float]custom_embedding_ratioDiscoveryEngineClient_clientr(   _serving_configr;   r   r"   r<   c                   s&   t  jdi | |   |   dS )zDInitialize the retriever with the appropriate version of the client.Nr#   )r?   r@   _validate_version_compatibility_initialize_client)rA   r;   rB   r#   r$   r@   W  s   z VertexAISearchRetriever.__init__c                 C  sX   | j | j| jd}| js(tdd | D r*tddd | D   dS dS dS )zValidate version compatibility of all components.

        Raises:
            Warning: If beta features are configured but beta=False
        )r   r   r   c                 s  s    | ]}|d uV  qd S rF   r#   ).0valuer#   r#   r$   	<genexpr>k  s    zJVertexAISearchRetriever._validate_version_compatibility.<locals>.<genexpr>zYBeta features are configured but beta=False. The following beta features will be ignored:c                 S  s   g | ]
\}}|d ur|qS rF   r#   )r   kvr#   r#   r$   
<listcomp>n  s    zKVertexAISearchRetriever._validate_version_compatibility.<locals>.<listcomp>N)	r   r   r   r=   anyrL   rR   rS   items)rA   Zbeta_featuresr#   r#   r$   r   _  s   z7VertexAISearchRetriever._validate_version_compatibilityc              
   C  s   z| j rddlm} nddlm} W n ty# } ztd|d}~ww || j| jtddd| _| jj	| j
| j| j| jd| _dS )	z>Initialize the appropriate version of the SearchServiceClient.r   r   zCould not import google-cloud-discoveryengine python package. Please, install vertexaisearch dependency group: `pip install langchain-google-community[vertexaisearch]`Nvertex-ai-searchmoduler.   rY   Zclient_infoprojectlocationZ
data_storeserving_config)r=   #google.cloud.discoveryengine_v1betar   google.cloud.discoveryengine_v1rQ   r.   rY   r   r   serving_config_pathr)   r,   r*   r-   r   )rA   r   rV   r#   r#   r$   r   q  s0   z*VertexAISearchRetriever._initialize_clientqueryr!   c              	   C  s   i }| j s|S | jdus| jdurZ| jdu rtd| jdu r#td| jdu r,tdddlm} |jj| j| j	|d}|
|j|gd| j d	| j d
d| j  dd |S )z7Get parameters that are only available in beta version.NzUPlease provide a custom embedding model if you provide a custom_embedding_field_path.zUPlease provide a custom_embedding_field_path if you provide a custom embedding model.zqPlease provide a custom_embedding_ratio if you provide a custom embedding model or a custom_embedding_field_path.r   r   )Z
field_pathZvector)Zembedding_vectorsz * dotProduct(z) + r   z * relevance_score)Zembedding_specZranking_expression)r=   r   r   
ValueErrorr   r   r   ZEmbeddingSpecZEmbeddingVectorZembed_queryrx   )rA   r   paramsr   Zembedding_vectorr#   r#   r$   _get_beta_specific_params  sD   





z1VertexAISearchRetriever._get_beta_specific_paramsOptional[Dict[str, Any]]c                 C  s   | j r
ddlm} nddlm} | jdkr6| jr!|jj| jd}n|jj| j	| j
| j| jd}t|d}|S | jdkr?d}|S | jdkrWt|jj| jd|jjd	d
d}|S tdd| j  )zPrepares a ContentSpec object.

        Note:
            The ContentSearchSpec is identical between beta and stable versions,
            but we import from the appropriate version for consistency.
        r   r   )r   )r   r   r   r   )extractive_content_specr   Nr/   T)Zreturn_snippet)r   Zsnippet_spec]Only data store type 0 (Unstructured), 1 (Structured),or 2 (Website) are supported currently. Got )r=   r   r   r   r5   r   ContentSearchSpecZExtractiveContentSpecr   r   r   r   r   dictZSnippetSpecNotImplementedError)rA   r   r   content_search_specr#   r#   r$   _get_content_spec_kwargs  sH   





z0VertexAISearchRetriever._get_content_spec_kwargs'DiscoveryEngineSearchRequest'c           	      C  s   | j r
ddlm} nddlm} |j| jd}|j| jd}|  }|dur.|j	di |nd}|| j
| j| j| j| j|||| jrJ|jdi | jndd
}| |}|| |di |S )z Prepares a SearchRequest object.r   r   )	conditionrJ   N)
r   r   r   r   r   Z	page_sizer   query_expansion_specspell_correction_specr   r#   )r=   r   r   r   ZQueryExpansionSpecr   ZSpellCorrectionSpecr   r   r   r   r   r   r   r   r   Z	BoostSpecr   rx   )	rA   r   r   r   r   Zcontent_search_spec_kwargsr   Zrequest_paramsZbeta_paramsr#   r#   r$   _create_search_request  s<   

z.VertexAISearchRetriever._create_search_requestrun_managerr   r\   c             
   C  s   |  |}z| j|}W n ty" } z	t||jd d}~ww | jdkr8| jr-dnd}| |j	|}|S | jdkrE| 
|j	}|S | jdkrZ| jrOdnd}| |j	|}|S td	d
| j  )#Get documents relevant for a query.z9 This might be due to engine_data_type not set correctly.Nr   rr   rs   r   r/   r|   r   r   )r   r   searchr   typemessager5   r   r{   rZ   rl   r~   r   )rA   r   r   Zsearch_requestresponserV   rm   ri   r#   r#   r$   _get_relevant_documents!  sF   





z/VertexAISearchRetriever._get_relevant_documentsr   )r"   r<   )r   r(   r"   r!   r"   r   )r   r(   r"   r   r   r(   r   r   r"   r\   )!r   r   r   __doc__r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r@   r   r   r   r   r   r   r   r#   r#   rB   r$   r     sV   
 



 
1
4+r   c                      sd   e Zd ZU dZdZded< 	 e Zded< e Zded< e	dd	d
Z
d fddZdddZ  ZS ) VertexAIMultiTurnSearchRetrieverzGoogle Vertex AI Search retriever for multi-turn conversations.
    Supports both stable (v1) and beta versions of the Discovery Engine API.
    -r(   conversation_id#DiscoveryEngineConversationalClientr   r   ignoreTr9   r:   r;   r   c                   s   t  jd
i | | jrddlm} nddlm} || j| jtddd| _	| j	j
| j| j| j| jd| _| jdkrDtdd	| j  d S )Nr   r   r   r   r   r   r   zOData store type 1 (Structured)is not currently supported for multi-turn search.r   r#   )r?   r@   r=   r   r   r   r.   rY   r   r   r   r)   r,   r*   r-   r   r5   r   )rA   r;   r   rB   r#   r$   r@   [  s,   

z)VertexAIMultiTurnSearchRetriever.__init__r   r   r   r"   r\   c                C  s   | j rddlm}m} nddlm}m} || j| j| j| j	| j
| j||dd}| j|}| jdkr<| |jdS | |jdS )r   r   )ConverseConversationRequest	TextInput)input)ra   r   r   r/   rr   )r=   r   r   r   r   r   Zconversation_pathr)   r,   r*   r   r   Zconverse_conversationr5   r~   Zsearch_resultsr{   )rA   r   r   r   r   requestr   r#   r#   r$   r   {  s*   

z8VertexAIMultiTurnSearchRetriever._get_relevant_documents)r;   r   r   )r   r   r   r   r   r   r   r   r   r   r   r@   r   r   r#   r#   rB   r$   r   K  s   
  r   c                      sx   e Zd ZU dZdZded< 	 dZded< 	 dZd	ed
< 	 ee	dZ
ded< 	 edddZd fddZdddZ  ZS )VertexAISearchSummaryToolzClass that exposes a tool to interface with an App in Vertex Search and
    Conversation and get the summary of the documents retrieved.
    Supports both stable (v1) and beta versions of the Discovery Engine API.
    Nr   summary_promptr   r4   summary_result_countTr6   summary_include_citations)default_factoryr!   summary_spec_kwargsr8   r   r"   r   c                   sd   | j r
ddlm} nddlm} t  pi }|jjd| j| j	|jjj
| jdd| j|d< |S )zAdds additional summary_spec parameters to the configuration of the search.
        Returns:
            kwargs for the specification of the content.
        r   r   )preamble)r   Zinclude_citationsZmodel_prompt_specZsummary_specNr#   )r=   r   r   r   r?   r   r   ZSummarySpecr   r   ZModelPromptSpecr   r   )rA   r   r;   rB   r#   r$   r     s   
	z2VertexAISearchSummaryTool._get_content_spec_kwargs
user_queryr(   c                 C  s   |  |}| j|}|jjS )zRuns the tool.
        Args:
            search_query: The query to run by the agent.
        Returns:
            The response from the agent.
        )r   r   r   summaryZsummary_text)rA   r   r   r   r#   r#   r$   _run  s   
zVertexAISearchSummaryTool._runr   )r   r(   r"   r(   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r#   r#   rB   r$   r     s    
 r   )r    r!   r"   r   );r   
__future__r   rg   rR   typingr   r   r   r   r   r   r	   Zgoogle.api_core.client_optionsr
   Zgoogle.api_core.exceptionsr   Zgoogle.protobuf.json_formatr   Zlangchain_core.callbacksr   Zlangchain_core.documentsr   Zlangchain_core.loadr   r   Zlangchain_core.retrieversr   Zlangchain_core.toolsr   Zlangchain_core.utilsr   Zpydanticr   r   r   r   Z!langchain_google_community._utilsr   r   r   r   r   r   r   Z%BetaConversationalSearchServiceClientZBetaSearchRequestZBetaSearchResponseZBetaSearchServiceClientr   ZDiscoveryEngineSearchRequestZSearchResultZDiscoveryEngineSearchResultr   r%   r'   r   r   r   r#   r#   r#   r$   <module>   sL    $

 L  FV