o ZhM@sddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZerBddlZdZGdd d eZdS) ) annotationsN) TYPE_CHECKINGAnyDictIterableListOptionalTupleType)Document) Embeddings) VectorStorec@seZdZdZdZdZdeeefdKddZedLddZe deeefdMddZ e ddeeefdNd"d#Z d$d%d d&d'd d&gfdOd+d,Z $dPdQd-d.Z dRdSd0d1ZdTd2d3Ze deeefdUd4d5ZefdVd8d9ZefdWd;d<ZefdXd>d?Z @dYdXdAdBZ dRdZdFdGZ dRd[dIdJZdS)\VearchZlangchain_vearchZcluster_client_dbNembedding_functionr path_or_url Optional[str] table_namestrdb_nameflagintkwargsrreturnNonec Ks:z |rddl}nddl}Wn tytdw|rF|dur#td|s9|j}|d7}|ttdd7}||_ ||_ | ||_n6|durSt dd } n|} t j| s`t | t j| d } t j| srt | || | |_| |_|s|j}|d7}|ttdd7}||_||_||_dS) zSInitialize vearch vector store flag 1 for cluster,0 for standalone rNzhCould not import suitable python package. Please install it with `pip install vearch or vearch_cluster`.zPlease input url of cluster_-\/log)vearch_clustervearch ImportError ValueError_DEFAULT_CLUSTER_DB_NAMEruuiduuid4split using_db_nameurlZ VearchClusterosgetcwdreplacepathisdirmakedirsjoinZEngineusing_metapath_DEFAULT_TABLE_NAMEusing_table_nameembedding_funcr) selfrrrrrrr"r#Z metadata_pathlog_pathr9^/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/vectorstores/vearch.py__init__sJ        zVearch.__init__Optional[Embeddings]cCs|jSN)r6r7r9r9r: embeddingsNszVearch.embeddingscls Type[Vearch] documentsList[Document] embeddingc Ks<dd|D}dd|D} |jd||| ||||d|S)Return Vearch VectorStorecSg|]}|jqSr9) page_content.0dr9r9r: _z)Vearch.from_documents..cSrFr9)metadatarHr9r9r:rK`rL)textsrD metadatasrrrrNr9) from_texts) r@rBrDrrrrrrNrOr9r9r:from_documentsRs zVearch.from_documentsrN List[str]rOOptional[List[dict]]c Ks&|||||||d} | j||d| S)rE)rrDrrrr)rNrO) add_texts) r@rNrDrOrrrrr vearch_dbr9r9r:rPmszVearch.from_textstextfieldtyperMdim field_list List[dict]c srtjjtjjddddddd}fdd |D}tjd tjjd |d d ddidd}|jj||j||d}|S)z Create VectorStore Table Args: dim:dimension of vector fields_list: the field you want to store Return: code,0 for success,1 for failed )rri'ZIVFPQi )Z ncentroidsZ nsubvector) index_sizeretrieval_typeretrieval_paramcs$g|]}t|d|dqSrX)r#ZGammaFieldInfo)rIfiZ type_dictr9r:rKsz(Vearch._create_table..text_embeddingT MemoryOnlyZ cache_sizeF)namerZZis_index dimensionZmodel_id store_typeZ store_paramZ has_source)rgfields vector_field)r#ZdataTypeINTSTRINGZGammaVectorInfoZVECTORZ create_tabler5)r7r[r\Z engine_inforjrk response_coder9rcr: _create_tables2  zVearch._create_tablec CsN|jdddddddidddiddid d |d d d d}|j|j|}|S)z Create VectorStore space Args: dim:dimension of vector Return: code,0 failed for ,1 for success rgammaZFLAT metric_typeZL2)rgr_r`rarZstringvectorTrf)rZindexrhri)rWrMrd)rgZ partition_numZ replica_numZengine properties)r5r#Z create_spacer*)r7r[Z space_configrnr9r9r: _create_spaces,  zVearch._create_space Iterable[str]cKsld}|jdur|jt|}|durtd|jr|j}|j|vr1|j|j}|s1td|j |j}|j |vrL| t |d}|sLtdg} |dur|durt |||D]M\} } } i} | | d<| d| d<t| }d |tj|i| d <|j|j|j | }|d d kr| |d q\|j|j|j | }| |d q\| Stj|j|j d}tj|st |d}||}|rtd|dur4|dur4g}t |||D]%\} } } i}| |d<| d|d<t| }|tj||d <||q|j|} d}t | t |kr/td|dkr"n |d7}t | t |ks|j| S)z^ Returns: List of ids from adding the texts into the vectorstore. Nzembeddings is Nonezcreate db failed!!!rzcreate space failed!!!rWsourcerMfeaturerdstatus_id.schemazcreate table failed!!!g?r)r6Zembed_documentslistr%rr#Zlist_dbsr*Z create_dbZ list_spacesr5rvlenzipnparraylinalgnormtolistZ insert_oneappendr,r/r2r3existsroaddtimesleepdump)r7rNrOrr?Zdbs_listZcreate_db_codeZ space_listZcreate_space_codeZdocidrWrMembedZprofilesZembed_npZ insert_resZ retry_insert table_pathr[rnZ doc_itemsZ profiles_vZt_timer9r9r:rTs                    zVearch.add_textscCs|jdS)z: load vearch engine for standalone vearch N)r#loadr>r9r9r:_load*sz Vearch._loadc Ks\|std|s tdtj||d}tj|std||||||d}||S)zLoad the local specified table of standalone vearch. Returns: Success or failure of loading the local specified table zNo metadata path!!!zNo table name!!!r}z$vearch vectorbase table not exist!!!)rrrrr)r%r,r/r2rr) r@rDrrrrrrrUr9r9r: load_local0s  zVearch.load_localquerykcKs.|jdur td|j|}|||}|S)z5 Return docs most similar to query. Nembedding_func is None!!!)r6r% embed_querysimilarity_search_by_vector)r7rrrr?docsr9r9r:similarity_searchPs   zVearch.similarity_search List[float]c Ks t|}|jr/dd|tj|dgi|ddgd}|j|j|j |}|dd}n!d|tj|dggdd d d |d }|j|}|d d}g}|D]/} d} i} |jra| d} | D]} | dkrn| | } qc| dkry| | | d<qcqc| t | | dqT|S)The most k similar documents and scores of the specified query. Args: embeddings: embedding vector of the query. k: The k most similar documents to the text query. min_score: the score of similar documents to the text query Returns: The k most similar documents to the specified text query. 0 is dissimilar, 1 is the most similar. sumrdrYryrWrMrsizerjhitsr InnerProductrqZnprobersrjZis_brute_searchraZtopnr result_itemsre_sourcerxrGrM) rrrrrrr#searchr*r5rr ) r7rDrrr query_data query_resultresritemcontent meta_dataitem_keyr9r9r:r`sT    z"Vearch.similarity_search_by_vectorList[Tuple[Document, float]]cKsV|jdur td|j|}t|}|jr>dd|tj|dgi|gdd}|j |j |j |}|dd}n!d|tj|dggd d d d |d }|j |}|dd}g} |D]E} d} i} |jrt| d} | d} | D]$}|dkr| |} qv|dkr| || d<qv|jd kr|dkr| |} qvqvt | | d| f}| |qc| S)rNrrrdr)rdrWrMrrrrrrrrrreZ_scorerrWrMrxscorer)r6r%rrrrrrrr#rr*r5r r)r7rrrr?rrrrresultsrrrrrtmp_resr9r9r:similarity_search_with_scoresd       z#Vearch.similarity_search_with_scorercKs|j||fi|Sr=)r)r7rrrr9r9r:(_similarity_search_with_relevance_scoressz/Vearch._similarity_search_with_relevance_scoresidsOptional[List[str]]Optional[bool]cKsrd}g}|dus|dkr|S|D]}|jr"|j|j|j|}n|j|}||qtdd|D}|S)aIDelete the documents which have the specified ids. Args: ids: The ids of the embedding vectors. **kwargs: Other keyword arguments that subclasses might use. Returns: Optional[bool]: True if deletion is successful. False otherwise, None if not implemented. Nrcss|]}|dkVqdS)rNr9)rIir9r9r: sz Vearch.delete..) __len__rr#deleter*r5Zdel_docrall)r7rrretrr|r9r9r:rs  z Vearch.deleteDict[str, Document]c Ks*i}|dus |dkr|S|jr\dd|ii}|j|j|j|}|D]6}|ddur,q#d}i}|dD]} | d krA|d| }q4| d krN|d| |d <q4q4t||d ||d <q#|S|D]4} |j| }|ikrkq^d}i}|D]} | d kr||| }qq| d kr|| |d <qqqqt||d ||d <q^|S)zReturn docs according ids. Args: ids: The ids of the embedding vectors. Returns: Documents which satisfy the input conditions. NrrrfoundFrerrWrMrxrr|)rrr#Z mget_by_idsr*r5r Z get_doc_by_id) r7rrrrZ docs_detailrecordrZ meta_inforYidr9r9r:get sV        z Vearch.get)rr rrrrrrrrrrrr)rr<)r@rArBrCrDr rrrrrrrrrrrr)r@rArNrRrDr rOrSrrrrrrrrrrrr)r[rr\r]rr)rV)r[rrrr=)rNrwrOrSrrrrR)rr)rDr rrrrrrrrrrrr)rrrrrrrrC)rDrrrrrrrC)rrrrrrrr)r)rrrrrr)rrrrrr)__name__ __module__ __qualname__r4r&Z_DEFAULT_VERSIONr;propertyr? classmethodrQrProrvrTrr DEFAULT_TOPNrrrrrrr9r9r9r:rsh 6  / - M"  C K  r) __future__rr,rr'typingrrrrrrr r numpyrZlangchain_core.documentsr Zlangchain_core.embeddingsr Zlangchain_core.vectorstoresr r#rrr9r9r9r:s (