o Zh@sddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZddlZddlmZddlmZddlmZddlmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&e'e(Z)ddddZ*dddZ+GdddeZ,dS)) annotationsN)Path) AnyCallableDictIterableListOptionalSequenceSizedTupleUnionDocument) Embeddings)run_in_executor) VectorStore) AddableMixinDocstore)InMemoryDocstore)DistanceStrategymaximal_marginal_relevanceno_avx2Optional[bool]returnrcCs\|durdtjvrttd}z|rddlm}W|Sddl}W|Sty-tdw)aM Import faiss if available, otherwise raise error. If FAISS_NO_AVX2 environment variable is set, it will be considered to load FAISS with no AVX2 optimization. Args: no_avx2: Load FAISS strictly with no AVX2 optimization so that the vectorstore is portable and compatible with other devices. NZ FAISS_NO_AVX2r) swigfaisszCould not import faiss python package. Please install it with `pip install faiss-gpu` (for CUDA supported GPU) or `pip install faiss-cpu` (depending on Python version).)osenvironboolgetenvfaissr ImportError)rr r"]/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/vectorstores/faiss.pydependable_faiss_import's   r$xyx_namestry_nameNonec CsZt|tr+t|tr+t|t|kr+t|d|d|dt|d|dt| dS)Nz and z% expected to be equal length but len(z)=z and len() isinstancer len ValueError)r%r&r'r)r"r"r#_len_check_if_sizedBs$ r.c@seZdZdZddejfdddZedddZdddZ dddZ dd"d#Z dd$d%Z  ddd-d.Z  ddd1d2Z  ddd3d4Z  ddd7d8Z 9  :dddBdCZ 9  :dddDdEZ 9  :dddGdHZ 9  :dddIdJZ 9  :dddMdNZ 9  :dddOdPZ 9  :dddQdRZ 9  :dddSdTZd9d:dUddVddYdZZd9d:dUddVdd[d\Z 9 : U ddd]d^Z 9 : U ddd_d`Z 9 : U dddadbZ 9 : U dddcddZdddfdgZddjdkZ e!dddejfddmdnZ"e!  dddodpZ#e!  dddrdsZ$e!  dddtduZ%e!  dddvdwZ&dddzd{Z'e! ddd|dd~dZ(dddZ)e!dd|dddZ*dddZ+ 9  :ddddZ, 9  :ddddZ-e.dddZ/dddZ0dS)FAISSuFAISS vector store integration. See [The FAISS Library](https://arxiv.org/pdf/2401.08281) paper. Setup: Install ``langchain_community`` and ``faiss-cpu`` python packages. .. code-block:: bash pip install -qU langchain_community faiss-cpu Key init args — indexing params: embedding_function: Embeddings Embedding function to use. Key init args — client params: index: Any FAISS index to use. docstore: Docstore Docstore to use. index_to_docstore_id: Dict[int, str] Mapping of index to docstore id. Instantiate: .. code-block:: python import faiss from langchain_community.vectorstores import FAISS from langchain_community.docstore.in_memory import InMemoryDocstore from langchain_openai import OpenAIEmbeddings index = faiss.IndexFlatL2(len(OpenAIEmbeddings().embed_query("hello world"))) vector_store = FAISS( embedding_function=OpenAIEmbeddings(), index=index, docstore= InMemoryDocstore(), index_to_docstore_id={} ) Add Documents: .. code-block:: python from langchain_core.documents import Document document_1 = Document(page_content="foo", metadata={"baz": "bar"}) document_2 = Document(page_content="thud", metadata={"bar": "baz"}) document_3 = Document(page_content="i will be deleted :(") documents = [document_1, document_2, document_3] ids = ["1", "2", "3"] vector_store.add_documents(documents=documents, ids=ids) Delete Documents: .. code-block:: python vector_store.delete(ids=["3"]) Search: .. code-block:: python results = vector_store.similarity_search(query="thud",k=1) for doc in results: print(f"* {doc.page_content} [{doc.metadata}]") .. code-block:: python * thud [{'bar': 'baz'}] Search with filter: .. code-block:: python results = vector_store.similarity_search(query="thud",k=1,filter={"bar": "baz"}) for doc in results: print(f"* {doc.page_content} [{doc.metadata}]") .. code-block:: python * thud [{'bar': 'baz'}] Search with score: .. code-block:: python results = vector_store.similarity_search_with_score(query="qux",k=1) for doc, score in results: print(f"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]") .. code-block:: python * [SIM=0.335304] foo [{'baz': 'bar'}] Async: .. code-block:: python # add documents # await vector_store.aadd_documents(documents=documents, ids=ids) # delete documents # await vector_store.adelete(ids=["3"]) # search # results = vector_store.asimilarity_search(query="thud",k=1) # search with score results = await vector_store.asimilarity_search_with_score(query="qux",k=1) for doc,score in results: print(f"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]") .. code-block:: python * [SIM=0.335304] foo [{'baz': 'bar'}] Use as Retriever: .. code-block:: python retriever = vector_store.as_retriever( search_type="mmr", search_kwargs={"k": 1, "fetch_k": 2, "lambda_mult": 0.5}, ) retriever.invoke("thud") .. code-block:: python [Document(metadata={'bar': 'baz'}, page_content='thud')] NFembedding_function/Union[Callable[[str], List[float]], Embeddings]indexrdocstorerindex_to_docstore_idDict[int, str]relevance_score_fn"Optional[Callable[[float], float]] normalize_L2rdistance_strategyrcCsnt|ts td||_||_||_||_||_||_ ||_ |jt j kr3|j r5t d|jdSdSdS)z%Initialize with necessary components.t`embedding_function` is expected to be an Embeddings object, support for passing in a function will soon be removed.z2Normalizing L2 is not applicable for metric type: N)r+rloggerwarningr0r2r3r4r9override_relevance_score_fn _normalize_L2rEUCLIDEAN_DISTANCEwarningswarn)selfr0r2r3r4r6r8r9r"r"r#__init__s(  zFAISS.__init__rOptional[Embeddings]cCst|jtr |jSdSN)r+r0rrBr"r"r# embeddingss zFAISS.embeddingstexts List[str]List[List[float]]cs*tjtr j|Sfdd|DS)Ncsg|]}|qSr")r0).0textrFr"r# z*FAISS._embed_documents..)r+r0rembed_documentsrBrHr"rFr#_embed_documentss  zFAISS._embed_documentsc(t|jtr|j|IdHStdNr:)r+r0raembed_documents ExceptionrPr"r"r#_aembed_documentss  zFAISS._aembed_documentsrLr( List[float]cCs"t|jtr |j|S||SrE)r+r0rZ embed_queryrBrLr"r"r# _embed_querys   zFAISS._embed_querycrRrS)r+r0rZ aembed_queryrUrXr"r"r# _aembed_querys  zFAISS._aembed_query Iterable[str]rGIterable[List[float]] metadatasOptional[Iterable[dict]]idsOptional[List[str]]c s t}t|jtstd|jdt||dd|p!dd|D}t||dd|p1dd |D}d dt|||D}t||d d |rTt|tt|krTtd t j |t j d}|j rd| ||j||jddt||Dt|jfddt|D} |j| |S)NzSIf trying to add texts, the underlying docstore should support adding items, which z does notrHr]cSsg|]}ttqSr")r(uuiduuid4rK_r"r"r#rM(zFAISS.__add..r_css|]}iVqdSrEr"rcr"r"r# +szFAISS.__add..cSs g|] \}}}t|||dqS))idZ page_contentmetadatar)rKid_tmr"r"r#rM,s  documentsrGz$Duplicate ids found in the ids list.ZdtypecSi|]\}}||qSr"r")rKridocr"r"r# <rNzFAISS.__add..csi|] \}}||qSr"r")rKjri starting_lenr"r#rp>)r$r+r3rr-r.zipr,setnparrayfloat32r>r8r2addr4 enumerateupdate) rBrHrGr]r_r Z _metadatasrlvector index_to_idr"rrr#Z__adds4      z FAISS.__addOptional[List[dict]]kwargscKs$t|}||}|j||||dS)alRun more texts through the embeddings and add to the vectorstore. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. r]r_)listrQ _FAISS__addrBrHr]r_rrGr"r"r# add_textsBs zFAISS.add_textscs,t|}||IdH}|j||||dS)aRun more texts through the embeddings and add to the vectorstore asynchronously. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. Nr)rrVrrr"r"r# aadd_textsWszFAISS.aadd_textstext_embeddings!Iterable[Tuple[str, List[float]]]cKst|\}}|j||||dS)aAdd the given texts and embeddings to the vectorstore. Args: text_embeddings: Iterable pairs of string and embedding to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. r)rur)rBrr]r_rrHrGr"r"r#add_embeddingsms zFAISS.add_embeddings embeddingkintfilter)Optional[Union[Callable, Dict[str, Any]]]fetch_kList[Tuple[Document, float]]c s@t}tj|gtjd}|jr|||j||dur|n|\}} g} |dur.||} t | dD]C\} } | dkr=q4|j | }|j |}t |t sWtd|d||durl| |jrk| ||d| fq4| ||d| fq4|ddur|jtjtjfvrtjntjfdd | D} | d|S) aReturn docs most similar to query. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Union[Callable, Dict[str, Any]]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. rmNrCould not find document for id , got score_thresholdcs"g|] \}}|r||fqSr"r")rKroZ similaritycmprr"r#rMs z@FAISS.similarity_search_with_score_by_vector..)r$rwrxryr>r8r2search_create_filter_funcr{r4r3r+rr-rhappendgetr9rMAX_INNER_PRODUCTZJACCARDoperatorgele)rBrrrrrr r}scoresindicesdocs filter_funcrqi_idror"rr#&similarity_search_with_score_by_vectors@          z,FAISS.similarity_search_with_score_by_vectorcs&td|j|f|||d|IdHS)aReturn docs most similar to query asynchronously. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, Any]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. Nrrr)rr)rBrrrrrr"r"r#'asimilarity_search_with_score_by_vectors z-FAISS.asimilarity_search_with_score_by_vectorquerycKs(||}|j||f||d|}|S)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. rr)rYrrBrrrrrrrr"r"r#similarity_search_with_scores z"FAISS.similarity_search_with_scorecs6||IdH}|j||f||d|IdH}|S)aReturn docs most similar to query asynchronously. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. Nr)rZrrr"r"r#asimilarity_search_with_score s z#FAISS.asimilarity_search_with_scoreOptional[Dict[str, Any]]List[Document]cK(|j||f||d|}dd|DS)aYReturn docs most similar to embedding vector. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rcSg|]\}}|qSr"r"rKrordr"r"r#rMMz5FAISS.similarity_search_by_vector..)rrBrrrrrdocs_and_scoresr"r"r#similarity_search_by_vector/sz!FAISS.similarity_search_by_vectorc0|j||f||d|IdH}dd|DS)ahReturn docs most similar to embedding vector asynchronously. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rNcSrr"r"rr"r"r#rMmrz6FAISS.asimilarity_search_by_vector..)rrr"r"r#asimilarity_search_by_vectorOs z"FAISS.asimilarity_search_by_vectorcKr)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rcSrr"r"rr"r"r#rMrz+FAISS.similarity_search..)rrBrrrrrrr"r"r#similarity_searchoszFAISS.similarity_searchcr)aReturn docs most similar to query asynchronously. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rNcSrr"r"rr"r"r#rMrz,FAISS.asimilarity_search..)rrr"r"r#asimilarity_searchs zFAISS.asimilarity_search?rr lambda_multrrfloatcs\jtj|gtjd|dur|n|d\}}|durY|}g} |dD]+} | dkr.q'j| } j| } t| t sHt d| d| || j rR| | q't| g}fdd |dD} t tj|gtjd| ||d }g}|D]4} |d| dkrqwj|d| } j| } t| t st d| d| | | |d| fqw|S) azReturn docs and their similarity scores selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents and similarity scores selected by maximal marginal relevance and score for each. rmNrrrrcs$g|]}|dkrjt|qS)r)r2Z reconstructr)rKrrFr"r#rMs$zLFAISS.max_marginal_relevance_search_with_score_by_vector..)rr)r2rrwrxryrr4r3r+rr-rhrr)rBrrrrrrrrZfiltered_indicesrrrorGZ mmr_selectedrr"rFr#2max_marginal_relevance_search_with_score_by_vectorsF          z8FAISS.max_marginal_relevance_search_with_score_by_vectorc s td|j|||||dIdHS)aReturn docs and their similarity scores selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents and similarity scores selected by maximal marginal relevance and score for each. Nr)rr)rBrrrrrr"r"r#3amax_marginal_relevance_search_with_score_by_vectors z9FAISS.amax_marginal_relevance_search_with_score_by_vectorcKs"|j|||||d}dd|DS)aReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. rcSrr"r"rr"r"r#rM)rzAFAISS.max_marginal_relevance_search_by_vector..)rrBrrrrrrrr"r"r#'max_marginal_relevance_search_by_vector s z-FAISS.max_marginal_relevance_search_by_vectorcs*|j|||||dIdH}dd|DS)a(Return docs selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. rNcSrr"r"rr"r"r#rMJrzBFAISS.amax_marginal_relevance_search_by_vector..)rrr"r"r#(amax_marginal_relevance_search_by_vector+s   z.FAISS.amax_marginal_relevance_search_by_vectorc Ks*||}|j|f||||d|}|S)aReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering (if needed) to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. r)rYr rBrrrrrrrrr"r"r#max_marginal_relevance_searchLs z#FAISS.max_marginal_relevance_searchc s8||IdH}|j|f||||d|IdH}|S)a+Return docs selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering (if needed) to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. Nr)rZrrr"r"r#amax_marginal_relevance_searchqs z$FAISS.amax_marginal_relevance_searchrc s|durtdt||j}|rtd|dd|jDfdd|D|jtj tj d|j |fd d t |jD}d dt|D|_d S) zDelete by ID. These are the IDs in the vectorstore. Args: ids: List of ids to delete. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. NzNo ids provided to delete.zESome specified ids do not exist in the current store. Ids not found: cSsi|]\}}||qSr"r")rKidxrir"r"r#rprNz FAISS.delete..csh|]}|qSr"r"rKri)reversed_indexr"r# rzFAISS.delete..rmcsg|] \}}|vr|qSr"r"rKrri)index_to_deleter"r#rMs z FAISS.delete..cSrnr"r"rr"r"r#rprNT)r-rv differencer4valuesitemsr2Z remove_idsrwZfromiterZint64r3deletesortedr{)rBr_rZ missing_idsZ remaining_idsr")rrr#rs$    z FAISS.deletetargetr*cCst|jts tdt|j}|j|jg}|jD]\}}|j |}t|t s0td| ||||fq|j dd|Ddd|D}|j |dS)zMerge another FAISS object with the current one. Add the target FAISS to the current one. Args: target: FAISS object you wish to merge into the current one Returns: None. z'Cannot merge with this type of docstorezDocument should be returnedcSsi|]\}}}||qSr"r")rKrdrror"r"r#rprez$FAISS.merge_from..cSsi|]\}}}||qSr"r")rKr2rrdr"r"r#rpreN)r+r3rr-r,r4r2 merge_fromrrrrrzr|)rBrrsZ full_inforZ target_idror~r"r"r#rs    zFAISS.merge_fromrcKst} |tjkr| t|d} n | t|d} |dt} |di} ||| | | f||d|} | j||||d| S)Nrr3r4)r8r9r) r$rrZ IndexFlatIPr,Z IndexFlatL2poprr)clsrHrGrr]r_r8r9rr r2r3r4Zvecstorer"r"r#Z__froms&   z FAISS.__fromcKs&||}|j|||f||d|S)aOConstruct FAISS wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() faiss = FAISS.from_texts(texts, embeddings) r)rO _FAISS__fromrrHrr]r_rrGr"r"r# from_textss zFAISS.from_texts list[str]cs.||IdH}|j|||f||d|S)aeConstruct FAISS wrapper from raw documents asynchronously. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() faiss = await FAISS.afrom_texts(texts, embeddings) Nr)rTrrr"r"r# afrom_textsszFAISS.afrom_textscKs0t|\}}|jt|t||f||d|S)aConstruct FAISS wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() text_embeddings = embeddings.embed_documents(texts) text_embedding_pairs = zip(texts, text_embeddings) faiss = FAISS.from_embeddings(text_embedding_pairs, embeddings) r)rurr)rrrr]r_rrHrGr"r"r#from_embeddingsBs zFAISS.from_embeddingscs|j||f||d|S)z:Construct FAISS wrapper from raw documents asynchronously.r)r)rrrr]r_rr"r"r#afrom_embeddingsis zFAISS.afrom_embeddings folder_path index_namecCst|}|jdddt}||jt||dt||dd}t|j |j f|WddS1s.)rr-rrBrrrrrrZdocs_and_rel_scoresr"rr#(_similarity_search_with_relevance_scoress$  z.FAISS._similarity_search_with_relevance_scorescsP|dur td|j|f|||d|IdH}fdd|D}|S)rNrrcrr"r"rrr"r#rM6rzCFAISS._asimilarity_search_with_relevance_scores..)rr-rrr"rr#)_asimilarity_search_with_relevance_scoress&  z/FAISS._asimilarity_search_with_relevance_scores Callable[[Dict[str, Any]], bool]c st|r|St|tstdt|ddlm}m}m}m }m }m }||||||d}ddddd}||Bt t gd } d |D]} | r\| d r\| | vr\td | qHdfdd dfdd |S)a Create a filter function based on the provided filter. Args: filter: A callable or a dictionary representing the filter conditions for documents. Returns: A function that takes Document's metadata and returns True if it satisfies the filter conditions, otherwise False. Raises: ValueError: If the filter is invalid or contains unsuported operators. z5filter must be a dict of metadata or a callable, not r)eqrgtrltne)z$eqz$neqz$gtz$ltz$gtez$ltecSs||vSrEr"abr"r"r#`z+FAISS._create_filter_func..cSs||vSrEr"r r"r"r#r ar )z$inz$nin)$and$or$not $&filter contains unsupported operator: fieldr( condition%Union[Dict[str, Any], List[Any], Any]rrcsttr.gD]\}}|vrtd|||fq d fdd }|SttrKtkrDtfdd Sfd d Sfd d S)a Creates a filter function based on field and condition. Args: field: The document field to filter on condition: Filter condition (dict for operators, list for in, or direct value for equality) Returns: A filter function that takes a document and returns boolean rroDict[str, Any]rrcs |tfddDS)aW Evaluates a document against a set of predefined operators and their values. This function applies multiple comparison/sequence operators to a specific field value from the document. All conditions must be satisfied for the function to return True. Args: doc (Dict[str, Any]): The document to evaluate, containing key-value pairs where keys are field names and values are the field values. The document must contain the field being filtered. Returns: bool: True if the document's field value satisfies all operator conditions, False otherwise. c3s|] \}}||VqdSrEr")rKopvalueZ doc_valuer"r#rfszYFAISS._create_filter_func..filter_func_cond..filter_fn..)rallro)r operatorsrr# filter_fns zFFAISS._create_filter_func..filter_func_cond..filter_fnc|vSrErr) condition_setrr"r#r zEFAISS._create_filter_func..filter_func_cond..crrEr rrrr"r#r r"cs|kSrEr rr#r"r#r r"N)rorrr)r+dictrr-rrr, frozenset)rrrrr) OPERATIONSSET_CONVERT_THRESHOLD)rr!rrr#filter_func_condls   z3FAISS._create_filter_func..filter_func_condrrcsd|vrfdd|dDfddSd|vr*fdd|dDfddSd |vr:|d fd dSfd d|Dfd dS) a Creates a filter function that evaluates documents against specified filter conditions. This function processes a dictionary of filter conditions and returns a callable that can evaluate documents against these conditions. It supports logical operators ($and, $or, $not) and field-level filtering. Args: filter (Dict[str, Any]): A dictionary containing filter conditions. Can include: - Logical operators ($and, $or, $not) with lists of sub-filters - Field-level conditions with comparison or sequence operators - Direct field-value mappings for equality comparison Returns: Callable[[Dict[str, Any]], bool]: A function that takes a document (as a dictionary) and returns True if the document matches all filter conditions, False otherwise. rcg|]}|qSr"r"rKZ sub_filterrr"r#rMrzBFAISS._create_filter_func..filter_func..ctfddDS)Nc3|]}|VqdSrEr"rKrrr"r#rfSFAISS._create_filter_func..filter_func....rrfiltersrr#r rNz@FAISS._create_filter_func..filter_func..rcr)r"r"r*r+r"r#rMrcr,)Nc3r-rEr"r.rr"r#rfr/r0)anyrr2rr#r rNrcs | SrEr"r)condr"r#r s csg|] \}}||qSr"r")rKrr)r(r"r#rMscr,)Nc3r-rEr")rKrrr"r#rfr/r0r1r) conditionsrr#r rN)r)r)rr()r5r6r3r#rs      z.FAISS._create_filter_func..filter_funcN)rr(rrrr)rrrr)callabler+r$r-typerrrrrrrr%r startswith) rrrrrrrZCOMPARISON_OPERATORSZSEQUENCE_OPERATORSZVALID_OPERATORSrr")r&r'rr(r#r;s6    4'zFAISS._create_filter_func Sequence[str]list[Document]cs fdd|D}dd|DS)Ncsg|]}j|qSr")r3rrrFr"r#rMrez$FAISS.get_by_ids..cSsg|] }t|tr|qSr")r+r)rKror"r"r#rMrtr")rBr_rr"rFr# get_by_idsszFAISS.get_by_ids)r0r1r2rr3rr4r5r6r7r8rr9r)rrD)rHrIrrJ)rLr(rrW)NN) rHr[rGr\r]r^r_r`rrI) rHr[r]rr_r`rrrrI) rrr]rr_r`rrrrI)rNr) rrWrrrrrrrrrr) rr(rrrrrrrrrr) rrWrrrrrrrrrr) rrWrrrrrrrrrr) rr(rrrrrrrrrr) rrWrrrrrrrrrr)rrrN)rrWrrrrrrrrrrrr)rr(rrrrrrrrrrrrrE)r_r`rrrr)rr/rr*)rHr[rGrJrrr]r^r_r`r8rr9rrrrr/) rHrIrrr]rr_r`rrrr/) rHrrrr]rr_r`rrrr/) rrrrr]r^r_r`rrrr/)r2)rr(rr(rr*) rr(rGrrr(rrrrrr/)rr) rrrGrrrrrrr/)rr)rrrr)r_r:rr;)1__name__ __module__ __qualname____doc__rr?rCpropertyrGrQrVrYrZrrrrrrrrrrrrrrrrrrrr classmethodrrrrrrrrrrrr staticmethodrr<r"r"r"r#r/Ks   #     -    D * % % # #  H* " $ ( % " $$&  3     r/rE)rrrr) r%rr&rr'r(r)r(rr*)- __future__rloggingrrrrar@pathlibrtypingrrrrrr r r r r numpyrwZlangchain_core.documentsrZlangchain_core.embeddingsrZlangchain_core.runnables.configrZlangchain_core.vectorstoresrZ!langchain_community.docstore.baserrZ&langchain_community.docstore.in_memoryrZ&langchain_community.vectorstores.utilsrr getLoggerr=r;r$r.r/r"r"r"r#s*  0