o Zh@sddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZe r`dd lmZdd lmZ Gd d d eZ!dS) ) annotationsN)contextmanager)StringIO) TYPE_CHECKINGAnyDict GeneratorIterableListOptionalTupleType) Embeddings) VectorStore)Document) connection)cursorc@seZdZdZGdddeejZGdddZdddddqddZ GdddZ drddZ drddZ  dsdtd!d"Z dudvd#d$Zdwd%d&Z dudxd.d/Zdyd3d4Ze  5 6 7 dzd{d:d;Z  dsd|d@dAZ 7d}d~dCdDZddHdIZ JdddNdOZ JdddRdSZ JdddTdUZ JdddVdWZ duddZd[Zdd^d_ZddadbZdrdcddZdrdedfZddidjZ ddkdlZ!ddmdnZ"ddodpZ#dS) YellowbrickzYellowbrick as a vector database. Example: .. code-block:: python from langchain_community.vectorstores import Yellowbrick from langchain_community.embeddings.openai import OpenAIEmbeddings ... c@seZdZdZdZdZdS)zYellowbrick.IndexTypez Helper function: create schema if not exists rsqlzE CREATE SCHEMA IF NOT EXISTS {s} )sN)r:rvrMexecuteSQLformat Identifier)r$rrvrrrrSs  zYellowbrick._create_schemac Csddlm}|jr |jfnd}|jg||j|jR}||j|jd}||dj||d|jr<|jfnd}|jg||jR}|jg||j|jR}||j|jd}||j|jd} ||d j|||| d d S) z= Helper function: create table if not exists rrur _pk_doc_ida0 CREATE TABLE IF NOT EXISTS {t} ( doc_id UUID NOT NULL, text VARCHAR(60000) NOT NULL, metadata VARCHAR(1024) NOT NULL, CONSTRAINT {c} PRIMARY KEY (doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) tcZ_pk_doc_id_embedding_id _fk_doc_ida CREATE TABLE IF NOT EXISTS {t1} ( doc_id UUID NOT NULL, embedding_id SMALLINT NOT NULL, embedding FLOAT NOT NULL, CONSTRAINT {c1} PRIMARY KEY (doc_id, embedding_id), CONSTRAINT {c2} FOREIGN KEY (doc_id) REFERENCES {t2}(doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) t1t2c1c2N) r:rvrMr{rNrHrxryrz) r$rrv schema_prefixr~rrrrrrrrrTs>   zYellowbrick._create_tableOptional['PgCursor']cCs\|dur$|j}|j|||dWddS1swYdS|j|||ddS)z Helper function: Drop data. If a cursor is provided, use it; otherwise, obtain a new cursor for the operation. N)r.)rrQ _drop_table)r$r3r.rrrrr0s  "zYellowbrick.dropcCsFddlm}|r|||}n||}|d|}||dS)zI Executes the drop table command using the given cursor. rruz1 DROP TABLE IF EXISTS {} CASCADE N)r:rvr{ryrzrx)r$rr3r.rv table_nameZdrop_table_queryrrrrs  zYellowbrick._drop_tablecCsh|j}d}|||d}Wdn1swY|dks.|dkr0dStd)zE Helper function: Test the database is UTF-8 encoded z SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database(); rNutf8zutf-8TzDatabase encoding is not UTF-8)rrQrxfetchonelower Exception)r$rqueryencodingrrrrPs   z Yellowbrick._check_database_utf8texts Iterable[str] metadatasOptional[List[dict]]kwargsr List[str]c Ksd}t|}|jt|}g}|sdd|D}|dp!t}|j}t} t} t j | ddt j d} t j | ddt j d} d} t |D]T\}}t t}||| ||t||gt ||D] \}}| |||gqj| d 7} | |kr||| | | d| d| d| dd} qH| dkr||| | Wdn1swY|jtjjkr||t||S) Ni'cSsg|]}iqSrr).0_rrr ?sz)Yellowbrick.add_texts.. index_params ") delimiter quotecharquotingr)listrOZembed_documentsr*rr-rrQrcsvwriter QUOTE_MINIMAL enumerater'uuiduuid4appendwriterowjsondumps _copy_to_dbseektruncaterrr _update_indexUUID)r$rrrZ batch_size embeddingsresultsrr content_io embeddings_ioZcontent_writerZembeddings_writerZcurrent_batch_sizeitextZdoc_uuid embedding_idr1rrr add_texts3sN         "zYellowbrick.add_textsrrrc Cs|d|dddlm}|jr|jfnd}|jg||j|jR}|dj|d}| |||jr=|jfnd}|jg||jR}|dj|d}| ||dS)Nrrurz COPY {table} (doc_id, text, metadata) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\t', QUOTE '"') )r3z COPY {table} (doc_id, embedding_id, embedding) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\t', QUOTE '"') ) rr:rvrMr{rNrHryrzZ copy_expert) r$rrrrvrr3Zcontent_copy_queryZembeddings_copy_queryrrrrjs&    zYellowbrick._copy_to_db langchainpublicr_Type[Yellowbrick]c Ks,||||||d} | jd||d|| S)aAdd texts to the vectorstore index. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. connection_string: URI to Yellowbrick instance embedding: Embedding function table: table to store embeddings kwargs: vectorstore specific parameters )r1r2r3r.r0)rrNr)r) r_rr1rr2r3r.r0rZvssrrr from_textsszYellowbrick.from_textsidsOptional[List[str]] delete_allOptional[bool]c  s|ddlm|rd}n%|dur/tfdd|D}d|}dj|d }ntd |jr:|jfnd }|j s}j g||j |j R} d j| |d } | | j g||j R} d j| |d } | | |j||j |jg|Rrj g||j |jR} d j| |d } | | WddSWddS1swYdS)zxDelete vectors by uuids. Args: ids: List of ids to delete, where each id is a uuid string. rruz' WHERE 1=1 Nc3s|]}|VqdSr#)Literal)ridrurr sz%Yellowbrick.delete..z, z5 WHERE doc_id IN ({ids}) )rz*Either ids or delete_all must be provided.rzDELETE FROM {table} {where_sql})r3 where_sql)r:rvrytuplejoinrz ValueErrorrMrrQr{rNrHrx _table_existsrF) r$rrrrZuuidsZ ids_formattedrrZtable_identifierrrrurdeletesh             zYellowbrick.deletercCsJddlm}||}||}||dj||d|ddkS)z> Checks if a table exists in the given schema rruz SELECT COUNT(*) FROM sys.table t INNER JOIN sys.schema s ON t.schema_id = s.schema_id WHERE s.name = {schema} AND t.name = {table_name} )r.r)r:rvrrxryrzr)r$rrr.rvrrrrs    zYellowbrick._table_existsvector List[float] uuid.UUIDcCsFddl}dtt|}||}|}tj|ddd}|S)Nr,)bytes) hashlibrmapr'sha1encodedigestrr)r$rrZ vector_strZ hash_objectZ hash_digestZ vector_uuidrrr_generate_vector_uuids z!Yellowbrick._generate_vector_uuidkintList[Tuple[Document, float]]c s ddlm}ddlm}|dpt}|j}d|j }| || d | |} || fddt|D} | d  | |} ||| | | |} |jr]|jfnd } |j g| |j R}|j g| |j |jR}|jtjjkr|j d }|||||jr|jfnd } |j g| |j |jR}| |}| d j | ||||||d dd}|||f|}n| dj | ||d}|||f|}Wdn1swYg}|D]}t|dpi}t|d|d}|||dfq|S)aPerform a similarity search with Yellowbrick with vector Args: embedding (List[float]): query embedding k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document, float]: List of Documents and scores rru)execute_valuesrZtmp_z CREATE TEMPORARY TABLE {} ( doc_id UUID, embedding_id SMALLINT, embedding FLOAT) ON COMMIT DROP DISTRIBUTE REPLICATE csg|] \}}t||fqSr)r')rrZembedding_valueZ tmp_doc_idrrr%s zFYellowbrick.similarity_search_with_score_by_vector..z:INSERT INTO {} (doc_id, embedding_id, embedding) VALUES %srZ _tmp_hasha/ WITH index_docs AS ( SELECT t1.doc_id, SUM(ABS(t1.hash-t2.hash)) as hamming_distance FROM {lsh_index} t1 INNER JOIN {input_hash_table} t2 ON t1.hash_index = t2.hash_index GROUP BY t1.doc_id HAVING hamming_distance <= {hamming_distance} ) SELECT text, metadata, SUM(v1.embedding * v2.embedding) / (SQRT(SUM(v1.embedding * v1.embedding)) * SQRT(SUM(v2.embedding * v2.embedding))) AS score FROM {v1} v1 INNER JOIN {v2} v2 ON v1.embedding_id = v2.embedding_id INNER JOIN {v3} v3 ON v2.doc_id = v3.doc_id INNER JOIN index_docs v4 ON v2.doc_id = v4.doc_id GROUP BY v3.doc_id, v3.text, v3.metadata ORDER BY score DESC LIMIT %s hamming_distance)v1v2v3 lsh_indexinput_hash_tablera SELECT text, metadata, score FROM (SELECT v2.doc_id doc_id, SUM(v1.embedding * v2.embedding) / (SQRT(SUM(v1.embedding * v1.embedding)) * SQRT(SUM(v2.embedding * v2.embedding))) AS score FROM {v1} v1 INNER JOIN {v2} v2 ON v1.embedding_id = v2.embedding_id GROUP BY v2.doc_id ORDER BY score DESC LIMIT %s ) v4 INNER JOIN {v3} v3 ON v4.doc_id = v3.doc_id ORDER BY score DESC )rrrNr)Z page_contentmetadata)r:rvZpsycopg2.extrasrr*rr-rrQrNrryrzr{rxrrMrHrrr_generate_tmp_lsh_hashesrFrr+Zfetchallrloadsrr)r$r1rrrvrrrZtmp_embeddings_tableZcreate_table_queryZ data_input insert_queryrrrrtmp_hash_tablerrZ sql_queryr documentsresultrdocrrr&similarity_search_with_score_by_vectors             # - |z2Yellowbrick.similarity_search_with_score_by_vectorrList[Document]cKs0|j|}|jd||d|}dd|DS)aePerform a similarity search with Yellowbrick Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of Documents r1rcSg|]\}}|qSrrrrrrrrrz1Yellowbrick.similarity_search..NrrOZ embed_queryrr$rrrr1rrrrsimilarity_searchs zYellowbrick.similarity_searchcKs&|j|}|jd||d|}|S)arPerform a similarity search with Yellowbrick Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of (Document, similarity) rNrrrrrrsimilarity_search_with_scores z(Yellowbrick.similarity_search_with_scorecKs$|jd||d|}dd|DS)aPerform a similarity search with Yellowbrick by vectors Args: embedding (List[float]): query embedding k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of documents rcSrrrrrrrrrz;Yellowbrick.similarity_search_by_vector..Nr)r)r$r1rrrrrrsimilarity_search_by_vectors z'Yellowbrick.similarity_search_by_vectordoc_idOptional[uuid.UUID]c Csddlm}|jr |jfnd}|jg||j|jR}|jg||j|jR}|jg||jR}|d|}|rN|dj| t |dn|d} |d} |d j|||| | d } | | d S) zAdd hashes to LSH indexrrurzINSERT INTO {}zWHERE e.doc_id = {doc_id})rrz GROUP BY 1, 2av {query_prefix} SELECT e.doc_id as doc_id, h.id as hash_index, CASE WHEN SUM(e.embedding * h.hyperplane) > 0 THEN 1 ELSE 0 END as hash FROM {embedding_table} e INNER JOIN {hyperplanes} h ON e.embedding_id = h.hyperplane_id {condition} {group_by} ) query_prefixembedding_table hyperplanes conditiongroup_byN) r:rvrMr{rNrGrFryrzrr'rx) r$rrrvrlsh_hyperplane_tableZlsh_index_table_idZembedding_table_idZquery_prefix_idrr input_queryrrr_update_lsh_hashess<     zYellowbrick._update_lsh_hashestmp_embedding_tablerc Csddlm}|jr |jfnd}|jg||j|jR}||}||}|d|} |d} |dj| ||| d} || dS) zGenerate temp LSHrrurz+CREATE TEMPORARY TABLE {} ON COMMIT DROP ASz GROUP BY 1a[ {query_prefix} SELECT h.id as hash_index, CASE WHEN SUM(e.embedding * h.hyperplane) > 0 THEN 1 ELSE 0 END as hash FROM {embedding_table} e INNER JOIN {hyperplanes} h ON e.embedding_id = h.hyperplane_id {group_by} DISTRIBUTE REPLICATE )rrrrN) r:rvrMr{rNrGryrzrx) r$rrrrvrrZtmp_embedding_table_idZtmp_hash_table_idrrrrrrrs.       z$Yellowbrick._generate_tmp_lsh_hashesnum_hyperplanesc Csddlm}|jr |jfnd}|jg||j|jR}||dj|d| ddkr3dS|jg||jR}||dj|d| d}|d7}|d j| || ||d }||dS) z4Generate random hyperplanes and store in YellowbrickrrurzSELECT COUNT(*) FROM {t})r~Nz!SELECT MAX(embedding_id) FROM {t}ra2 WITH parameters AS ( SELECT {num_hyperplanes} AS num_hyperplanes, {dims_per_hyperplane} AS dims_per_hyperplane ) INSERT INTO {hyperplanes_table} (id, hyperplane_id, hyperplane) SELECT id, hyperplane_id, (random() * 2 - 1) AS hyperplane FROM (SELECT range-1 id FROM sys.rowgenerator WHERE range BETWEEN 1 AND (SELECT num_hyperplanes FROM parameters) AND worker_lid = 0 AND thread_id = 0) a, (SELECT range-1 hyperplane_id FROM sys.rowgenerator WHERE range BETWEEN 1 AND (SELECT dims_per_hyperplane FROM parameters) AND worker_lid = 0 AND thread_id = 0) b )rZdims_per_hyperplanehyperplanes_table) r:rvrMr{rNrGrxryrzrr) r$rrrvrrr~Znum_dimensionsrrrr_populate_hyperplanes*s.   z!Yellowbrick._populate_hyperplanesc Csddlm}|jr |jfnd}|jg||j|jR}|jg||j|jR}||j|jd}||j|jd}||dj ||||d|jrW|jfnd}|jg||j|j R}||j|j d} ||d j || d d S) z&Create LSH index and hyperplane tablesrrurr|ra CREATE TABLE IF NOT EXISTS {t1} ( doc_id UUID NOT NULL, hash_index SMALLINT NOT NULL, hash SMALLINT NOT NULL, CONSTRAINT {c1} PRIMARY KEY (doc_id, hash_index), CONSTRAINT {c2} FOREIGN KEY (doc_id) REFERENCES {t2}(doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) rZ _pk_id_hp_ida2 CREATE TABLE IF NOT EXISTS {t} ( id SMALLINT NOT NULL, hyperplane_id SMALLINT NOT NULL, hyperplane FLOAT NOT NULL, CONSTRAINT {c} PRIMARY KEY (id, hyperplane_id)) DISTRIBUTE REPLICATE SORT ON (id) r}N) r:rvrMr{rNrFrHrxryrzrG) r$rrvrrrrrr~rrrr_create_lsh_index_tablesTs:   z$Yellowbrick._create_lsh_index_tablescCs8|j|j|j|j|d|j|j|j|j|ddS)zDrop LSH index tables)r.r3rN)r0rMrNrFrG)r$rrrrrRs  z"Yellowbrick._drop_lsh_index_tablesrrcCst|jtjjkr8|j"}||||||| dd| |WddS1s1wYdSdS)z"Create index from existing vectorsrN) rrrrrrQrRrrr+rr$rrrrr create_indexs     "zYellowbrick.create_indexcCsL|jtjjkr$|j}||WddS1swYdSdS)z Drop an indexN)rrrrrrQrRrrrr drop_indexs   "zYellowbrick.drop_indexcCsN|jtjjkr%|j}|||WddS1swYdSdS)zHUpdate an index with a new or modified embedding in the embeddings tableN)rrrrrrQr)r$rrrrrrrs  "zYellowbrick._update_indexc Cs,ddlm}z{|jk}|jr|jfnd}|jg||jR}|jg||jdR}|jg||j|jR}|dj ||d}| || ||dj ||d}| ||dj ||d} | | WdWdS1szwYWdSt y} zt d | | d} ~ ww) NrrurZ_v1zALTER TABLE {t1} RENAME TO {t2})rrz INSERT INTO {t1} (doc_id, embedding_id, embedding) SELECT id, embedding_id, embedding FROM {t2} z INSERT INTO {t1} (doc_id, text, metadata) SELECT DISTINCT id, text, metadata FROM {t2} zFailed to migrate schema: )r:rvrrQrMr{rNrHryrzrxrTrrk) r$rvrrrZold_embeddingscontentZalter_table_queryrZinsert_content_queryrnrrrmigrate_schema_v1_to_v2sH        &!z#Yellowbrick.migrate_schema_v1_to_v2)r1rr2r'r3r'r.r4r/r5r0r6r)r7)rrtr)r7r,)r3r'r.r4rrr)r7r#)rrtr3r'r.r4r)r7)r)r6)rrrrrrr)r)rrtrrrrr)r7)NrrrF)r_rrrr1rrrr2r'r3r'r.r'r0r6rrr)r)rrrrrrr)r7)r)rrtrr'r.r'r)r6)rrr)r)r)r1rrrrrr)r)rr'rrrrr)r)rr'rrrrr)r)r1rrrrrr)r)rrtrrr)r7)rrtrr'rr'r)r7)rrtrrr)r7)rrr)r7)rrrrr)r7rq)$rrrrr'enumEnumrr-r%rIrSrTr0rrPrr classmethodrrrrrrrrrrrrrRrrrr rrrrr!sh> 6 6   7  >      - $ * / r)" __future__rrJrr rr;r contextlibriortypingrrrrr r r r r Zlangchain_core.embeddingsrZlangchain_core.vectorstoresrZ%langchain_community.docstore.documentrZpsycopg2.extensionsrZ PgConnectionrZPgCursorrrrrrs"   ,