o Zh<@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZmZmZmZmZddlmZddlmZddlmZe rXddlmZeeZ Gd d d e Z Gd d d Z Gd ddeZ!GdddeZ"dS)) annotationsN) HTMLParser) TYPE_CHECKINGAnyDictListOptionalTupleUnion) BaseLoader)Document) TextSplitter) Connectioncs@eZdZdZdfdd Zdd d Zdd dZdddZZS)ParseOracleDocMetadatazParse Oracle doc metadata...returnNonecs"t|d|_i|_dS)NF)super__init__resetmatchmetadataself __class__d/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/document_loaders/oracleai.pyr%s  zParseOracleDocMetadata.__init__tagstrattrsList[Tuple[str, Optional[str]]]cCsV|dkr d}|D]\}}|dkr|}|dkr|r||j|<qdS|dkr)d|_dSdS)NmetanamecontenttitleT)rr)rrrentryr#valuerrrhandle_starttag+s   z&ParseOracleDocMetadata.handle_starttagdatacCs|jr ||jd<d|_dSdS)Nr%F)rr)rr)rrr handle_data7s  z"ParseOracleDocMetadata.handle_dataDict[str, Any]cCs|jSN)rrrrr get_metadata<sz#ParseOracleDocMetadata.get_metadata)rr)rrrr rr)r)rrr)rr+) __name__ __module__ __qualname____doc__rr(r*r- __classcell__rrrrr"s   rc@s.eZdZdZeddddZedddZdS)OracleDocReaderz Read a fileN input_stringUnion[str, None]rrc Csd}d}|durdtjddd}tt}td|}t|  }|d|}tdt d}|||}| }| |}|d|}|S)N r"Z>abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789)kz>I)joinrandomchoicesinttimestructpackhashlibsha256encodedigest getrandbitshexzfill) r4Z out_lengthZhash_len timestampZ timestamp_binZ hashval_binZ counter_binZ object_idZ object_id_hexrrrgenerate_object_idFs*     z"OracleDocReader.generate_object_idconnr file_pathparamsdictUnion[Document, None]c Csi}zddl}Wnty}ztd|d}~wwzd|j_|}t|d }|}Wdn1s7wY|durGtd|dWS||j } ||j } |j d|t || | d | | durki}nt| } | d s{| d rt} | | | }t|jd |} | |d <||d<| durtd|dWStt| |dWSty}ztd|td|| WYd}~dSd}~ww)zRead a file using OracleReader Args: conn: Oracle Connection, file_path: Oracle Directory, params: ONNX file name. Returns: Plain text and metadata as Langchain Document. rNIUnable to import oracledb, please install with `pip install -U oracledb`.Frbr"Z page_contentra declare input blob; begin input := :blob; :mdata := dbms_vector_chain.utl_to_text(input, json(:pref)); :text := dbms_vector_chain.utl_to_text(input); end;)ZblobZprefmdatatext$_oid_fileAn exception occurred :: zSkip processing )oracledb ImportErrordefaults fetch_lobscursoropenreadr varZ DB_TYPE_CLOBexecutejsondumpsclosergetvalue startswithrfeedr-r3rIusername Exceptionloggerinfo)rJrKrLrrZer^fr)rRrSZdoc_datapdoc_idexrrr read_fileisf        zOracleDocReader.read_filer,)r4r5rr)rJrrKrrLrMrrN)r.r/r0r1 staticmethodrIrrrrrrr3Cs "r3cs,eZdZdZdfdd Zdd d ZZS)OracleDocLoaderzwRead documents using OracleDocLoader Args: conn: Oracle Connection, params: Loader parameters. rJrrLr+kwargsrc s.||_tt||_tjdi|dS)Nr)rJrcloadsrdrLrr)rrJrLrurrrrszOracleDocLoader.__init__rList[Document]c Cszddl}Wnty}ztd|d}~wwd}g}i}ddi}z|jdurM|jd|_|jd|_|jd|_|jd |_|jd |_nt d d |j _ |jrmt |j|j|}|durh|WS|||jrd}t|jD]/} tj|j| } tj| rt |j| |}|dur|d }td|dqx||qx|jr#zW|jdus|jdurt d|j} |jd|_|jdurt|jdkrt dd} | j| |j|jd| } | D]}|d|jvr|d dvrt dqd|_|jdur|jD] }|jd||_qd|jdt|d|jd|jd|jd|jd } | | | D]}i}|durtt |jj!d!|jd!|jd!|j}||d"<|t"d#|d$qF|ddurt#|d}|$d%s|$d&rt%}|&||'}t |jj!d!|jd!|jd!|jd!t#|d'}||d"<|d'|d(<|jdurt|j}t(d|D]}||d'||j|<q|d dur|t"d#|d$qF|t"t#|d |d$qFW|WSt y"}ztd)|t)*| +d}~ww|WSt y?}z td)|t)*d}~ww)*z,Load data into LangChain Document objects...rNrOZ plaintextfalsefiledirowner tablenamecolnamezMissing loader parametersFzTotal skipped:  z%Missing owner or column name or both. mdata_colsz?Exceeds the max number of columns you can request for metadata.zgselect column_name, data_type from all_tab_columns where owner = :ownername and table_name = :tablename)Z ownernamer|)NUMBERZ BINARY_DOUBLEZ BINARY_FLOATLONGZDATE TIMESTAMPZVARCHAR2zDThe datatype for the column requested for metadata is not supported.z, rowidz, z'select dbms_vector_chain.utl_to_text(t.z, json('z+')) mdata, dbms_vector_chain.utl_to_text(t.z) textz from .z trVrWr"rQrTrUZ_rowidrY),rZr[rLgetryrzr{r|r}rjr\r]r3rrrJappendoslistdirpathr:isfilerkrlr^rlenrbupperZfetchallZmdata_cols_sqlrcrdrIrir rrgrrhr-range traceback print_excre)rrZrmZncolsresultsrZm_paramsdocZ skip_count file_namerKr^sqlrowsrowcolrpr)roirqrrrloads\                         >zOracleDocLoader.load)rJrrLr+rur)rrw)r.r/r0r1rrr2rrrrrtsrtcs,eZdZdZdfd d ZdddZZS)OracleTextSplitterz$Splitting text using Oracle chunker.rJrrLr+rurrrc s||_||_tjdi|z%ddl}zddl}Wnty+}ztd|d}~ww||_||_WdSty?tdw)z Initialize.rNrOzjoracledb or json or both are not installed. Please install them. Recommendations: `pip install oracledb`. r) rJrLrrrcrZr[ _oracledb_json)rrJrLrurcrZrmrrrrs,   zOracleTextSplitter.__init__rSr List[str]c Cszddl}Wnty}ztd|d}~wwg}z;d|jj_|j}|j|jd|j d||j |j d | }|durF |WS|j |d}||d q:tyn}z td |td}~ww) z&Split incoming text and return chunks.rNrOF)r$zUselect t.column_value from dbms_vector_chain.utl_to_chunks(:content, json(:params)) t)r$rLTZ chunk_datarY)rZr[rr\r]rJr^Z setinputsizesZCLOBrbrrdrLZfetchonervrrjrkrlrr) rrSrZrmZsplitsr^rdrqrrr split_textsD     zOracleTextSplitter.split_text)rJrrLr+rurrr)rSrrr)r.r/r0r1rrr2rrrrr|sr)# __future__rrArcloggingrr;r?r>r html.parserrtypingrrrrrr r Zlangchain_core.document_loadersr Zlangchain_core.documentsr Zlangchain_text_splittersr rZr getLoggerr.rkrr3rtrrrrrs2  $     rE