o $if,@s(ddlZddlmZddlmZddlZddlmZddlm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZmZiaddZd d Zejd ejd ejd ejdejdejdejdejdej dej!dej"dddej#dej$diZ%ddZ&ddZ'ddZ(d d!Z)d"d#Z*d$d%Z+d&d'Z,d(d)Z-d*d+Z.d,d-Z/d.d/Z0d0d1Z1d2d3Z2d4d5Z3d^d6d7Z4  9d_d:d;Z5dd?Z7d@dAZ8 dadCdDZ9hdEZ:d^dFdGZ;dHdIZ d^dNdOZ?dPdQZ@dRdSZAddTdTej#ej$ej#ejej"ejBdU ZCdVdWZDdXdYZEdZd[ZFd\d]ZGdS)bN)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesc Cstsjtitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA Type_BOOL Type_INT8 Type_INT16 Type_INT32 Type_INT64 Type_UINT8 Type_UINT16 Type_UINT32 Type_UINT64Type_HALF_FLOAT Type_FLOAT Type_DOUBLE Type_DATE32 Type_DATE64 Type_TIME32 Type_TIME64 Type_BINARYType_FIXED_SIZE_BINARY Type_STRINGr1r1N/var/www/html/corbot_env/lib/python3.10/site-packages/pyarrow/pandas_compat.pyget_logical_type_map)sR     r3cCst}z||jWStyLt|tjjrYdSt|tjjr+dt |j YSt|tjj r?|j dur;dYSdYSt|tjj rIYdSYdSw)N categoricalzlist[{}] datetimetzdatetimedecimalobject)r3idKeyError isinstancerrDictionaryTypeListTypeformatget_logical_type value_type TimestampTypetzDecimal128Type) arrow_typelogical_type_mapr1r1r2r?Fs  r?r r r r r rrrrrrz datetime64[D]rstringrcCsdzt|jjWSty1t|jdrYdSt|jdkr!YdSt|}|dkr-YdS|YSw)NrBr5datetime64[ns]rFr)_numpy_logical_type_mapdtypetyper:hasattrstrr infer_dtype)pandas_collectionresultr1r1r2get_logical_type_from_numpyis   rPcCs|j}t|dkr(t|d|}|dusJt|j|jd}t|jj}||fSt|drrJ__name__)r[rdrDre logical_type string_dtypeextra_metadatar1r1r2get_column_metadatas,   rmc Cstdd|D}t|}|d||} |||d} g} t||| D]\} } }t| | || d}| |q%g}|durg}t|| |D],\}}}t|trQqF|jdurbt|jtsb||jt|t|j||d}||qFt|dkrt j d|dt d d g}t |j d |j g}t |j d |j jg}t||D]\}}t||}||qng}}}d t||| |dtjdtjddiS)aReturns a dictionary containing enough metadata to reconstruct a pandas DataFrame as an Arrow Table, including index columns. Parameters ---------- columns_to_convert : list[pd.Series] df : pandas.DataFrame index_levels : List[pd.Index] index_descriptors : List[Dict] preserve_index : bool types : List[pyarrow.DataType] Returns ------- dict cSsg|] }t|ts|qSr1)r;dict).0descrr1r1r2  z&construct_metadata..N)rdrDreFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesspandaspyarrow)libraryversion) index_columnscolumn_indexescolumnscreatorpandas_versionutf8)rWziprmappendr;rnrdrL_column_name_to_stringswarningswarn UserWarningrVr}_get_simple_index_descriptorjsondumpsr __version__rrzencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indextypesnum_serialized_index_levelsntypesdf_types index_typescolumn_metadatacolsanitized_namerDr]index_column_metadatanon_str_index_nameslevel descriptorr|rvrwrdr1r1r2construct_metadatasl          rcCsTt|\}}t|}d|vrtjdtdd|dkr"|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rsrtrencodingUTF-8rc)r_rPrrr)rrdrkrlrfr1r1r2rs rcCsdt|tr|St|tr|dSt|trtttt|St|tr(td|dur.dSt|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" rz%Unsupported type for MultiIndex levelN) r;rLrdecodetuplemaprrrhrdr1r1r2r"s     rcCs(|jdur|j|vrt|jSd|S)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str Nz__index_level_{:d}__)rdrr>)indexirr1r1r2_index_level_nameIs  rcCs:t|||}|jjstdt|j|durt|||Sg}|dur)t|jng}g}g}|D]%}||} t |}t | rGt d|| | | d| |q1g} g} t|D]0\} } t| | |}t| t jjry|duryt| }n| | | d|}| || |q_|| }||| | |||fS)Nz Duplicate column names found: {}F-Sparse pandas data (column {}) not supported.)_resolve_columns_of_interestr} is_unique ValueErrorr>list$_get_columns_to_convert_given_schema_get_index_level_valuesrrr is_sparserhr enumeraterr;pd RangeIndex_get_range_index_descriptor)rschemarr}rrrconvert_fieldsrdrrindex_column_namesr index_levelrp all_namesr1r1r2_get_columns_to_convert\sN             rc Cs:g}g}g}g}g}g}|jD]} z|| } d} Wn@tyYzt|| } Wnttfy7td| w|durCtd| |durUt| tjj rUtd| d} Ynwt | } t | rjt d| | | } || || || | r|| || || q||} | ||||||fS)z Specialized version of _get_columns_to_convert in case a Schema is specified. In that case, the Schema is used as the single point of truth for the table structure (types, which columns are included, order of columns, ...). FzNname '{}' present in the specified schema is not found in the columns or indexzlname '{}' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifiedNaname '{}' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTr)rwr:_get_index_level IndexErrorr>rr;rrrrrrhfieldr)rrrrrrrrrrdris_indexrrr1r1r2rsf           rcCs8|}||jjvrt|rt|tdd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). __index_level_)rrw_is_generated_index_nameintrWget_level_values)rrdkeyr1r1r2rs rcCs,zt||WStyt|YSwN)rrrhrLrr1r1r2 _level_names    rcCs.dt|jt|dt|dt|ddS)Nrangestartstopstep)kindrdrrr)rrdrget_rangeindex_attribute)rr1r1r2rs    rcs(ttdg}fddt|DS)Nrvcsg|]}|qSr1)r)rorrr1r2rqsz+_get_index_level_values..)rWrVr)rnr1rr2rsrcsR|dur |dur td|dur|j}|S|dur$fdd|D}|Sj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themcsg|] }|jvr|qSr1)r}rocrr1r2rq sz0_resolve_columns_of_interest..)rrwr})rrr}r1rr2rsrcCst|d||\}}}}}}}g} |D]T} | j} t| r%tj| ddj} nField {} was non-nullable but pandas column had {} null values)nullablerJrr ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsr>rdrI null_countrrL)rrfield_nullablerrOe)rr1r2convert_columnGs0  z+dataframe_to_arrays..convert_columncSs$t|tjo|jjot|jjtjSr) r;npndarrayflags contiguous issubclassrIrJinteger)arrr1r1r2_can_definitely_zero_copy]s z6dataframe_to_arrays.._can_definitely_zero_copycsg|] \}}||qSr1r1)rorf)rr1r2rqcsz'dataframe_to_arrays..cSsg|]}|jqSr1rJ)roxr1r1r2rqrsrbrrrrrr)rrWr}r cpu_countrrThreadPoolExecutorrrsubmitrr;FuturerOrrrr]rrnr with_metadatarr)!rrrnthreadsr}rrrrrr{rrnrowsncolsrarraysexecutorrrr maybe_futrfieldsrdrpandas_metadatar]n_rowsrrrrr1)rrr2dataframe_to_arrays2sz                rcCsd|jjtjkr ||fSt|r$|dur$|j}|j}t ||}||fS|dur.t |j}||fSr) rIrJr datetime64r is_datetimetzrBunitr timestampfrom_numpy_dtype)rrIrrBrr1r1r2rs  rcCs$ddlmm}|dd}|d}d|vr%tjj||d|dd}n`d|vrZt|j \}} t ||d} t rItj j |d | d d }n<|}|rY|j|||j| d } | Sn+d |vr|d }t|dksjJ||d} || } t| ds}td| |}n|}|r|j||dS||fS)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryrT)rXrTrUr F)rIcopy)r klassrIpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r )pandas.core.internalscore internalsgetrcategorical_type from_codesr datetime_datarImake_datetimetz is_ge_v21rrview make_blockDatetimeTZBlockrWrKrr)itemr}extension_columns return_block_int block_arrr rrrrIr rd pandas_dtyper1r1r2_reconstruct_blocksF     r"cCs&trd}tj|}tj||dS)NnsrB)ris_v1rrstring_to_tzinfodatetimetz_type)rrBr1r1r2rs rFcsNg}g}|jj}|s1|dur1|d}|dg}|d}t||}t||||\}} t|||n tj|j } t|g|t |t |||} |j t j|||t} truddlm} fdd| D} | | | | d}|Sdd lm}dd lm}fd d| D} | | g}|| |}tr|||j}|S||}|S) Nr}r|r{r)create_dataframe_from_blockscsg|] }t|ddqS)F)rr"rorrext_columns_dtypesr1r2rqs z&table_to_dataframe..)rr}) BlockManager) DataFramecsg|]}t|qSr1r)r*r+r1r2rqs )rrr_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_indexrrrtable_to_blocksrkeysis_ge_v3pandas.api.internalsr(rr-pandasr.r _from_mgraxes)optionstablerXignore_metadata types_mapper all_columnsr|rrrr}rOr(blocksrr-r.r;mgrr1r+r2table_to_dataframesR             rC> r r r r r rr8rrrrrrc Csi}tjdur |S|D]2}z|d}Wn ty |d}Ynw|d}|tvr=t|}t|tjr=t|dr=|||<q |jD] }|j} t| t j raz| }Wn t y[YqAw|||j <qA|rz|jD]}|j} || }|dury|||j <qg|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. Nrerdrgr)rextension_dtyper:_pandas_supported_numpy_typesr!r;rKrrJrBaseExtensionTypeto_pandas_dtypeNotImplementedErrorrd) r=columns_metadatar? ext_columnscol_metardrIr!rtypr1r1r2r12sD              r1cCstdd|Ds JdS)Ncss0|]}|ddurd|vp|dduVqdS)rdNrer1rr1r1r2 ps  z:_check_data_column_metadata_consistency..)all)r@r1r1r2r3ksr3cs|rdd|Dfdd|jD}n|j}t|dkr2tjjjtttj |dd|Dd}ntjj ||r>|dd ndd }t|dkrMt ||}|S) NcSs&i|]}|dt|d|dqSrerd)rrrr1r1r2 xsz-_deserialize_column_index..csg|]}||qSr1r)rordcolumns_name_dictr1r2rq|s z-_deserialize_column_index..rcSsg|]}|dqSrr1)ro col_indexr1r1r2rqsrwrrdr) rrWrr MultiIndex from_tuplesrrast literal_evalIndex"_reconstruct_columns_from_metadata) block_tabler@r|columns_valuesr}r1rRr2r4vs&     r4c Cs6dd|D}g}g}|}|D]O}t|tr't|||||\}} } | dur&qn-|ddkrK|d} tjj|d|d|d| d } t| t|krJqn td |d| | | | qtj} t|d krt| j j ||d } || fSt|d kr|d } t| | j s| j | |d d} || fS| |j } || fS)NcSsi|] }|d|d|qSrOrQrr1r1r2rPsz&_reconstruct_index..rrrdrrr)rrdzUnrecognized index kind: {}rrUrr)r;rL_extract_index_levelrrrrWrr>rrV from_arraysrZr2) r=rr@r?field_name_to_metadata index_arrays index_names result_tablerpr index_namerrr1r1r2r0sP            r0c Csl||d}t||}|j|}|dkr|ddfS||}|j|d} d| _||j|}|| |fS)Nrd)r?) _backwards_compatible_index_namerget_field_indexr[ to_pandasrd remove_column) r=rcrer`r? logical_namerdrrrr1r1r2r^s        r^cCs||kr t|r dS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namerjr1r1r2rfsrfcCsd}t||duS)Nz^__index_level_\d+__$)rematch)rdpatternr1r1r2rsrrG) rr6r5rrrFrfloatingrcCs:zt|WStyd|vrtjYSt|YSw)a Get the numpy dtype that corresponds to a pandas type. Parameters ---------- pandas_type : str The result of a call to pandas.lib.infer_dtype. Returns ------- dtype : np.dtype The dtype that corresponds to `pandas_type`. r)_pandas_logical_type_mapr:robject_rI)rfr1r1r2_pandas_type_to_numpy_type s   rrc Cs(tj}t|ddp |g}t|ddpdg}ddt||idD}g}tdd}|D]K\}} } t| } | tjkr>| |}| d krYt j |d d d } |j |d d| }n |j| krc|| }|j| krq| d krq|| }||q+t|dkr|j|||jdS|j|d |d j|jdS)a_Construct a pandas MultiIndex from `columns` and column index metadata in `column_indexes`. Parameters ---------- columns : List[pd.Index] The columns coming from a pyarrow.Table column_indexes : List[Dict[str, str]] The column index metadata deserialized from the JSON schema metadata in a :class:`~pyarrow.Table`. Returns ------- result : MultiIndex The index reconstructed using `column_indexes` metadata with levels of the correct type. Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` rvNrYcSs0g|]\}}||dt|j|ddfqS)rfrgN)rrLrI)rorrTr1r1r2rq=s  z6_reconstruct_columns_from_metadata..) fillvaluerrr5rr]rUT)utcrrU)rIrd)rrrVroperator methodcallerrrrbytes_rrrr& to_datetime tz_convertrIastyperrWrVrwrZrd) r}r|rrvlabels levels_dtypes new_levelsencoderrr! numpy_dtyperIrBr1r1r2r[ s6        r[cCsi}i}|j}|d}dd|D}t|}t|d|}t|dD]q\}} | d} | sC| d} ||kr=|||} | durCd} || } | dkr| d d kr|| } t| jtjj s_q$| d } | sfq$| d }|r|| jj kr| }tj d |d}tj j||d}t|| j||| <||| <q$t|dkrg}g}tt|jD]$}||vr||||||q|||||j|qtjj|t|dS|S)Nr{cSsg|] }t|tr|qSr1)r;rL)roidx_colr1r1r2rqjrrz%_add_any_metadata..r}rerdrbrerfr5r]rUr#r$rr)r)rrWrrrgr;rJrrrArBrhrArrayrrrdrrTabler_)r=rmodified_columnsmodified_fieldsrr{n_index_levels n_columnsrrKrkidxrr] metadata_tz converted tz_aware_typerr}rr1r1r2r/bs^       r/cCs$tj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz rt)rrr&dt tz_localizery)seriesrBr1r1r2 make_tz_awares  rr)rNT)NNT)NFN)HrXcollections.abcr concurrentrconcurrent.futures.threadr r itertoolsrrrurlrnumpyrrxr pyarrow.librrrr3r?bool_r r r r rrrrrrstr_rwrHrPr_rmrrrrrrrrrrrrrrr"rrCrEr1r3r4r0r^rfrrqrprrr[r/rr1r1r1r2s    -V'@@    " b E 7 9 !7  B A