o Zh6@s@ddlZddlmZddlmZddlZddlmZddlm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZmZiaddZd d Zejd ejd ejd ejdejdejdejdejdej dej!dej"dddej#dej$diZ%ddZ&ddZ'ddZ(d d!Z)d"d#Z*d$d%Z+d&d'Z,d(d)Z-d*d+Z.d,d-Z/d.d/Z0d0d1Z1d2d3Z2d4d5Z3ddd6d7Z4  9ded:d;Z5dd?Z7d@dAZ8 dgdCdDZ9hdEZ:dddFdGZ;dHdIZ dddNdOZ?dPdQZ@dRdSZAddTdTej#ej$ej#ejej"ejBdU ZCdVdWZDdXdYZEdZd[ZFd\d]ZGd^d_ZHd`daZIdbdcZJdS)hN)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesc Cstsjtitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibZType_NAZ Type_BOOLZ Type_INT8Z Type_INT16Z Type_INT32Z Type_INT64Z Type_UINT8Z Type_UINT16Z Type_UINT32Z Type_UINT64ZType_HALF_FLOATZ Type_FLOATZ Type_DOUBLEZ Type_DATE32Z Type_DATE64Z Type_TIME32Z Type_TIME64Z Type_BINARYZType_FIXED_SIZE_BINARYZ Type_STRINGrrL/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/pandas_compat.pyget_logical_type_map)sR     rcCst}z||jWStyLt|tjjrYdSt|tjjr+dt |j YSt|tjj r?|j dur;dYSdYSt|tjj rIYdSYdSw)NZ categoricalzlist[{}] datetimetzdatetimedecimalobject)ridKeyError isinstancerrZDictionaryTypeZListTypeformatget_logical_typeZ value_type TimestampTypetzZDecimal128Type) arrow_typeZlogical_type_maprrrr(Fs  r(r r r r r rrrrrrz datetime64[D]rstringrcCsdzt|jjWSty1t|jdrYdSt|jdkr!YdSt|}|dkr-YdS|YSw)Nr*r datetime64[ns]r,r)_numpy_logical_type_mapdtypetyper%hasattrstrrZ infer_dtype)Zpandas_collectionresultrrrget_logical_type_from_numpyis   r4cCs|j}t|dkr(t|d|}|dusJt|j|jd}t|jj}||fSt|dr) r(r?r@rAr&r2 TypeErrorr'r0__name__)r=rDr+rEZ logical_type string_dtypeextra_metadatarrrget_column_metadatas,   rLc Cstdd|D}t|}|d||} |||d} g} t||| D]\} } }t| | || d}| |q%g}|durg}t|| |D],\}}}t|trQqF|jdurbt|jtsb||jt|t|j||d}||qFt|dkrt j d|dt d d g}t |j d |j g}t |j d |j jg}t||D]\}}t||}||qng}}}d t||| |dtjdtjddiS)aReturns a dictionary containing enough metadata to reconstruct a pandas DataFrame as an Arrow Table, including index columns. Parameters ---------- columns_to_convert : list[pd.Series] df : pandas.DataFrame index_levels : List[pd.Index] index_descriptors : List[Dict] preserve_index : bool types : List[pyarrow.DataType] Returns ------- dict cSsg|] }t|ts|qSr)r&dict).0descrrrr  z&construct_metadata..N)rDr+rEFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesspandaspyarrow)Zlibraryversion) index_columnscolumn_indexescolumnsZcreatorZpandas_versionutf8)r:ziprLappendr&rMrDr2_column_name_to_stringswarningswarn UserWarningr9r[_get_simple_index_descriptorjsondumpsr __version__rrXencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indextypesZnum_serialized_index_levelsZntypesZdf_typesZ index_typesZcolumn_metadatacolZsanitized_namer+r>Zindex_column_metadataZnon_str_index_nameslevelZ descriptorrZrUrVrDrrrconstruct_metadatasl          rqcCsTt|\}}t|}d|vrtjdtdd|dkr"|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rRrSrencodingUTF-8rC)r?r4r`rarb)rprDrJrKrFrrrrcs rccCsdt|tr|St|tr|dSt|trtttt|St|tr(td|dur.dSt|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" r\z%Unsupported type for MultiIndex levelN) r&r2rdecodetuplemapr_rrHrDrrrr_"s     r_cCs(|jdur|j|vrt|jSd|S)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str Nz__index_level_{:d}__)rDr_r')indexirjrrr_index_level_nameIs  r{cCs:t|||}|jjstdt|j|durt|||Sg}|dur)t|jng}g}g}|D]%}||} t |}t | rGt d|| | | d| |q1g} g} t|D]0\} } t| | |}t| t jjry|duryt| }n| | | d|}| || |q_|| }||| | |||fS)Nz Duplicate column names found: {}F-Sparse pandas data (column {}) not supported.)_resolve_columns_of_interestr[ is_unique ValueErrorr'list$_get_columns_to_convert_given_schema_get_index_level_valuesryr_r is_sparserHr^ enumerater{r&pd RangeIndex_get_range_index_descriptor)rischemarmr[rjrkrhconvert_fieldsrDrorlindex_column_namesrz index_levelrO all_namesrrr_get_columns_to_convert\sN             rc Cs:g}g}g}g}g}g}|jD]} z|| } d} Wn@tyYzt|| } Wnttfy7td| w|durCtd| |durUt| tjj rUtd| d} Ynwt | } t | rjt d| | | } || || || | r|| || || q||} | ||||||fS)z Specialized version of _get_columns_to_convert in case a Schema is specified. In that case, the Schema is used as the single point of truth for the table structure (types, which columns are included, order of columns, ...). FzNname '{}' present in the specified schema is not found in the columns or indexzlname '{}' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifiedNaname '{}' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTr|)rVr%_get_index_level IndexErrorr'rr&rrrr_rrHfieldr^)rirrmrjrhrrlrrkrDroZis_indexrrrrrrsf           rcCs8|}||jjvrt|rt|tdd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). Z__index_level_)ryrV_is_generated_index_nameintr:get_level_values)rirDkeyrrrrs rcCs,zt||WStyt|YSwN)rdrerHr2rxrrr _level_names    rcCs.dt|jt|dt|dt|ddS)Nrangestartstopstep)kindrDrrr)rrDrZget_rangeindex_attribute)rprrrrs    rcs(ttdg}fddt|DS)NrUcsg|]}|qSr)r)rNrzryrrrPsz+_get_index_level_values..)r:r9r)rynrrrrsrcsR|dur |dur td|dur|j}|S|dur$fdd|D}|Sj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themcsg|] }|jvr|qSr)r[rNcrirrrP sz0_resolve_columns_of_interest..)rrVr[)rirr[rrrr}sr}cCst|d||\}}}}}}}g} |D]T} | j} t| r%tj| ddj} nrrrdataframe_to_typess<      rTc! sDt||||\}}}} } } } |dur-t|t|j} }| |dkr+|dkr+t}nd}fdddd}|dkrHfddt| | D}nNg}t|*}t| | D]\}}||jrg| ||qU| | ||qUWdn1s|wYt |D]\}}t |tj r|||<qd d|D}|durg}t||D]\}}|dur|nd }| t||qt|}t| ||| | ||}|jrt|jnt}||||}d}t|d krz'| d d }|d kr| d d}| d d}| d d} tt||| }Wn tyYnw|||fS)Ndrc s|dur d}d}n|j}|j}z tj||dd}Wn"tjtjtjfy<}z|jd|j |j f7_|d}~ww|sO|j dkrOt dt ||j |S)NT)r0rsafez0Conversion failed for column {!s} with type {!s}rz>Field {} was non-nullable but pandas column had {} null values)Znullabler0rrZ ArrowInvalidZArrowNotImplementedErrorZArrowTypeErrorargsr'rDr/Z null_countrr2)rorZfield_nullablerr3e)rrrconvert_columnGs0  z+dataframe_to_arrays..convert_columncSs$t|tjo|jjot|jjtjSr) r&npZndarrayflags contiguous issubclassr/r0integer)arrrrr_can_definitely_zero_copy]s z6dataframe_to_arrays.._can_definitely_zero_copycsg|] \}}||qSrr)rNrf)rrrrPcsz'dataframe_to_arrays..cSsg|]}|jqSrr0rNxrrrrPrsrBrrrrrr)rr:r[r cpu_countr]rZThreadPoolExecutorrr^Zsubmitrr&Futurer3rrrqr>rrMr with_metadatarr)!rirrmZnthreadsr[rrrjrrlrYrhrZnrowsZncolsrZarraysexecutorrrrzZ maybe_futrnfieldsrDrpandas_metadatar>Zn_rowsrrrrr)rrrdataframe_to_arrays2sz                rcCsd|jjtjkr ||fSt|r$|dur$|j}|j}t ||}||fS|dur.t |j}||fSr) r/r0rZ datetime64rZ is_datetimetzr*unitr timestampZfrom_numpy_dtype)rr/rr*rrrrrs  rcCs>ddlmm}|dd}|d}d|vr-tjj||d|dd}|j||d}|Sd |vret |j \}} t ||d } t rYtj j|d | d d } |j| |d}|S|j|||j| d }|Sd|vr|d} t|dksuJ||d} || }t|dstd|| }|j||d}|S|j||d}|S)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryr7)r;r7)rr8r Fr/copy)rklassr/Zpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)pandas.core.internalscoreZ internalsgetrZcategorical_typeZ from_codesZ make_blockrZ datetime_datar/make_datetimetz is_ge_v21rrviewZDatetimeTZBlockr:r1rr)itemr[extension_columns_intZ block_arrrr6rrrr/Zpd_arrrrD pandas_dtypeZ pd_ext_arrrrr_reconstruct_blocksH     rcCs&trd}tj|}tj||dS)Nnsr*)rZis_v1rrstring_to_tzinfoZdatetimetz_type)rr*rrrrs rFcCsddlm}ddlm}g}g}|jj} |s=| dur=| d}| dg}| d} t|| }t|| ||\}} t |||} n t j |j } t |g|} t|t|||} t|||| }| | g}|||}t rq|||j}|S||}|S)Nr) BlockManager) DataFramer[rZrY)rrZpandasrrrr_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_index_table_to_blocksrZ _from_mgraxes)optionstabler;Zignore_metadata types_mapperrr all_columnsrZrrlryZext_columns_dtypesr[blocksrZmgrrirrrtable_to_dataframes8        r> rrrrr rrr#r r r r rc Csi}tjdur |S|D]2}z|d}Wn ty |d}Ynw|d}|tvr=t|}t|tjr=t|dr=|||<q |jD] }|j} t| t j raz| }Wn t y[YqAw|||j <qA|rz|jD]}|j} || }|dury|||j <qg|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. NrErDrGr)rZextension_dtyper%_pandas_supported_numpy_typesrr&r1rr0rZBaseExtensionTypeZto_pandas_dtypeNotImplementedErrorrD) rZcolumns_metadatarZ ext_columnscol_metarDr/rrtyprrrrsD              rcCstdd|Ds JdS)Ncss0|]}|ddurd|vp|dduVqdS)rDNrErrrrr \s  z:_check_data_column_metadata_consistency..)all)rrrrrWsrcsdd|jD}|rdd|Dfdd|D}n|}t|dkr&tjndd}|s3tj|}ntjjjt t ||d d|DpEdd }t|d krSt ||}t |}|S) NcSs"g|] }t|tr t|n|qSr)r&rrrrrrrPcsz-_deserialize_column_index..cSs&i|]}|dt|d|dqSrErD)rr_rrrr fsz-_deserialize_column_index..csg|]}||qSrr)rNrDZcolumns_name_dictrrrPjs rcSs|fSrr)rrrrssz+_deserialize_column_index..cSsg|]}|dqSrxr)rN col_indexrrrrP}srVr) rjr:ast literal_evalrrIndex MultiIndex from_tuplesrrw"_reconstruct_columns_from_metadata _flatten_single_level_multiindex) block_tablerrZZcolumn_stringsZcolumns_valuesZto_pairr[rrrrbs,    rc Cs6dd|D}g}g}|}|D]O}t|tr't|||||\}} } | dur&qn-|ddkrK|d} tjj|d|d|d| d } t| t|krJqn td |d| | | | qtj} t|d krt| j j ||d } || fSt|d kr|d } t| | j s| j | |d d} || fS| |j } || fS)NcSsi|] }|d|d|qSrrrrrrrsz&_reconstruct_index..rrrDrrr)rrDzUnrecognized index kind: {}rrrrx)r&r2_extract_index_levelrrrr:rr'r^r from_arraysrr) rrlrrfield_name_to_metadataZ index_arraysZ index_names result_tablerOr index_namerryrrrrsP            rc Cs||d}t||}|j|}|dkr|ddfStj}||} | j|dj} t| dr6| j j s6| } t | j tjjrQ| j jdurQt|j| dd| j j} n |j| | jdd} ||j|}|| |fS)NrD)rrF)rr) _backwards_compatible_index_namerget_field_indexrrr= to_pandasrr1rZ writeablerr&r0rrr)r* make_tz_awarerr/Z remove_column) rrrErr logical_namerrzrrorrrrrrs"       rcCs||kr t|r dS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namer rrrrsrcCsd}t||duS)Nz^__index_level_\d+__$)rematch)rDpatternrrrrsrr-) rr!r rrr,rZfloatingrcCs:zt|WStyd|vrtjYSt|YSw)a Get the numpy dtype that corresponds to a pandas type. Parameters ---------- pandas_type : str The result of a call to pandas.lib.infer_dtype. Returns ------- dtype : np.dtype The dtype that corresponds to `pandas_type`. rr)_pandas_logical_type_mapr%robject_r/)rFrrr_pandas_type_to_numpy_types   rcCst|tjjr |jSdSr)r&rrrr<)mirrr_get_multiindex_codessrc stjt|ddp |g}t|pfdd|D}ddt||idD}g}tdd}|D]K\}}} t|} | tj krB| |}|d kr]t j |d d d } j|d d| }n |j| krg|| }|j| kru|d kru|| }||q/j|||jdS)a_Construct a pandas MultiIndex from `columns` and column index metadata in `column_indexes`. Parameters ---------- columns : List[pd.Index] The columns coming from a pyarrow.Table column_indexes : List[Dict[str, str]] The column index metadata deserialized from the JSON schema metadata in a :class:`~pyarrow.Table`. Returns ------- result : MultiIndex The index reconstructed using `column_indexes` metadata with levels of the correct type. Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` rUNcsg|] }t|qSr)rr:)rNrprrrrP?sz6_reconstruct_columns_from_metadata..cSs0g|]\}}||dt|j|ddfqS)rFrGN)rr2r/)rNrprrrrrPDs  ) fillvaluergrtr rr>r8T)utcr)rrr9rroperator methodcallerrrbytes_rwrrr to_datetime tz_convertr/Zastyper^rrV) r[rZrUlabelsZ levels_dtypesZ new_levelsencoderrprZ numpy_dtyper/r*rrrr%s6       rcs4|jtj|||t}fdd|DS)Ncsg|]}t|qSr)r)rNrr[rrrrPmsz$_table_to_blocks..)rjrrZtable_to_blocksrkeys)rrr;rr3rrrrfs   rcsjtj}t||jr3|jdkr3|j\t|\}j}|js!t d|j fdd|D||j ddS|S)NrzFound non-unique column indexcs g|] }|dkr |ndqS)rNr)rNZ_labelrUrrrP~s z4_flatten_single_level_multiindex..r)r/rD) rrr&rZnlevelsrUrr/r~rrrV)ryrrr/rrrrqs rcCsi}i}|j}|d}dd|D}t|}t|d|}t|dD]q\}} | d} | sC| d} ||kr=|||} | durCd} || } | dkr| d d kr|| } t| jtjj s_q$| d } | sfq$| d }|r|| jj kr| }tj d |d}tj j||d}t|| j||| <||| <q$t|dkrg}g}tt|jD]$}||vr||||||q|||||j|qtjj|t|dS|S)NrYcSsg|] }t|tr|qSr)r&r2)rNZidx_colrrrrPrQz%_add_any_metadata..r[rErDrBrrFr r>r8rrrr)r)rr:rrrr&r0rrr)r*rrZArrayrrrDrr^Tabler)rrZmodified_columnsZmodified_fieldsrrYZn_index_levelsZ n_columnsrzrr idxror>Z metadata_tzZ convertedZ tz_aware_typerr[rrrrrs^       rcCs$tj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz r)rrrdtZ tz_localizer)Zseriesr*rrrrs  rr)rNT)NN)NFN)Krcollections.abcrZ concurrentrZconcurrent.futures.threadrr itertoolsrrdrr r`numpyrrWrZ pyarrow.librrrrr(Zbool_r r r r rrrrrrZstr_rr.r4r?rLrqrcr_r{rrrrrrr}rrrrrrrrrrrrrrrrrrrrrrrrrrrs    -V'@@    " b B & 9 (7  A  A