o թZh,@sddlmZddlmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZdd lmZdd lmZddlZdd lmZdd lmZmZdd lmZer\ddlmZGdddeZdS)) annotations) TYPE_CHECKINGN)using_pyarrow_string_dtype)lib)import_optional_dependency) ParserError ParserWarning)find_stack_level) pandas_dtype) is_integer) DataFrame)_arrow_dtype_mappingarrow_string_types_mapper) ParserBase) ReadBuffercsTeZdZdZdfdd Zddd Zdd d ZdddZdddZdddZ Z S)ArrowParserWrapperz7 Wrapper for the pyarrow engine for read_csv() srcReadBuffer[bytes]returnNonec s$t|||_||_|dSN)super__init__kwdsr _parse_kwds)selfrr __class__]/var/www/html/lang_env/lib/python3.10/site-packages/pandas/io/parsers/arrow_parser_wrapper.pyr%s  zArrowParserWrapper.__init__cCsN|jd}|dur dn||_|jd}t|trtdt|jd|_dS)z? Validates keywords before passing to pyarrow. encodingNzutf-8 na_valuesz?The pyarrow engine doesn't support passing a dict for na_values)rgetr isinstancedict ValueErrorlistr!)rr r!rrrr,s   zArrowParserWrapper._parse_kwdscCsddddddd}|D]\}}||jvr'|j|dur'|j||j|<q |j}t|tr4|g}nd}||jd <d d |jD|_|jd }|durt|rZ||jd <n*|t j j krfd|jd <n|t j j krwddd}||jd <n |t j j krdd|jd <dd |jD|_d|jdv|jd<|jdurd|jvrdd|jdD|jd<|jdu|jdur|jn|jd|jd|_dS)z: Rename some arguments to pass to pyarrow include_columns null_values escape_charignore_empty_lines decimal_point quote_char)usecolsr! escapecharZskip_blank_linesdecimal quotecharNtimestamp_parserscS&i|]\}}|dur|dvr||qS)N) delimiterr,r)r*r.0Z option_nameZ option_valuerrr Yz;ArrowParserWrapper._get_pyarrow_options.. on_bad_linesZinvalid_row_handlerrstrcSs.tjd|jd|jd|jttddS)Nz Expected z columns, but found z: ) stacklevelskip)warningswarnZexpected_columnsZactual_columnstextrr )Z invalid_rowrrrhandle_warningks z?ArrowParserWrapper._get_pyarrow_options..handle_warningcSsdS)Nr;r)_rrrvsz9ArrowParserWrapper._get_pyarrow_options..cSr2)N)r'r(Z true_valuesZ false_valuesr+r1rr4rrrr6xr7Zstrings_can_be_nullcSsg|]}d|qS)fr)r5nrrr s z;ArrowParserWrapper._get_pyarrow_options..Zskiprows)Zautogenerate_column_namesZ skip_rowsr )rr9)itemsrr"pop date_formatr#r9 parse_optionscallablerZBadLineHandleMethodERRORWARNZSKIPconvert_optionsheaderr read_options)rmappingZ pandas_nameZ pyarrow_namerHr8r?rrr_get_pyarrow_options:s^           z'ArrowParserWrapper._get_pyarrow_optionsframer c stj}d}|jdur7|jdur|jdurt||_t|j|kr3tt|t|j|j|_d}|j_|j\}|jdur|j}t |jD]S\}}t |r_j|||<n |jvrlt d|d|j dur|j |dur||j |fn j||j j|f\}} | dur|| |<|j |=qOj|ddd|jdur|sdgtjjj_|j durt|j trևfdd|j D|_ nt|j |_ z |j WSty} zt | d} ~ wwS) z Processes data read in based on kwargs. Parameters ---------- frame: DataFrame The DataFrame to process. Returns ------- DataFrame The processed DataFrame. TNFzIndex z invalid)ZdropZinplacecs$i|]\}}|jvr|t|qSr)columnsr )r5kvrRrrr6s  z>ArrowParserWrapper._finalize_pandas_output..)lenrSrNnamesranger&Z_do_date_conversionsZ index_colcopy enumerater r%Zdtyper"ZastypeZ set_indexindexr#r$rFr TypeError) rrRZnum_colsZmulti_index_namedr@Z index_to_setiitemkeyZ new_dtypeerrVr_finalize_pandas_outputsX             z*ArrowParserWrapper._finalize_pandas_outputcCs8t|rtdd|Dstdt|rtddS)Ncs|]}t|tVqdSrr#r9r5xrrr sz7ArrowParserWrapper._validate_usecols..zwThe pyarrow engine does not allow 'usecols' to be integer column positions. Pass a list of string column names instead.z=The pyarrow engine does not allow 'usecols' to be a callable.)r is_list_likeallr%rJ)rr-rrr_validate_usecolssz$ArrowParserWrapper._validate_usecolsc Cstd}td}|z |jd i|j}Wn2tyI|jdd}|dur.|||jdt}t |rDt dd|DsHtdwz|j |j |j d i|j|jd i|j|d }Wn|jyv}zt||d}~ww|jd }|tjur|j} |} t|jjD]\} } |j| r| | | | | } q|| }|dkr|jtjd } n&|d krt }t!||"<|j|jd } nt#r|jt$d } n|} |%| S)z Reads the contents of a CSV file into a DataFrame and processes it according to the kwargs passed in the constructor. Returns ------- DataFrame The DataFrame created from the CSV file. Zpyarrowz pyarrow.csvr'Nr(csrcrrdrerrrrgs  z*ArrowParserWrapper.read..z9The 'pyarrow' engine requires all na_values to be strings)rOrIrM dtype_backend)Z types_mapperZnumpy_nullabler)&rrQZConvertOptionsrMr]r"rjsetrrhriZread_csvrZ ReadOptionsrOZ ParseOptionsrIZ ArrowInvalidrrZ no_defaultZschemaZfloat64r[typesZis_nullfieldZ with_typecastZ to_pandaspdZ ArrowDtyper Z Int64Dtypenullrrrb)rpaZ pyarrow_csvrMincludeZnullstablerarkZ new_schemanew_typer^Z arrow_typerRZ dtype_mappingrrrreadsd          zArrowParserWrapper.read)rrrr)rr)rRr rr )rr ) __name__ __module__ __qualname____doc__rrrQrbrjrv __classcell__rrrrr s   [ I r) __future__rtypingrr<Zpandas._configrZ pandas._libsrZpandas.compat._optionalrZ pandas.errorsrrZpandas.util._exceptionsr Zpandas.core.dtypes.commonr Zpandas.core.dtypes.inferencer Zpandasrpr Zpandas.io._utilr rZpandas.io.parsers.base_parserrZpandas._typingrrrrrrs"