o TZh@sUddlZddlmZddlmZmZmZddlZddl Z ddl m Z ddl mZddlmZdd lmZer?ddlZddlZeZdaeeed <Gd d d eed efZdS)N)Mapping) TYPE_CHECKINGDictOptional)config) get_logger) map_nested)TensorFormatterDEVICE_MAPPINGcseZdZdfdd ZedeedffddZdd Zd d Z d d Z de fddZ de jdefddZde jddfddZde jdefddZZS) JaxFormatterNc stj|dddl}ddlm}t||r#td|dt|dt|tr*|nt| d|_ t dur;| a |j t t vrgtd|j dt t d t| dd t| d|_ ||_dS) N)featuresr)Devicez Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxZjaxlib.xla_clientr isinstance ValueErrortypestrdevicesdevicer _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfrrrrr __class__X/var/www/html/lang_env/lib/python3.10/site-packages/datasets/formatting/jax_formatter.pyr's(      zJaxFormatter.__init__returnzjaxlib.xla_extension.DevicecCsddl}dd|DS)NrcSsi|]}t||qSr#)r).0rr#r#r$ Fz4JaxFormatter._map_devices_to_str..)rr)rr#r#r$rBsz JaxFormatter._map_devices_to_strcsLddlddlm}ttr$r$tfddDr$|jddSS)Nrc3s<|]}t|jo|jdjko|jdjkVqdS)rN)rArrayshapedtype)r&xcolumnrr#r$ Ms, z,JaxFormatter._consolidate..)Zaxis)r jax.numpynumpyrrallstack)r r.jnpr#r-r$ _consolidateHs zJaxFormatter._consolidatecCsZddl}ddlm}t|tttdfr|St|tjtj fr+t |j tjr+| Si}t|tj tj frNt |j tjrN|jjrHd|ji}n5d|ji}n/t|tj tj fret |j tjred|ji}ntjr}dtjvr}ddl}t||jjr}t|}tdur|a|t|j|j|fii||j WdS1swYdS)Nrr+PIL)!rr0r1rrbytesrnp characterndarrayZ issubdtyper+tolistnumberintegerrZjax_enable_x64Zint64Zint32ZfloatingZfloat32Z PIL_AVAILABLEsysmodulesZ PIL.ImageZImageZasarrayr rZdefault_devicerarrayr)r valuerr4Z default_dtyper6r#r#r$ _tensorizeSs, ""  "  $zJaxFormatter._tensorizecsddl}tjr#dtjvr#ddl}t||jr#| dSt |dr2t||j s2|}t|tjrJ|jtkrIfdd|DSnt|ttfr]fdd|DS|S)Nrtorchr# __array__cg|]}|qSr#recursive_tensorizer&Z substructr r#r$ r(z5JaxFormatter._recursive_tensorize..crEr#rFrHrIr#r$rJr()rrZTORCH_AVAILABLEr>r?rCrZTensorrBdetachcpur1hasattrr)rDr8r:r+objectr5rtuple)r data_structrrCr#rIr$_recursive_tensorizexs    z!JaxFormatter._recursive_tensorizerPcCst|j|ddS)NF)Zmap_list)r rQ)r rPr#r#r$rGsz JaxFormatter.recursive_tensorizepa_tablecCs$||}|j|}||SN)numpy_arrow_extractorZ extract_rowpython_features_decoderZ decode_rowrG)r rRrowr#r#r$ format_rows  zJaxFormatter.format_row jax.ArraycCs:||}|j||jd}||}||}|S)Nr)rTZextract_columnrUZ decode_columnZ column_namesrGr5)r rRr.r#r#r$ format_columns   zJaxFormatter.format_columncCsD||}|j|}||}|D] }|||||<q|SrS)rTZ extract_batchrUZ decode_batchrGr5)r rRbatchZ column_namer#r#r$ format_batchs   zJaxFormatter.format_batch)NN)__name__ __module__ __qualname__r staticmethodrrrr5rBrQdictrGpaTablerrWrYr[ __classcell__r#r#r!r$r &s %r rX)r>collections.abcrtypingrrrr1r8ZpyarrowrarZ utils.loggingrZutils.py_utilsr Z formattingr rZjaxlibrr r`__annotations__r r#r#r#r$s