o
    TZh1                     @   s~   d dl Z d dlmZ d dlmZ d dlZd dlZddl	m
Z
 ddlmZ ddlmZ er0d dlZG d	d
 d
eedef ZdS )    N)Mapping)TYPE_CHECKING   )config)
map_nested   )TensorFormatterc                       s   e Zd Zd fdd	Zdd Zdd Zdd	 Zd
efddZde	j
defddZde	j
ddfddZde	j
defddZ  ZS )TFFormatterNc                    s    t  j|d || _dd l}d S )N)featuresr   )super__init__tf_tensor_kwargs
tensorflow)selfr
   r   tf	__class__ W/var/www/html/lang_env/lib/python3.10/site-packages/datasets/formatting/tf_formatter.pyr   !   s   zTFFormatter.__init__c                    s`   dd l t tr. r.t fdd D r S t fdd D r.j S  S )Nr   c                 3   s<    | ]}t |jo|j d  jko|j d  jkV  qdS )r   N)
isinstanceTensorshapedtype.0xcolumnr   r   r   	<genexpr>*   s    ,
z+TFFormatter._consolidate.<locals>.<genexpr>c                 3   s<    | ]}t |jjfo|jd ko|j d jkV  qdS )r   r   N)r   r   ZRaggedTensorndimr   r   r   r   r   r   .   s
    *
)r   r   listallstackZragged)r   r   r   r   r   _consolidate&   s   
zTFFormatter._consolidatec                 C   s   dd l }|d u r
|S i }t|tjtjfr#t|jtjr#d|ji}n/t|tjtjfr:t|jtj	r:d|j
i}ntjrRdtjv rRdd l}t||jjrRt|}|j|fi i || jS )Nr   r   PIL)r   r   npnumberndarrayZ
issubdtyper   integerZint64ZfloatingZfloat32r   ZPIL_AVAILABLEsysmodulesZ	PIL.ImageZImageZasarrayZconvert_to_tensorr   )r   valuer   Zdefault_dtyper$   r   r   r   
_tensorize7   s   ""
zTFFormatter._tensorizec                    s   dd l }tjr#dtjv r#dd l}t||jr# |	 
  d S t|dr2t||js2| }t|tjrJ|jtkrI  fdd|D S nt|ttfr]  fdd|D S  |S )Nr   torchr   	__array__c                       g | ]}  |qS r   recursive_tensorizer   Z	substructr   r   r   
<listcomp>Y       z4TFFormatter._recursive_tensorize.<locals>.<listcomp>c                    r/   r   r0   r2   r3   r   r   r4   [   r5   )r   r   ZTORCH_AVAILABLEr)   r*   r-   r   r   r,   detachcpunumpyhasattrr.   r%   r'   r   objectr#   r    tuple)r   data_structr   r-   r   r3   r   _recursive_tensorizeK   s   

z TFFormatter._recursive_tensorizer<   c                 C   s   t | j|ddS )NF)Zmap_list)r   r=   )r   r<   r   r   r   r1   ^   s   zTFFormatter.recursive_tensorizepa_tablereturnc                 C   s$   |   |}| j|}| |S N)numpy_arrow_extractorZextract_rowpython_features_decoderZ
decode_rowr1   )r   r>   rowr   r   r   
format_rowa   s   
zTFFormatter.format_row	tf.Tensorc                 C   s:   |   |}| j||jd }| |}| |}|S )Nr   )rA   Zextract_columnrB   Zdecode_columnZcolumn_namesr1   r#   )r   r>   r   r   r   r   format_columnf   s
   

zTFFormatter.format_columnc                 C   sD   |   |}| j|}| |}|D ]}| || ||< q|S r@   )rA   Zextract_batchrB   Zdecode_batchr1   r#   )r   r>   batchZcolumn_namer   r   r   format_batchm   s   
zTFFormatter.format_batchr@   )__name__
__module____qualname__r   r#   r,   r=   dictr1   paTabler   rD   rF   rH   __classcell__r   r   r   r   r	       s    r	   rE   )r)   collections.abcr   typingr   r8   r%   ZpyarrowrM    r   Zutils.py_utilsr   Z
formattingr   r   r   r	   r   r   r   r   <module>   s   