o
    +if|                     @   st   d dl mZmZ d dlmZmZmZmZmZm	Z	m
Z
 d dlZer&ddlmZ eG dd dZeG dd	 d	ZdS )
    )	dataclassfield)TYPE_CHECKINGAnyClassVarDictListOptionalUnionN   )FeatureTypec                   @   s   e Zd ZU dZee ed< dZee ed< dZ	e
e ed< dZe
e ed< ed ddd	Zeed
< dd Zdedeedf f fddZdS )Translationa  `FeatureConnector` for translations with fixed languages per example.
    Here for compatiblity with tfds.

    Args:
        languages (`dict`):
            A dictionary for each example mapping string language codes to string translations.

    Example:

    ```python
    >>> # At construction time:
    >>> datasets.features.Translation(languages=['en', 'fr', 'de'])
    >>> # During data generation:
    >>> yield {
    ...         'en': 'the cat',
    ...         'fr': 'le chat',
    ...         'de': 'die katze'
    ... }
    ```
    	languagesNiddictdtypepa_typeFdefaultinitrepr_typec                 C   s   t dd t| jD S )Nc                 S   s   i | ]}|t  qS  )pastring).0langr   r   V/var/www/html/corbot_env/lib/python3.10/site-packages/datasets/features/translation.py
<dictcomp>*       z(Translation.__call__.<locals>.<dictcomp>)r   structsortedr   selfr   r   r   __call__)   s   zTranslation.__call__returnr   c                    s$   ddl m   fddt| jD S )z2Flatten the Translation feature into a dictionary.r   Valuec                    s   i | ]}| d qS )r   r   )r   kr&   r   r   r   0   r   z'Translation.flatten.<locals>.<dictcomp>)featuresr'   r!   r   r"   r   r&   r   flatten,   s   zTranslation.flatten)__name__
__module____qualname____doc__r   str__annotations__r   r	   r   r   r   r   r   r   r$   r
   r   r*   r   r   r   r   r      s   
 "r   c                   @   s   e Zd ZU dZdZee ed< dZee	 ed< dZ
ee ed< dZee ed< dZee ed< ed d	d	d
Zeed< dd Zdd Zdd Zdedeedf f fddZdS )TranslationVariableLanguagesa$  `FeatureConnector` for translations with variable languages per example.
    Here for compatiblity with tfds.

    Args:
        languages (`dict`):
            A dictionary for each example mapping string language codes to one or more string translations.
            The languages present may vary from example to example.

    Returns:
        - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`):
            Language codes sorted in ascending order or plain text translations, sorted to align with language codes.

    Example:

    ```python
    >>> # At construction time:
    >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
    >>> # During data generation:
    >>> yield {
    ...         'en': 'the cat',
    ...         'fr': ['le chat', 'la chatte,']
    ...         'de': 'die katze'
    ... }
    >>> # Tensor returned :
    >>> {
    ...         'language': ['en', 'de', 'fr', 'fr'],
    ...         'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
    ... }
    ```
    Nr   num_languagesr   r   r   r   Fr   r   c                 C   s:   | j r
tt| j nd | _ | j rt| j | _d S d | _d S )N)r   r!   setlenr2   r"   r   r   r   __post_init__\   s    z*TranslationVariableLanguages.__post_init__c                 C   s$   t t t  t t  dS )Nlanguagetranslation)r   r    list_r   r"   r   r   r   r$   `   s   $z%TranslationVariableLanguages.__call__c                    s   t | j}t |ddhkr|S | jr/t || r/tddtt ||  dd| dg }| D ]\ }t|trF| |f q5|	 fdd|D  q5t
t| \}}||d	S )
Nr7   r8   zSome languages in example (z, z) are not in valid set (z).c                    s   g | ]} |fqS r   r   )r   elr   r   r   
<listcomp>s   s    z?TranslationVariableLanguages.encode_example.<locals>.<listcomp>r6   )r3   r   
ValueErrorjoinr!   items
isinstancer/   appendextendzip)r#   translation_dictlang_settranslation_tuplestextr   translationsr   r;   r   encode_examplec   s   
(

z+TranslationVariableLanguages.encode_exampler%   r   c                 C   s*   ddl m}m} ||d||ddS )zCFlatten the TranslationVariableLanguages feature into a dictionary.r   )Sequencer'   r   r6   )r)   rJ   r'   )r#   rJ   r'   r   r   r   r*   z   s   

z$TranslationVariableLanguages.flatten)r+   r,   r-   r.   r   r	   r   r0   r2   intr   r/   r   r   r   r   r   r   r5   r$   rI   r
   r   r*   r   r   r   r   r1   3   s   
 "r1   )dataclassesr   r   typingr   r   r   r   r   r	   r
   pyarrowr   r)   r   r   r1   r   r   r   r   <module>   s    $'