o
    Zho                     @   s   d dl Z d dlZd dlZd dlmZ e jjZ	dddZ
zd dlmZ W n ey.   dZY nw dddZG dd	 d	eZe jd
ddd Ze jd
ddd Ze jd
ddddZe jd
ddddZe jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
dd d! Ze jd
dd"d# Ze jd
dd$d% Ze jd
dd&d' Zd(dd)d*d+Z d,d- Z!d.d/ Z"d0d1 Z#d2d3 Z$d4d5 Z%d6d7 Z&d8d9 Z'd:d; Z(d<d= Z)d>d? Z*d@dA Z+dBdC Z,dDdE Z-dFdG Z.dHdI Z/dJdK Z0dLdM Z1dNdO Z2dPdQ Z3dRdS Z4dTdU Z5dVdW Z6dXdY Z7dZd[ Z8d\d] Z9d^d_ Z:d`da Z;dbdc Z<ddde Z=dfdg Z>dhdi Z?djdk Z@dldm ZAdndo ZBdpdq ZCdrds ZDdtdu ZEdvdw ZFdxdy ZGdzd{ ZHd|d} ZIe jjJd~d ZKe jjJdd ZLe jjJdd ZMdS )    N)compute summarydescription
   c                 C   s   ddl m} |t | S )Nr   )_get_udf_context)Zpyarrow._computer   padefault_memory_pool)batch_lengthr    r   M/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/test_udf.pymock_udf_context&   s   r   c                   @   s   e Zd ZdS )MyErrorN)__name__
__module____qualname__r   r   r   r   r   +   s    r   session)scopec                  C   6   dd } d}t }t| ||dt it  | |fS )z4
    Register a unary aggregate function (mean)
    c                 W   s   t t|S N)r	   scalarnpZnansum)ctxxargsr   r   r   func4      z"sum_agg_func_fixture.<locals>.funcsum_udfr   )empty_udf_docpcregister_aggregate_functionr	   float64r   	func_namefunc_docr   r   r   sum_agg_func_fixture/   s   r&   c                  C   r   )Nc                 S      t d)NOops)RuntimeErrorr	   r   lenr   r   r   r   r   r   G      z(exception_agg_func_fixture.<locals>.funcy=exception_len(x)r   r   r    r!   r	   int64r#   r   r   r   exception_agg_func_fixtureE   s   r0   c                 C   6   dd }d}t }t|||dt it  ||fS )Nc                 S   s   t t|t  S r   )r	   r   r*   int32r+   r   r   r   r   [   s   z1wrong_output_dtype_agg_func_fixture.<locals>.funcy=wrong_output_dtype(x)r   r.   r   r   r$   r%   r   r   r   #wrong_output_dtype_agg_func_fixtureY      r5   c                 C   r1   )Nc                 S   s   t |S r   )r*   r+   r   r   r   r   n   r,   z0wrong_output_type_agg_func_fixture.<locals>.funcy=wrong_output_type(x)r   r.   r4   r   r   r   "wrong_output_type_agg_func_fixturel   r6   r8   c                  C   B   dd } d}ddd}t | ||t t dt  | |fS )z,
    Register a binary scalar function.
    c                 S   s   t jd||g| jdS )Nmultiplymemory_poolr    call_functionr<   )r   mr   r   r   r   binary_function   s   z,binary_func_fixture.<locals>.binary_functionzy=mxzfind y from y = mxr   )r?   r   r    register_scalar_functionr	   r/   )r@   r$   Z
binary_docr   r   r   binary_func_fixture   s   rC   c               	   C   sH   dd } ddd}d}t | ||t t t dt  | |fS )z-
    Register a ternary scalar function.
    c                 S   s,   t jd||g| jd}t jd||g| jdS )Nr:   r;   addr=   )r   r?   r   cZmxr   r   r   ternary_function   s   z.ternary_func_fixture.<locals>.ternary_functionzy=mx+czfind y from y = mx + cr   )array1array2Zarray3rA   )rF   Zternary_docr$   r   r   r   ternary_func_fixture   s   	rI   c                  C   r9   )zI
    Register a varargs scalar function with at least two arguments.
    c                 W   s(   |}|D ]}t jd||g| jd}q|S )NrD   r;   r=   )r   firstvaluesaccvalr   r   r   varargs_function   s   z.varargs_func_fixture.<locals>.varargs_functionz	z=ax+by+czfind z from z = ax + by + cr   )rG   rH   rA   )rN   r$   Zvarargs_docr   r   r   varargs_func_fixture   s   rO   c                  C   s4   dd } ddd}d}t | ||i t  | |fS )z-
    Register a nullary scalar function.
    c                 S   s   t jdg| j t  | jdS )N*   typer<   )r	   arrayr   r/   r<   contextr   r   r   nullary_func   s   z*nullary_func_fixture.<locals>.nullary_funczrandom functionzgenerates a random valuer   Ztest_nullary_funcrA   )rV   r%   r$   r   r   r   nullary_func_fixture   s   rW   c                  C   <   dd } d}i }t  }ddd}t| |||| | |fS )zi
    Register a scalar function which returns something that is neither
    a Arrow scalar or array.
    c                 S   s   dS )NrP   r   r   r   r   r   wrong_output_type      z9wrong_output_type_func_fixture.<locals>.wrong_output_typetest_wrong_output_typezreturn wrong output typer   r   r	   r/   r    rB   )rZ   r$   in_typesout_typedocr   r   r   wrong_output_type_func_fixture      
ra   c                  C   sD   dd } d}dt  i}t  }ddd}t| |||| | |fS )zq
    Register a scalar function whose actual output DataType doesn't
    match the declared output DataType.
    c                 S      t d|dgS NrD      r    r>   )r   rS   r   r   r   wrong_output_datatype   r   zAwrong_output_datatype_func_fixture.<locals>.wrong_output_datatypetest_wrong_output_datatyperS   zreturn wrong output datatyper   r   )r	   r/   Zint16r    rB   )rg   r$   r^   r_   r`   r   r   r   "wrong_output_datatype_func_fixture   s   
ri   c                  C   rX   )z>
    Register a scalar function with the wrong signature.
    c                   S   s   t jdt  dS )Nre   rR   )r	   r   r/   r   r   r   r   wrong_signature  s   z5wrong_signature_func_fixture.<locals>.wrong_signaturetest_wrong_signaturezUDF with wrong signaturer   r   r]   )rk   r$   r^   r_   r`   r   r   r   wrong_signature_func_fixture  rb   rm   c                  C   s4   dd } d}ddd}t | ||i t  | |fS )zE
    Register a scalar function which raises a custom exception.
    c                 S   r'   )Nerror raised by scalar UDF)r   rY   r   r   r   raising_func"  r,   z*raising_func_fixture.<locals>.raising_funcZ
test_raisezraising functionr   r   rA   )ro   r$   r`   r   r   r   raising_func_fixture  s   
rp   c                  C   r   )z$
    Register a vector function
    c                 S   s   t |  jddS )NT)Zpct)r	   rS   	to_pandascopyZrankr+   r   r   r   pct_rank3  s   z+unary_vector_func_fixture.<locals>.pct_ranky=pct_rank(x)r   )r   r    register_vector_functionr	   r"   )rs   r$   r`   r   r   r   unary_vector_func_fixture.  s   
rv   c                  C   sb   dd } d}t }t| ||t t t dtdt fdt fdt fg | |fS )z@
    Register a vector function that returns a struct array
    c                 S   sB   t jj|||gg dd }|jdddd }t j| S )NkvrE   namesrE   ry   rx   )columnsrK   index)r	   RecordBatchfrom_arraysrq   pivotZreset_indexZfrom_pandasto_struct_array)r   rx   ry   rE   dfZdf_pivotr   r   r   r   D  s   z)struct_vector_func_fixture.<locals>.pivot
y=pivot(x)rw   rx   v1v2)r   r    ru   r	   r/   r"   utf8struct)r   r$   r`   r   r   r   struct_vector_func_fixture?  s   &r   Trun_in_datasetr   c                C   s   | \}}|d u rd}|D ]}t |tjrd}t|}q|rd}t|}|j|ks+J tj|||d}	|t|g|R  }
|	|
ksCJ |rzdd |D }tj	
||}t|}dd |D }|jdtd	||id
}|djd |
ks|J d S d S )NTFre   lengthc                 S   s   g | ]	\}}d | qS )fieldr   ).0r}   Zin_arrr   r   r   
<listcomp>k  s    z)check_scalar_function.<locals>.<listcomp>c                 S   s   g | ]}t |qS r   )dsr   )r   
field_namer   r   r   r   n  s    resultr   )r|   r   )
isinstancer	   ZArrayr*   r    Zget_functionnamer>   r   Tabler   r   datasetZto_tabler   Z_callcolumnchunks)Zfunc_fixtureZinputsr   r   functionr   Z
all_scalarargr   r   Zexpected_outputfield_namestabler   	func_argsZresult_tabler   r   r   check_scalar_functionT  s4   

r   c                 C   s    t | tddgt g d S )Nr      r   r	   rS   r/   )unary_func_fixturer   r   r   test_udf_array_unaryt  s   r   c                 C   s2   t | tddgt tddgt g d S )Nr   r         r   )rC   r   r   r   test_udf_array_binary|  s
   r   c              	   C   sD   t | tddgt tddgt tddgt g d S )Nr   r   r   r      r   )rI   r   r   r   test_udf_array_ternary  s   r   c                 C   sh   t | tddgt tddgt tddgt tddgt tddgt g d S )Nr      r   r         r   r   )rO   r   r   r   test_udf_array_varargs  s   r   c                  C   s  ddd} dt  i}t  }dd }tt t|d | || W d    n1 s,w   Y  tjtdd td d	| || W d    n1 sLw   Y  d
}tjt|d t|d| |d  W d    n1 snw   Y  d}tjt|d t|d| d | W d    n1 sw   Y  t|d| i | d}tjt|d t|d| i | W d    d S 1 sw   Y  d S )Nztest udf inputzparameters are validatedr   r   c                 S   s   t dgS )Nr   )r	   rS   rT   r   r   r   test_reg_function     z3test_registration_errors.<locals>.test_reg_functionzfunc must be a callablematchZtest_none_function)DataType expected, got <class 'NoneType'>Ztest_output_functionz)in_types must be a dictionary of DataTypeZtest_input_functionr   z?Already have a function registered with name: test_reg_function)r	   r/   pytestraises	TypeErrorr    rB   KeyError)r`   r^   r_   r   expected_exprr   r   r   test_registration_errors  sT   "r   c                 C   sN   | \}}d}t jt|d t|dg W d    d S 1 s w   Y  d S )Nz9VarArgs function 'z=ax\+by\+c' needs at least 2 argumentsr   rP   )r   r   
ValueErrorr    r>   )rO   _r$   	error_msgr   r   r    test_varargs_function_validation  s
   "r   c                  C   s   dt  i} t  }ddi}dd }tjtdd t|d|| | W d    n1 s-w   Y  d	d
i}tjtdd t|d|| | W d    d S 1 sRw   Y  d S )Nr   r   Zdescc                 S   rc   rd   rf   )r   r   r   r   r   	add_const  r   z/test_function_doc_validation.<locals>.add_constz#Function doc must contain a summaryr   Ztest_no_summaryr   ztest summaryz'Function doc must contain a descriptionZtest_no_desc)r	   r/   r   r   r   r    rB   )r^   r_   r%   r   r   r   r   test_function_doc_validation  s.   "r   c                 C   s   t | g ddd d S )NFre   r   )r   )rW   r   r   r   test_nullary_function  s   
r   c                 C   L   | \}}t jtdd tj|g dd W d    d S 1 sw   Y  d S )NzUnexpected output type: intr   re   r   r   r   r   r    r>   )ra   r   r$   r   r   r   r\     s   "r\   c                 C   sX   | \}}d}t jt|d t|tddgg W d    d S 1 s%w   Y  d S )NzDExpected output datatype int16, but function returned datatype int64r   r   r   )r   r   r   r    r>   r	   rS   )ri   r   r$   r   r   r   r   rh     s
   "rh   c                 C   sP   | \}}d}t jt|d tj|g dd W d    d S 1 s!w   Y  d S )Nz@wrong_signature\(\) takes 0 positional arguments but 1 was givenr   re   r   r   )rm   r   r$   r   r   r   r   rl     s
   "rl   c                  C   sl   dd } d}dt  i}i }ddd}tjtdd	 t| |||| W d    d S 1 s/w   Y  d S )
Nc                 S      |S r   r   r   rM   r   r   r   identity  r[   z1test_wrong_datatype_declaration.<locals>.identitytest_wrong_datatype_declarationrS   ztest output valueztest outputr   z%DataType expected, got <class 'dict'>r   r	   r/   r   r   r   r    rB   r   r$   r^   r_   r`   r   r   r   r     s   "r   c                  C   sl   dd } d}dd i}t  }ddd}tjtdd	 t| |||| W d    d S 1 s/w   Y  d S )
Nc                 S   r   r   r   r   r   r   r   r   *  r[   z3test_wrong_input_type_declaration.<locals>.identity!test_wrong_input_type_declarationrS   ztest invalid input typezinvalid input functionr   r   r   r   r   r   r   r   r   )  s   
"r   c                 C   s   t t  }| \}}tj|t jdgd t  dg|d}|t jdgd t  dks.J | dks6J d }| dks@J d S )Nre     rj   r;   r   @  r   )r	   proxy_memory_poolr
   r    r>   rS   r/   bytes_allocated)r   
proxy_poolr   r$   resr   r   r   test_scalar_udf_context:  s    r   c                 C   r   )Nrn   r   re   r   )r   r   r   r    r>   )rp   r   r$   r   r   r   test_raising_funcI  s   "r   c                 C   s2   | \}}t |tdg}|tdksJ d S )Nr      )r    r>   r	   r   )r   r   r$   r   r   r   r   test_scalar_inputO  s   r   c                 C   s   | \}}t t  }| dksJ t jdgd t  |d}| dks(J t||g | dks7J d }| dksAJ d S )Nr   re   r   rQ   r   )r	   r   r
   r   rS   r/   r    r>   )r   r   r$   r   ry   r   r   r   test_input_lifetimeU  s   r   c                    s&    fddt |D }tjj| dS )Nc                    s(   g | ]\}}t jt| | jd qS rj   )r	   rS   listrR   )r   iry   schemar   r   r   e  s    z,_record_batch_from_iters.<locals>.<listcomp>arraysr   )	enumerater	   r~   r   )r   Zitersr   r   r   r   _record_batch_from_itersd  s   
r   c                 C   s$   t | t||d t|d |d S )Nr   re   r   )r   range)r   nr   r   r   _record_batch_for_rangej  s   r   c                    s    fdd}|S )Nc                    s   G  fddd}| S )Nc                       s"   e Zd Zdd Z fddZdS )z,make_udt_func.<locals>.udf_func.<locals>.UDTc                 S   s
   d | _ d S r   )callerselfr   r   r   __init__s     
z5make_udt_func.<locals>.udf_func.<locals>.UDT.__init__c                    sh   z| j d u r |jd | _ }|  |}W | S  ty3   dd D }tjj|d}Y | S w )Nc                 S   s   g | ]
}t jg |jd qS r   )r	   rS   rR   )r   r   r   r   r   r   |  s    zImake_udt_func.<locals>.udf_func.<locals>.UDT.__call__.<locals>.<listcomp>r   )r   sendStopIterationr	   r~   r   r   )r   r   batchr   	batch_genr   r   r   __call__v  s   
z5make_udt_func.<locals>.udf_func.<locals>.UDT.__call__Nr   r   r   r   r   r   r   r   r   UDTr  s    r   r   )r   r   r   r   r   udf_funcq  s   zmake_udt_func.<locals>.udf_funcr   )r   r   r   r   r   r   make_udt_funcp  s   r   c                      s$   t  G fddd  fddS )zA short datasetc                       s    e Zd Zdd Z fddZdS )z%datasource1_direct.<locals>.Generatorc                 S   s
   d| _ d S )Nr   )r   r   r   r   r   r     r   z.datasource1_direct.<locals>.Generator.__init__c                    s@   | j dkrt g g }| S |  j d8  _ t | j }| S )Nr   re   )r   r   r   r   )r   r   r   r   r   r   r     s   
z.datasource1_direct.<locals>.Generator.__call__Nr   r   r   r   r   	Generator  s    r   c                    s     S r   r   rY   )r   r   r   <lambda>  s    z$datasource1_direct.<locals>.<lambda>)datasource1_schemar   r   )r   r   r   datasource1_direct  s   r   c                        t    fdd} t | S )Nc                 3   s(    t dddD ]
}t |d V  qd S )Nr   r   re   )r   r   r   r   r   r   r   r     s   z(datasource1_generator.<locals>.batch_genr   r   r   r   r   r   datasource1_generator  s   
r   c                     r   )Nc                 3   s,    t dddD ]
}t |d V  qtd)Nr   r   r   re   datasource1_exception)r   r   r)   r   r   r   r   r     s   z(datasource1_exception.<locals>.batch_genr   r   r   r   r   r     s   
r   c                   C   s   t dt  fdt  fgS )Nr   )r	   r   r2   r   r   r   r   r     s   r   c                 C   s@   | ddd}i }t dt  fdt  fg}| ||||fS )Nz UDTztest {func_name} UDTr   r   )r	   r   r2   )r   r$   r%   r^   r_   r   r   r   datasource1_args  s   r   c                 C   sX   t  }|  }| j}t||}tj|  d}t|D ]}|d8 }|t||ks)J qd S )Nr   re   )r   r   r   r    Zregister_tabular_functionZcall_tabular_functionr   )Z
func_makerr   r   r$   r   r   itemr   r   r   _test_datasource1_udt  s   

r   c                   C      t t d S r   )r   r   r   r   r   r   test_udt_datasource1_direct  r   r   c                   C   r   r   )r   r   r   r   r   r   test_udt_datasource1_generator  r   r   c                   C   s<   t jtdd tt W d    d S 1 sw   Y  d S )Nr   r   )r   r   r)   r   r   r   r   r   r   test_udt_datasource1_exception  s   
"r   c                 C   s<   t g dt  }td|g}t d}||ksJ d S )Ng      $@g      4@      >@g      D@g      I@mean_udfr   )r	   rS   r"   r    r>   r   )unary_agg_func_fixturearrr   expectedr   r   r   test_scalar_agg_basic  s   
r  c                 C   sT   t g t  }tjt jdd td|g W d    d S 1 s#w   Y  d S )Nzempty inputsr   r  )r	   rS   r"   r   r   ZArrowInvalidr    r>   )r  emptyr   r   r   test_scalar_agg_empty  s   "r  c                 C   X   t g dt  }tjt jdd td|g W d    d S 1 s%w   Y  d S )Nr   r   r   (   2   output datatyper   r3   r	   rS   r/   r   r   ArrowTypeErrorr    r>   )r5   r  r   r   r   "test_scalar_agg_wrong_output_dtype     "r  c                 C   r  )Nr	  output typer   r7   r  )r8   r  r   r   r   !test_scalar_agg_wrong_output_type  r  r  c                 C   sR   t g dt  }t g dt  }td||g}t d}||ks'J d S )Nr	  )      ?       @      @      @g      @Zsum_meang     @@)r	   rS   r/   r"   r    r>   r   )Zvarargs_agg_func_fixturearr1arr2r   r  r   r   r   test_scalar_agg_varargs  s   
r  c                 C   sV   t g dt  }tjtdd td|g W d    d S 1 s$w   Y  d S )N)r   r   r   r
  r  <   r(   r   r-   )r	   rS   r/   r   r   r)   r    r>   )r0   r  r   r   r   test_scalar_agg_exception  s   "r  c           
      C   s   t g dt  }t g dt  }t g dt  }t g dt  }t j||gddgd}t j||gddgd}t ||g}|ddg}|dd	gdd
g}	|	d|		dkshJ d S )Nr   r   r   re   r   re   )g      N@g     Q@g      T@g     V@g      Y@)r   re   re   r   re   idvaluerz   r  r  )r  meanvalue_mean_udf)
r	   rS   r"   r2   r   Zconcat_tablesgroup_by	aggregaterename_columnssort_by)
r  r  r  Zarr3Zarr4Ztable1Ztable2r   r   r  r   r   r   test_hash_agg_basic  s   

r&  c                 C   s   t g t  }t g t  }t j||gddgd}|ddg}t jt g t  t g t  gddgd}||ksAJ d S )Nr  r  rz   r  r!  )r	   rS   r"   r2   r   r"  r#  )r  r  r  r   r   r  r   r   r   test_hash_agg_empty  s   r'  c                 C      t g dt  }t g dt  }t j||gddgd}tjt jdd |d	dg W d    d S 1 s<w   Y  d S )	Nr	  r  r  r  rz   r  r   )r  r3   
r	   rS   r/   r2   r   r   r   r  r"  r#  )r5   r  r  r   r   r   r    test_hash_agg_wrong_output_dtype  s   "r*  c                 C   r(  )	Nr	  r  r  r  rz   r  r   )r  r7   r)  )r8   r  r  r   r   r   r   test_hash_agg_wrong_output_type$  s   "r+  c                 C   s   t g dt  }t g dt  }t j||gddgd}tjtdd |d	dg W d    d S 1 s;w   Y  d S )	Nr	  r  r  r  rz   r(   r   )r  r-   )
r	   rS   r/   r2   r   r   r   r)   r"  r#  )r0   r  r  r   r   r   r   test_hash_agg_exception-  s   "r,  c                 C   s   d}d}t td|t  }t tj||t  }t j||gddgd}|	d
dg}|	d
dgdd	g}|d|dksKJ d
S )z2Test hash aggregate udf with randomly sampled datai@B r   re   r  r  rz   )r  r   )r  sumZvalue_sum_udfN)r	   rS   r   repeatr"   randomchoicer2   r   r"  r#  r$  r%  )r&   Z	value_numZ	group_numr  r  r   r   r  r   r   r   test_hash_agg_random6  s   

r1  c                 C   s@   t g dt  }td|g}| d d |}||ksJ d S )Nr   rt   r   r	   rS   r"   r    r>   rv   r  r   r  r   r   r   test_vector_basicH  s   r4  c                 C   s>   t dgt  }td|g}| d d |}||ksJ d S )Nre   rt   r   r2  r3  r   r   r   test_vector_emptyP  s   r5  c                 C   sj   t g dt  }t g dt  }t g d}td|||g}| d d |||}||ks3J d S )N)re   re   r   r   )r  r  r  r  )r   r   r   r   r   r   )r	   rS   r/   r"   r    r>   )r   rx   ry   rE   r   r  r   r   r   test_vector_structX  s   r6  )r   )r   )Nr   numpyr   Zpyarrowr	   r   r    markr   Z
pytestmarkr   Zpyarrow.datasetr   ImportErrorr   r)   r   Zfixturer&   r0   r5   r8   rC   rI   rO   rW   ra   ri   rm   rp   rv   r   r   r   r   r   r   r   r   r   r   r\   rh   rl   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r&  r'  r*  r+  r,  r1  Zpandasr4  r5  r6  r   r   r   r   <module>   s   



























 	
2	


			

