o
    Zhwm                     @   s  d Z ddlZddlZddlZddlZddlZedZ	e
 ZedkZejje de dZdZdZdd Zd	d
 Zdd Zejdg ddd Zdd ZdCddZejdg ddd Zejdg ddd Zdd Zejdg ddd Zejdg ddd Zejdg dd d! Zejdg dd"d# Zejd$d%d&gejdg dd'd( Z ejdg dd)d* Z!d+d, Z"d-d. Z#d/d0 Z$d1d2 Z%d3d4 Z&d5d6 Z'd7d8 Z(d9d: Z)d;d< Z*d=d> Z+d?d@ Z,eejdg ddAdB Z-dS )Dz
UNTESTED:
read_message
    Nzpyarrow.cudazlinux-x86_64z'CUDA IPC not supported in platform `%s`)reasonc                 C   s&   t d| _t t j d | _d S )Nr      )cudaContextglobal_contextget_num_devicesglobal_context1module r   N/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/test_cuda.pysetup_module.   s   r   c                 C   s   | ` d S N)r   r	   r   r   r   teardown_module3   s   r   c                   C   s|   t j dks	J tjdksJ tjt j d ksJ tjtdd t t j  W d    d S 1 s7w   Y  d S )Nr   r   z5device_number argument must be non-negative less thanmatch)	r   r   r   r   device_numberr   pytestraises
ValueErrorr   r   r   r   test_Context7   s   "r   size)r   r     c                 C   sj   t | }tj|tjd}d|| d d|  d < | }tj|tjd}tj|| |j| ks3J d S )Ndtyper         )	r   new_host_buffernp
frombufferuint8copytestingassert_equalr   )r   bufarrZarr_cparr2r   r   r   test_manage_allocate_free_hostB   s   
r'   c                  C   s6   t j} t d}t j| d ksJ ~t j| ksJ d S )N   )r   bytes_allocated
new_buffer)r)   cudabufr   r   r   test_context_allocate_delM   s
   
r,   hostc                 C   s   |dkrS| dks
J t | }|j| ksJ tj|tjd}|j| ks%J tjjdd| tjd|dd< | dks@| dks@J tj|tjd}tj	
|| ||fS |dkrwt| dd	\}}t| }|j| kskJ |j|d| d
 ||fS td)z5Return a host or device buffer with random data.
    r-   r   r   r      )lowhighr   r   Ndevicetargetpositionnbyteszinvalid target value)paallocate_bufferr   r   r   r    randomrandintsumr"   r#   make_random_bufferr   r*   copy_from_hostr   )r   r3   r$   r%   arr_dbufr   r   r   r<   U   s$   

r<   c           
      C   s  t | \}}t|}|j| ksJ tj| tjd}tj	|| t
t t| W d    n1 s7w   Y  t|}|j| ksHJ tj| tjd}tj	|| t| }|j| ksgJ tj| tjd}tj	|| |d|j}|j| ksJ tj| tjd}tj	|| | dkr||| d d   tj| tjd}tj	t|| d d  || d d  f| ||d | d   t|}|j| ksJ tj| tjd}tj	|| ||| d d   tj| tjd}tj	|| |d|jd }|j| ksJ tj| tjd}tj	|| || d | d }|j| | d  ksDJ tj| tjd}tj	|| d d  | | d }d|  d }tj|||d}|j|ksuJ tj| tjd}tj	||||  | t|j||d}|j|ksJ tj| tjd}tj	||||  | tj|||d}|j|ksJ tj| tjd}tj	||||  | t||||  }|j|ksJ tj| tjd}tj	||||  | tj| ||d}|j|ksJ tj| tjd}tj	||||  | t| }|j| ks<J t|}|||}|j|ksOJ tj| tjd}tj	||||  | t| }tj|tjd}	||	d d < t|}|j| ksJ tj| tjd}tj	|| tj|||d}|j|ksJ tj| tjd}tj	||||  | t|j||d}|j|ksJ tj| tjd}tj	||||  | d S )	Nr   r   r      
   r   )offsetr   rB   length)r<   r   buffer_from_datar   r   r   copy_to_hostr    r"   r#   r   r   BufferError
memoryviewtobytesslicer=   Zconcatenater*   r   r   )
r   r%   r$   r+   r&   Zcudabuf2Zarr3ZsoffsetZssizer>   r   r   r   test_context_device_bufferl   s   


*






rK   c                 C   s@  t }t| dd\}}|j}t| |jj }|tj||dd d < ||}|j	|j	ks.J tj|
 |d}tj|| ||}|j	|j	ksKJ tj|
 |d}tj|| tjtjdd |td W d    n1 svw   Y  tjtjdd |tg d W d    d S 1 sw   Y  d S )	Nr1   r2   r   z$buffer is not backed by a CudaBufferr   s   123z8cannot create device buffer view from .* 'numpy.ndarray')r   r@   r   )r   r<   r   r   r   itemsizer   r   Zbuffer_from_objectr   rF   r"   r#   r   r   r7   ZArrowTypeError	py_bufferarray)r   ctxr%   cbufr   hbufZcbuf2r&   r   r   r   test_context_from_object   s.   

"rR   c                  C   s   t } ttj}d}t||j }t|}| 	|j
|j|}t||d ks+J ~t||ks5J | 	|j
|j|}~|  t||j }| 	|j
|j}~tjtjdd |  W d    d S 1 slw   Y  d S )NrA   r   zCuda error r   )r   r   r   r    r   r   rL   sysgetrefcountZforeign_bufferaddressr   rF   r   r   r7   ZArrowIOError)rO   r   r   rQ   rcZfbufr   r   r   test_foreign_buffer  s(   

"rW   c                 C   s(  t | \}}| | ksJ t|}|j| ksJ |jr!J | | ks+J | dkr6|jdks6J t| D ]}|| || ksFJ q:t	d t	| d | d fD ]}||  ||  ksdJ qT|	| d | d }|j
|ksvJ tjtdd t  W d    d S 1 sw   Y  d S )Nr   r   r@   z-Do not call CudaBuffer's constructor directlyr   )r<   rI   
to_pybytesr   rE   r   is_cpurU   rangerJ   parentr   r   	TypeErrorr   
CudaBuffer)r   r%   r$   rP   issbufr   r   r   test_CudaBuffer'  s*   


"ra   c                 C   s,  t | \}}| | ksJ t| }|tj|tjdd d < |j| ks(J |j	s-J | | ks7J t
| D ]}|| || ksGJ q;td t| d | d fD ]}||  ||  kseJ qU|| d | d }|j|kswJ ~tjtdd t  W d    d S 1 sw   Y  d S )Nr   r   r@   z-Do not call HostBuffer's constructor directlyr   )r<   rI   rX   r   r   r   r   r    r   rY   rZ   rJ   r[   r   r   r\   Z
HostBuffer)r   r%   r$   rQ   r^   r_   r`   r   r   r   test_HostBufferC  s*   


"rb   c           	      C   s
  t d}| |j }tj|dd}t|tjsJ t|tjr!J t j	||d}|j
| ks/J t| |d d < t j	||d}t j|| t|}t|tjsRJ t|tjsZJ |j
|ksaJ |jrfJ |j|d|d |jd|d}t j	||d}t j|| d S )NZuint16T)Z	resizabler   r   r4   )r   r   rL   r7   r8   
isinstanceBufferr   r]   r   r   rZ   r"   r#   r   r*   rY   r=   rF   )	r   dtr6   r$   r%   r>   Zdevice_bufferbuf2r&   r   r   r   test_copy_from_to_host^  s&   



rg   c              	   C   sF  t | dd\}}| }|jsJ tj|tj|tjd |j| d d}|js+J tj|| d d  tj|tjd |j| d | d d}|jsNJ tj|| d | d | d   tj|tjd |j| d dd}|jsuJ |jdks|J | d	 d
fd| d dfdfD ]"\}}t	j
tdd |j||d W d    n1 sw   Y  qd| d f| d	 | d d	 d f| dffD ]"\}}t	j
tdd |j||d W d    n1 sw   Y  qt| d }|j|d tj|d | d  tj|tjd | dk rd S |j|dd tj|dd| d   tj|tjd |j|dd tj|d d tj|tjdd d  |j|ddd tj|dd tj|tjdd d  d| d fd| d fd| d	 f| d | d d ffD ]%\}}t	j
tdd |j|||d W d    n	1 sw   Y  q{d S )Nr1   r2   r   r   r5      r4   r   r@   rj   r   r   !position argument is out-of-ranger   8requested more to copy than available from device buffer)r$      )r$   r5   )r$   r6      )r$   r6   r5      rA      z,requested copy does not fit into host buffer)r$   r5   r6   )r<   rF   rY   r   r"   r#   r   r    r   r   r   r   r7   r8   )r   r%   r?   r$   r5   r6   r   r   r   test_copy_to_hostz  sp   

&
 
&&
*ru   dest_ctxZsameanotherc              	      s   t |dd\} | }| dkrt}  jj| jkrtd n j} | | fdd}| |ks5J |dkr||d d|d |d  |d | d   ksSJ | |ksZJ |d	|d
 d|d d	 |d |d
   |||d
  d	  d   ksJ |d
 dfd|d	 dfdfD ]!\}}tjt	dd |||d W d    n1 sw   Y  qd|d	 ffD ]!\}}tjt	dd |||d W d    n1 sw   Y  q|dk rd S |d
 |d	 d
 d	 ffD ]"\}}tjt	dd |||d W d    n	1 sw   Y  qd S )Nr1   r   r3   rw   znot a multi-GPU systemc                     4   j  g| R i |  }tj|tjd S Nr   )Zcopy_from_devicerF   r   r   r    tolistargskwargsZrbufr$   r?   r   r   put     z"test_copy_from_device.<locals>.putr   rh   r   r@   r4   rj   rk   r   rm   ro   r   rp   6requested more to copy than available in device buffer)
r<   r{   r   contextr   r   skipr*   r   r   )rv   r   r%   lstr   r5   r6   r   r   r   test_copy_from_device  sX   

44
r   c              	      s  t | dd\} | }t|  fdd}| |ksJ | dkri|| d d|d | d  |d |  d   ks=J | |ksDJ |d| d d	|d d |d | d   || | d  d  d   ksiJ | d d
fd| d dfdfD ]!\}}tjtdd |||d	 W d    n1 sw   Y  qwd| d ffD ]!\}}tjtdd |||d	 W d    n1 sw   Y  q| dk rd S | d | d d d ffD ]!\}}tjtdd |||d	 W d    n1 sw   Y  qd S )Nr-   rx   c                     ry   rz   )r=   rF   r   r   r    r{   r|   r   r   r   r     r   z test_copy_from_host.<locals>.putr   rh   r   r@   r4   rj   rk   r   rm   ro   r   z6requested more to copy than available from host bufferr   )r<   r{   r   r*   r   r   r   )r   r%   r   r   r5   r6   r   r   r   test_copy_from_host  sL   
44
r   c                     s  dd  d fdd	} d\}}| || | |||d   d\}}| tjdtjd	 |d
tjdtjd	 | tjdtjd	 |  t| tj}tj	|d d
 tjd
tjd	 tj	|d
d tjdtjd	 tj	|dd  tjdtjd	 d S )Nc                 S   s   t | }t|}||fS r   )r   r*   r   BufferWriter)r   rP   writerr   r   r   allocate  s   

z#test_BufferWriter.<locals>.allocater   c                    s   | \}}t | dd\}}|dkr||_| }|dksJ ||j|d | |ks0J |d | }|dks?J || k r\t|| | }||j||d ||7 }|| k sC|  |j| ksgJ |j	
  | }	|j	
  |	j| ks|J tj|	tjd}
tj||
 d S )Nr-   rx   r   rD   rC   r   )r<   buffer_sizetellwriterJ   seekminflushr   r   synchronizerF   r   r   r    r"   r#   )
total_size	chunksizer   rP   r   r%   r$   r5   Zbytes_to_writerf   r&   r   r   r   test_writes  s0   


z&test_BufferWriter.<locals>.test_writes)i   r      d   r   2      K   )r   )
r   r   Zaranger    Zwriteatr   r   rF   r"   r#   )r   r   
chunk_sizerP   r   r%   r   r   r   test_BufferWriter  s   
""&r   c                  C   sL  d} t | }t|}t| dd\}}|jdksJ d|_|jdks%J ||jdd | dks6J ||jdd d|_|jdksIJ |j	dksPJ ||jdd	d
 |j	dksaJ ||jddd
 |j	dksrJ ||jddd
 ||jddd
 |
  | }|j| ksJ tj|tjd}tj|| d S )Nr   r-   rx   r   r   r   rA      i,  rC   i6  i  i  i  r   )r   r*   r   r   r<   r   r   rJ   r   Znum_bytes_bufferedr   rF   r   r   r   r    r"   r#   )r   rP   r   r%   r$   rf   r&   r   r   r   test_BufferWriter_edge_casesL  s0   

r   c                  C   s  d} t | dd\}}t|}|d | dksJ |d}t|dks)J | dks1J |d tjdtj	d}|
|}|d	ksIJ | dksQJ tj|dd  |d d	  |d
 | d
ksmJ | }tj| tj	d}tj|| d S )Nr   r1   rx   i  r   r   i  r   r   r   )r<   r   BufferReaderr   r   readlenr   Zzerosr    readintor"   r#   read_bufferr   rF   )r   r%   rP   readerdatar&   nrf   r   r   r   test_BufferReaderp  s&   





r   c                  C   sz   t ddd\} }t|}|d | }t|dksJ | dks&J | }tj	|
 tjd}tj| | d S )Nr   r1   rx   r   )r<   r   r   r   r   r   r   r   r   r   rF   r    r"   r#   )r%   rP   r   r   rf   r&   r   r   r   test_BufferReader_zero_size  s   

r   c                 C   st   t t dt  t dt  g}t tjjdd| tjd}t tjjdd| tjd}t j||g|d}|S )NZf0f1r   r.   )r   r   schema)	r7   r   fieldint16rN   r   r9   r:   record_batch)rD   r   a0a1batchr   r   r   make_recordbatch  s   r   c                  C   s   t d} |  }t| t}t|| j}t|tj	sJ | j|jks%J | j
|j
ks-J | j|jks5J | }||s@J tj|| j}|| sQJ | j
|j
ksYJ | j|jksaJ | d|dsnJ | |suJ d S )NrA   r   )r   	serializer   Zserialize_record_batchr   Zread_record_batchr   rc   r7   ZRecordBatchnum_columnsnum_rowsrF   equalsipccolumn)r   rQ   rP   Zcbatchr$   Zbatch2r   r   r   test_batch_serialize  s    r   c            
      C   s   t jg dt  d} t jddgdgg d gt t  d}t jg dt dt  fdt  fgd}t jj	g d	t d
dgd}t jj	g d|d}t jj	g d|d}| |||||g}t 
dd t|D }t j||d}t j|g}	|	S )N)r   r   *   N)typer   r   r@   ))abT)ZcdeF)NNNstrsZbools)r   r   r   NZfoobar)indices
dictionary)r@   r   r@   N)r@   r   r   Nc                 S   s    g | ]\}}d  ||jfqS )zf{})formatr   ).0r^   r%   r   r   r   
<listcomp>  s    zmake_table.<locals>.<listcomp>r   )r7   rN   r   list_Zint32structutf8Zbool_ZDictionaryArrayfrom_arraysr   	enumerater   TableZfrom_batches)
r   r   Za2a3Za4Za5Zarraysr   r   tabler   r   r   
make_table  s4   &


r   c                  C   s   t  } t }tj|| j}||  W d    n1 sw   Y  t| 	 }t
t|}|j|t|d tjt| }|| ||fS )N)r6   )r   r7   ZBufferOutputStreamr   Z
new_streamr   Zwrite_tablerM   getvaluerX   r   r*   r   r=   open_streamr   r   read_all)htableZsinkoutrQ   r?   dtabler   r   r   make_table_cuda  s   r   c                  C   sr   t  \} }}}|j|jksJ |j|jksJ |j|jksJ | | s(J |tj| 	 s7J d S r   )
r   r   r   r   r   rF   r7   r   r   r   )rQ   r   r?   r   r   r   r   test_table_deserialize  s   
r   c                  C   s   t  \} }}}tj|j|j}|j|jksJ |j|jks J |j|jks(J | 	|
 s1J |	tj|
  s@J d S r   )r   r7   r   r   columnsZcolumn_namesr   r   r   r   rF   r   r   r   )rQ   r   r?   r   Zdtable2r   r   r   %test_create_table_with_device_buffers  s   
r   c                 C   sx   t jd}t jj| }||}|j  | }|j	|j	ks+J t
|j	|j	ftj||jd}tj|| d S )Nr   r   )r7   r   r   ZIpcMemHandleZfrom_bufferZopen_ipc_bufferr   r   rF   r   reprr   r   r   r"   r#   )handle_bufferZexpected_arrZother_context
ipc_handleZipc_bufr$   r%   r   r   r   other_process_for_test_IPC  s   

 r   c                 C   sf   dd l }|d}t| dd\}}| }| }|jt||fd}|  |  |j	dks1J d S )Nr   Zspawnr1   rx   )r3   r}   )
multiprocessingZget_contextr<   Zexport_for_ipcr   Processr   startjoinexitcode)r   r   rO   r%   rP   r   r   pr   r   r   test_IPC  s   
r   )r-   ).__doc__rS   	sysconfigr   Zpyarrowr7   numpyr   Zimportorskipr   get_platformplatformZhas_ipc_supportmarkZskipifZcuda_ipcr   r   r   r   r   Zparametrizer'   r,   r<   rK   rR   rW   ra   rb   rg   ru   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   sl   









?1
+3$	