o
    Zh/"                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZmZm	Z	m
Z
mZmZ d dlZd dlmZ d dlmZ d dlmZmZmZ d dlmZ G d	d
 d
eeeZG dd deZG dd deeZdS )    N)ABC)datetime)AnyCallableDictIteratorListLiteralOptionalUnionDocumentget_from_dict_or_env)	BaseModelfield_validatormodel_validator)
BaseLoaderc                   @   sn   e Zd ZU dZeed< 	 eed< 	 dZeed< 	 eddede	d	e
fd
dZed	e	eef fddZdS )BaseGitHubLoaderz Load `GitHub` repository Issues.repoaccess_tokenzhttps://api.github.comgithub_api_urlbefore)modevaluesreturnc                 C   s   t |dd|d< |S )z1Validate that access token exists in environment.r   ZGITHUB_PERSONAL_ACCESS_TOKENr   )clsr    r   b/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/document_loaders/github.pyvalidate_environment   s   z%BaseGitHubLoader.validate_environmentc                 C   s   dd| j  dS )Nzapplication/vnd.github+jsonzBearer )AcceptAuthorization)r   selfr   r   r   headers!   s   
zBaseGitHubLoader.headersN)__name__
__module____qualname____doc__str__annotations__r   r   classmethodr   r   r   propertyr$   r   r   r   r   r      s   
 r   c                   @   sn  e Zd ZU dZdZeed< 	 dZee	e
d df ed< 	 dZee
d  ed< 	 dZee ed	< 	 dZee ed
< 	 dZee ed< 	 dZeee  ed< 	 dZee
d  ed< 	 dZee
d  ed< 	 dZee ed< 	 dZee	 ed< 	 dZee	 ed< 	 ededee dee fddZdee fddZdedefddZe defddZ!e defdd Z"dS )!GitHubIssuesLoaderz#Load issues of a GitHub repository.Tinclude_prsN)*none	milestone)openclosedallstateassigneecreator	mentionedlabels)createdupdatedcommentssort)ascZdesc	directionsincepageper_pagevr   c                 C   s8   |rz	t |d W |S  ty   td| w |S )Nz%Y-%m-%dT%H:%M:%SZz\Invalid value for 'since'. Expected a date string in YYYY-MM-DDTHH:MM:SSZ format. Received: )r   strptime
ValueError)r   rC   r   r   r   validate_sinceL   s   z!GitHubIssuesLoader.validate_sincec                 c   s    | j }|rItj|| jd}|  | }|D ]}| |}| js(|jd r(q|V  q|j	rC|j	drC| j
sC| jsC|j	d d }nd}|sdS dS )a  
        Get issues of a GitHub repository.

        Returns:
            A list of Documents with attributes:
                - page_content
                - metadata
                    - url
                    - title
                    - creator
                    - created_at
                    - last_update_time
                    - closed_time
                    - number of comments
                    - state
                    - labels
                    - assignee
                    - assignees
                    - milestone
                    - locked
                    - number
                    - is_pull_request
        r$   is_pull_requestnexturlN)rJ   requestsgetr$   raise_for_statusjsonparse_issuer.   metadatalinksrA   rB   )r#   rJ   responseZissuesissuedocr   r   r   	lazy_loadY   s,   

zGitHubIssuesLoader.lazy_loadrS   c                 C   s   |d |d |d d |d |d |d dd	 |d
 D |d r&|d d nd|d r1|d d nd|d |d d|v d}|d durH|d nd}t ||dS )z5Create Document objects from a list of GitHub issues.Zhtml_urltitleuserlogin
created_atr<   r5   c                 S   s   g | ]}|d  qS )namer   ).0labelr   r   r   
<listcomp>   s    z2GitHubIssuesLoader.parse_issue.<locals>.<listcomp>r9   r6   Nr1   lockednumberZpull_request)rJ   rV   r7   rY   r<   r5   r9   r6   r1   r^   r_   rH   body Zpage_contentrP   r   )r#   rS   rP   contentr   r   r   rO      s   
zGitHubIssuesLoader.parse_issuec                 C   sh   | j r	d| j n| j }| j| j| j| j| j|| j| j| j	| j
| jd}dd | D }d|}|S )z'Create query parameters for GitHub API.,)r1   r5   r6   r7   r8   r9   r=   r?   r@   rA   rB   c                 S   s&   g | ]\}}|d ur| d| qS )N=r   )r[   krC   r   r   r   r]      s     z3GitHubIssuesLoader.query_params.<locals>.<listcomp>&)r9   joinr1   r5   r6   r7   r8   r=   r?   r@   rA   rB   items)r#   r9   Zquery_params_dictZquery_params_listquery_paramsr   r   r   rj      s$   
zGitHubIssuesLoader.query_paramsc                 C   s   | j  d| j d| j S )zCreate URL for GitHub API./repos/z/issues?)r   r   rj   r"   r   r   r   rJ      s   zGitHubIssuesLoader.url)#r%   r&   r'   r(   r.   boolr*   r1   r   intr	   r5   r
   r6   r)   r7   r8   r9   r   r=   r?   r@   rA   rB   r   r+   rF   r   r   rU   dictrO   r,   rj   rJ   r   r   r   r   r-   )   sF   
 +r-   c                   @   sj   e Zd ZU dZdZeed< eeege	f  ed< de
e fddZdedefd	d
Zdee fddZdS )GithubFileLoaderzLoad GitHub Filemainbranchfile_filterr   c                    sT    j  d j d j d}tj| jd}|  | d }	  fdd|D S )Nrk   z/git/trees/z?recursive=1rG   treec                    s$   g | ]} j r  |d  r|qS )path)rr   )r[   fr"   r   r   r]      s    z3GithubFileLoader.get_file_paths.<locals>.<listcomp>)r   r   rq   rK   rL   r$   rM   rN   )r#   base_urlrR   	all_filesr   r"   r   get_file_paths   s   
	zGithubFileLoader.get_file_pathsrt   c                 C   sx   | j r	d| j  nd}| j d| j d| | }tj|| jd}|  t| t	r:| d }t
|dS dS )Nz?ref=ra   rk   z
/contents/rG   rc   zutf-8)rq   r   r   rK   rL   r$   rM   
isinstancerN   rn   base64	b64decodedecode)r#   rt   Zqueryparamsrv   rR   Zcontent_encodedr   r   r   get_file_content_by_path   s   z)GithubFileLoader.get_file_content_by_pathc                 c   s|    |   }|D ]4}| |d }|dkrq|d |d | j d| j d|d  d| j d|d  	d}t||dV  qd S )Nrt   ra   sha/type)rt   r~   sourcerb   )rx   r}   r   r   rq   r   )r#   filesfilerc   rP   r   r   r   rU      s    zGithubFileLoader.lazy_loadN)r%   r&   r'   r(   rq   r)   r*   r
   r   rl   r   r   rx   r}   r   r   rU   r   r   r   r   ro      s   
 ro   )rz   abcr   r   typingr   r   r   r   r   r	   r
   r   rK   Zlangchain_core.documentsr   Zlangchain_core.utilsr   Zpydanticr   r   r   Z)langchain_community.document_loaders.baser   r   r-   ro   r   r   r   r   <module>   s    ( 