o
    Zh                     @   sH   d dl mZmZmZ d dlZd dlmZ d dlmZ G dd deZ	dS )    )IteratorListUnionN)Document)
BaseLoaderc                   @   sH   e Zd ZdZ	ddedeeee f defddZde	e
 fd	d
ZdS )BrowserlessLoaderz3Load webpages with `Browserless` /content endpoint.T	api_tokenurlstext_contentc                 C   s   || _ 	 || _	 || _dS )z0Initialize with API token and the URLs to scrapeN)r   r	   r
   )selfr   r	   r
    r   g/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/document_loaders/browserless.py__init__   s
   
zBrowserlessLoader.__init__returnc                 c   s    | j D ]C}| jr0tjdd| ji|ddigdd}t| d d d	 d d
 d|idV  qtjdd| jid|id}t|jd|idV  qdS )zLazy load Documents from URLs.z$https://chrome.browserless.io/scrapetokenselectorbody)urlelements)paramsjsondatar   resultstextsource)Zpage_contentmetadataz%https://chrome.browserless.io/contentr   N)r	   r
   requestspostr   r   r   r   )r   r   responser   r   r   	lazy_load   s>   



zBrowserlessLoader.lazy_loadN)T)__name__
__module____qualname____doc__strr   r   boolr   r   r   r   r   r   r   r   r   	   s    

r   )
typingr   r   r   r   Zlangchain_core.documentsr   Z)langchain_community.document_loaders.baser   r   r   r   r   r   <module>   s
    