o ZhIH@sdZddlmZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZddlmZmZddlmZdd lmZdd lmZdd lmZdd lmZeeZd gZ eGdddZ!ddhZ"hdZ#dddZ$GdddeZ%GdddeZ&eGdddeZ'dS) zLoads YouTube transcript.) annotationsN)Enum)Path)AnyDict GeneratorListOptionalSequenceUnion)parse_qsurlparse) ParseError)Document)model_validator) dataclass) BaseLoaderz0https://www.googleapis.com/auth/youtube.readonlyc@seZdZUdZeddZded<eddZded<eddZ ded<dd d Z e d de dddZ dddZdS)GoogleApiClientaGeneric Google API Client. To use, you should have the ``google_auth_oauthlib,youtube_transcript_api,google`` python package installed. As the google api expects credentials you need to set up a google account and register your Service. "https://developers.google.com/docs/api/quickstart/python" *Security Note*: Note that parsing of the transcripts relies on the standard xml library but the input is viewed as trusted in this case. Example: .. code-block:: python from langchain_community.document_loaders import GoogleApiClient google_api_client = GoogleApiClient( service_account_path=Path("path_to_your_sec_file.json") ) z .credentialszcredentials.jsonrcredentials_pathservice_account_pathz token.json token_pathreturnNonecCs||_dSN)_load_credentialscredsselfrc/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/document_loaders/youtube.py __post_init__2szGoogleApiClient.__post_init__beforemodevaluesrcC&|jds|jdstd|jS)DValidate that either folder_id or document_ids is set, but not both.rr-Must specify either channel_name or video_idskwargsget ValueErrorclsr$rrr#validate_channel_or_videoIds_is_set5s z3GoogleApiClient.validate_channel_or_videoIds_is_setc Csz ddlm}ddlm}ddlm}ddlm}ddlm }Wn t y+t dwd}|j r<|j t|j S|j rJ|t|jt}|rO|js|r^|jr^|jr^||n|t|jt}|jdd }t|jd }||Wd|S1swY|S) zLoad credentials.r)Request)service_account) Credentials)InstalledAppFlowYouTubeTranscriptApiYou must run`pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib youtube-transcript-api` to use the Google Drive loaderN)portw)Zgoogle.auth.transport.requestsr/Z google.oauth2r0Zgoogle.oauth2.credentialsr1Zgoogle_auth_oauthlib.flowr2youtube_transcript_apir4 ImportErrorrexistsZfrom_service_account_filestrrZfrom_authorized_user_fileSCOPESZvalidZexpiredZ refresh_tokenrefreshZfrom_client_secrets_filerZrun_local_serveropenwriteto_json) rr/r0r1r2r4rZflowtokenrrrr@s>           z!GoogleApiClient._load_credentialsNrrr$rrr)rr)__name__ __module__ __qualname____doc__rhomer__annotations__rrr r classmethodr.rrrrrrs   rhttphttps>z youtube.comzwww.youtube.comz m.youtube.comzvid.pluszwww.youtube-nocookie.comzyoutu.beurlr;r Optional[str]cCst|}|jtvr dS|jtvrdS|j}|dr7|j}t|}d|vr5|d}t |t r0|n|d}ndS|j d}| dd}t |dkrLdS|S)zEParse a YouTube URL and return the video ID if valid, otherwise None.Nz/watchvr/ )r schemeALLOWED_SCHEMESnetlocALLOWED_NETLOCSpathendswithqueryr isinstancer;lstripsplitlen)rM parsed_urlrWrYZ parsed_queryZidsvideo_idrrr_parse_video_idts$     r`c@seZdZdZdZdZdZdS)TranscriptFormatz3Output formats of transcripts from `YoutubeLoader`.textlineschunksN)rDrErFrGTEXTLINESCHUNKSrrrrras rac@sneZdZdZdddejddfd-ddZed.ddZe d/ddZ d0d!d"Z d1d%d&Z d2d(d)Z d3d+d,ZdS)4 YoutubeLoaderz!Load `YouTube` video transcripts.FenNxr_r;add_video_infoboollanguageUnion[str, Sequence[str]] translationrNtranscript_formatracontinue_on_failurechunk_size_secondsintcCsR||_d|i|_||_||_t|tr|g|_n||_||_||_||_||_ dS)z!Initialize with YouTube video ID.sourceN) r_ _metadatarkrmrZr;rorprqrr)rr_rkrmrorprqrrrrr__init__s    zYoutubeLoader.__init__ youtube_urlrcCs t|}|std|d|S)z*Extract video ID from common YouTube URLs.z.Could not determine the video ID for the URL "z".)r`r+)rwr_rrrextract_video_ids  zYoutubeLoader.extract_video_idr)rcKs||}||fi|S)z|Given a YouTube URL, construct a loader. See `YoutubeLoader()` constructor for a list of keyword arguments. )rx)r-rwr)r_rrrfrom_youtube_urls zYoutubeLoader.from_youtube_url chunk_pieces List[Dict]chunk_start_secondsrc Cspt|d\}}t|d\}}tdtdd|i|j||dd|dd|dd|jd|d d d S) z0Create Document from chunk of transcript pieces.< cS|ddSNrbr~strip)Z chunk_piecerrrz4YoutubeLoader._make_chunk_document..02d: https://www.youtube.com/watch?v=z&t=s)Z start_secondsZstart_timestamprt page_contentmetadata)divmodrjoinmaprur_)rrzr|mrhrrr_make_chunk_documents   z"YoutubeLoader._make_chunk_documenttranscript_piecesGenerator[Document, None, None]ccsg}d}|j}|D]%}|d|d}||kr*|r!|||Vg}|}||j7}||q t|dkr?|||VdSdS)Nrstartduration)rrrappendr])rrrzr|Zchunk_time_limittranscript_pieceZ piece_endrrr_get_transcript_chunkss    z$YoutubeLoader._get_transcript_chunksList[Document]cCs0z ddlm}m}m}Wn tytdw|jr%|}|j|z| |j }Wn |y8gYSwz| |j }Wn|yP| dg}Ynw|j dur\||j }|}|jtjkrxdtdd|}t||jd gS|jtjkrttd d|S|jtjkrt||Std ) z1Load YouTube transcripts into `Document` objects.r)NoTranscriptFoundTranscriptsDisabledr4zvCould not import "youtube_transcript_api" Python package. Please install it with `pip install youtube-transcript-api`.riNr~cSrrrrrrrrrz$YoutubeLoader.load..rcSs(t|ddttdd|dS)Nrbr~cSs |ddkS)Nrrbr)itemrrrr"s z6YoutubeLoader.load....r)rrdictfilteritemsrrrrrs  zUnknown transcript format.)r8rrr4r9rk_get_video_inforuupdatelist_transcriptsr_find_transcriptrmro translatefetchrprarerrrrflistrgrr+)rrrr4 video_infotranscript_list transcriptrrrrloadsR         zYoutubeLoader.loadrcCszddlm}Wn tytdw|d|j}|jp d|jp$d|jp(d|jp,d|jr5|j dnd|j p:d|j p>dd}|S)zGet important video information. Components include: - title - description - thumbnail URL, - publish_date - channel author - and more. r)YouTubezVCould not import "pytube" Python package. Please install it with `pip install pytube`.rUnknownz%Y-%m-%d %H:%M:%S)title descriptionZ view_count thumbnail_url publish_datelengthauthor) Zpytuberr9r_rrZviewsrrstrftimerr)rrZytrrrrr/s&   zYoutubeLoader._get_video_info)r_r;rkrlrmrnrorNrprarqrlrrrs)rwr;rr;)rwr;r)rrrh)rzr{r|rsrr)rr{rrrr)rr)rDrErFrGrarerv staticmethodrxrJryrrrrrrrrrhs"      ?rhc@seZdZUdZded<dZded<dZded<d Zd ed <d Zd ed<dZ d ed<d0ddZ d1ddZ e dde d2ddZd3dd Zd4d#d$Zd5d%d&Zd6d(d)Zd7d,d-Zd8d.d/ZdS)9GoogleApiYoutubeLoaderaLoad all Videos from a `YouTube` Channel. To use, you should have the ``googleapiclient,youtube_transcript_api`` python package installed. As the service needs a google_api_client, you first have to initialize the GoogleApiClient. Additionally you have to either provide a channel name or a list of videoids "https://developers.google.com/docs/api/quickstart/python" Example: .. code-block:: python from langchain_community.document_loaders import GoogleApiClient from langchain_community.document_loaders import GoogleApiYoutubeLoader google_api_client = GoogleApiClient( service_account_path=Path("path_to_your_sec_file.json") ) loader = GoogleApiYoutubeLoader( google_api_client=google_api_client, channel_name = "CodeAesthetic" ) load.load() rgoogle_api_clientNrN channel_namezOptional[List[str]] video_idsTrlrkrir;captions_languageFrqrrcCs||jj|_dSr)_build_youtube_clientrryoutube_clientrrrrr vsz$GoogleApiYoutubeLoader.__post_init__rrcCsBzddlm}ddlm}Wn tytdw|dd|dS)Nr)buildr3r5ZyoutubeZv3) credentials)Zgoogleapiclient.discoveryrr8r4r9)rrrr4rrrrys   z,GoogleApiYoutubeLoader._build_youtube_clientr!r"r$cCr%)r&rrr'r(r,rrrr.sz:GoogleApiYoutubeLoader.validate_channel_or_videoIds_is_setr_cCsrddlm}m}||}z ||jg}Wn|y*|D]}||j}qYnw|}ddd|DS)Nr)rr4r~cSsg|] }|ddqS)rbr~r).0trrr szGGoogleApiYoutubeLoader._get_transcripe_for_video_id..) r8rr4rrrrrr)rr_rr4rrZavailable_transcriptrrrr_get_transcripe_for_video_ids   z3GoogleApiYoutubeLoader._get_transcripe_for_video_idr)rcKs8||}|jjd|d}t||dddS)N id,snippetpartidrrr)rrZvideosrexecuterr*)rr_r)ZcaptionsZvideo_responserrr_get_document_for_video_ids  z1GoogleApiYoutubeLoader._get_document_for_video_idcCs8|jjd|ddd}|}|dddd}|S)Nrchannel)rqtype maxResultsrrZ channelId)rsearchrr)rrrequestresponse channel_idrrr_get_channel_ids z&GoogleApiYoutubeLoader._get_channel_idrcCs4|jjd|d}|}|dddddS)NZcontentDetailsrrrZrelatedPlaylistsZuploads)rZchannelsrr)rrrrrrr_get_uploads_playlist_ids z/GoogleApiYoutubeLoader._get_uploads_playlist_idrrc KsHz ddlm}m}Wn tytdw||}||}|jjd|dd}g}|dur| } | dD][} | dd d } d | i} |j rX| d d | | dz| | } |t| | d Wq9||tfy}z|jrtd d| dd d|n|WYd}~q9d}~ww|j|| }|dus1|S)Nr)rrzTYou must run`pip install --upgrade youtube-transcript-api` to use the youtube loaderr2)rZ playlistIdrrZsnippetZ resourceIdZvideoIdZ thumbnailsrzError fetching transscript r~rz , exception: )r8rrr9rrrZ playlistItemsrrrkpoprrrrrrqloggererrorrZ list_next)rrr)rrrZuploads_playlist_idrrrrr_Z meta_datarerrr_get_document_for_channelsZ        z0GoogleApiYoutubeLoader._get_document_for_channelcsLg}jr|j|Sjr"|fddjD|Std)zLoad documents.csg|]}|qSr)r)rr_rrrrsz/GoogleApiYoutubeLoader.load..r')rextendrrr+)rZ document_listrrrrs  zGoogleApiYoutubeLoader.loadrB)rrrrrC)r_r;rr;)r_r;r)rrr)rr;rr;)rr;rr;)rr;r)rrrr)rDrErFrGrIrrrkrrqr rrrJr.rrrrrrrrrrrQs&            5r)rMr;rrN)(rG __future__rloggingenumrpathlibrtypingrrrrr r r urllib.parser r Zxml.etree.ElementTreerZlangchain_core.documentsrZpydanticrZpydantic.dataclassesrZ)langchain_community.document_loaders.baser getLoggerrDrr<rrTrVr`rarhrrrrrs0   $      Q 8