o ZhT @sHdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZddlmZmZddlmZddlmZmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%e rxddl&Z'ddl(m)m*m+Z,e-e.Z/d Z0d Z1d Z2d+ddZ3GdddZ4e4Z5e4Z6Gddde7e Z8Gddde!Z9Gddde!Z:GdddZ;dee7deddd e7fd d!Z?d"edd ee7ddffd#d$Z@eeAe4fZBe7ZCGd%d&d&ZDeDZEe7ZFGd'd(d(e9e:eeEeFfZGee7eefZHeAZIGd)d*d*e9e:eeHeIfZJeGZKeJZLeDZMdS),z*A common module for NVIDIA Riva Runnables.N)Enum) TYPE_CHECKINGAnyAsyncGenerator AsyncIteratorDict GeneratorIteratorListOptionalTupleUnioncast) AnyMessage BaseMessage) PromptValue)RunnableConfigRunnableSerializable) AnyHttpUrl BaseModelField parse_obj_asroot_validator validator?i) .!?¡¿return riva.clientc Cs4zddl}W|jSty}ztd|d}~ww)z5Import the riva client and raise an error on failure.rNziCould not import the NVIDIA Riva client library. Please install it with `pip install nvidia-riva-client`.) riva.client ImportErrorclient)rivaerrr(`/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/utilities/nvidia_riva.py_import_riva_client1s r*c@seZdZdZdS) SentinelTzAn empty Sentinel type.N)__name__ __module__ __qualname____doc__r(r(r(r)r+>sr+c@sLeZdZdZdZdZdZdZdZdZ e de d dfd d Z e dd dZdS)RivaAudioEncodingaAn enum of the possible choices for Riva audio encoding. The list of types exposed by the Riva GRPC Protobuf files can be found with the following commands: ```python import riva.client print(riva.client.AudioEncoding.keys()) # noqa: T201 ``` ALAWENCODING_UNSPECIFIEDFLAC LINEAR_PCMMULAWOGGOPUS format_coder!c CsBz |j|j|jd|WSty }ztd||d}~ww)zReturn the audio encoding specified by the format code in the wave file. ref: https://mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html )z>The following wave file format code is not supported by Riva: N)r4r1r5KeyErrorNotImplementedError)clsr7r'r(r(r)from_wave_format_codeXsz'RivaAudioEncoding.from_wave_format_coderiva.client.AudioEncodingcCst}t|j|S)z-Returns the Riva API object for the encoding.)r*getattrZ AudioEncodingself riva_clientr(r(r)riva_pb2fs zRivaAudioEncoding.riva_pb2N)r!r?)r,r-r.r/r1r2r3r4r5r6 classmethodintr>propertyrDr(r(r(r)r0Fs  r0c@seZdZUdZeeddddgdZeeefe d<eddd Z e ee d <e dd dZ eddddeded efddZdS) RivaAuthMixinzBConfiguration for the authentication to a Riva service connection.zhttp://localhost:50051z1The full URL where the Riva service can be found.z"https://user@pass:riva.example.com) descriptionZexamplesurlNz@A full path to the file where Riva's public ssl key can be read.rIssl_certr!riva.client.AuthcCsBt}tt|j}|jdk}t|jdd}|j|j||dS)z!Return a riva client auth object.https/)rLuse_ssluri) r*rrrJschemestrsplitZAuthrL)rBrCrJrQZ url_no_schemer(r(r)authzs  zRivaAuthMixin.authT)preZ allow_reusevalcCs$t|tr tttt|Stt|S)z:Do some initial conversations for the URL before checking.) isinstancerTrrr)r=rXr(r(r) _validate_urls  zRivaAuthMixin._validate_url)r!rM)r,r-r.r/rrrJr rT__annotations__rLr rGrVrrErrZr(r(r(r)rHms   rHc@sPeZdZUdZeejddZeed<edddZ e ed<edd dZ e ed <d S) RivaCommonConfigMixinz%A collection of common Riva settings.z!The encoding on the audio stream.)defaultrIencodingi@z*The sample rate frequency of audio stream.sample_rate_hertzzen-USzaThe [BCP-47 language code](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) for the target language. language_codeN) r,r-r.r/rr0r4r^r[r_rFr`rTr(r(r(r)r\s r\c@sfeZdZUdZejed<ejed<dddZddd Z dd d Z de fd d Z dddZ dddZdS)_Eventz3A combined event that is threadsafe and async safe._event_aeventr!NcCst|_t|_dS)zInitialize the event.N) threadingEventrbasynciorcrBr(r(r)__init__ z_Event.__init__cC|j|jdSzSet the event.N)rbsetrcrgr(r(r)rlriz _Event.setcCrjrk)rbclearrcrgr(r(r)rmriz _Event.clearcC |jS)zIndicate if the event is set.)rbis_setrgr(r(r)ros z _Event.is_setcCs|jdS)zWait for the event to be set.N)rbwaitrgr(r(r)rpsz _Event.waitcs|jIdHdS)z#Async wait for the event to be set.N)rcrprgr(r(r) async_waitsz_Event.async_waitr!N)r,r-r.r/rdrer[rfrhrlrmboolrorprqr(r(r(r)ras       raoutput_directory sample_ratecCsr|r7tjddd|d }|j}Wdn1swYt|d}|d|d||||fSd S) zECreate a new wave file and return the wave write object and filename.bxz.wavF)modesuffixdeletedirNwbr8rP)NN)tempfileNamedTemporaryFilenamewaveopenZ setnchannelsZ setsampwidthZ setframerate)rtruf wav_file_namewav_filer(r(r) _mk_wave_files    rrX TTSInputTypecCs.t|tr |St|trt|jSt|S)zAttempt to coerce the input value to a string. This is particularly useful for converting LangChain message to strings. )rYrZ to_stringrrTcontentrXr(r(r)_coerce_strings   rinputsccsd}|D]D}t|}tD]}||vr(||d\}}|||Vd}||vsq ||7}t|tkrItdt|tD] }|||dVq;d}q|rQ|VdSdS)z9Filter the input chunks are return strings ready for TTS.r8rN)r_SENTENCE_TERMINATORSrUlen_MAX_TEXT_LENGTHrange)rbufferchunk terminatorZ last_sentenceidxr(r(r)_process_chunkss(  rc@sZeZdZUdZejed<ejed<ejed<e ed<e ed<e ed<e ej ed<d)d e d d fd dZ d eed d ffddZd eefddZed efddZed efddZed efddZed efddZd*dede e d d fddZd*dede e d d fdd Zd*de e d d fd!d"Zd*de e d d fd#d$Zd%ed&d d fd'd(Zd S)+ AudioStreamz%A message containing streaming audio. _put_lock_queueoutputhangup user_talking user_quiet_workerrmaxsizer!NcCsDt|_tj|d|_t|_t|_t|_ t|_ d|_ dS)zInitialize the queue.)rN) rdLockrqueueQueuerrrarrrr)rBrr(r(r)rh s   zAudioStream.__init__ccsJ z |jdt}Wn tjyYqw|tkrdS|V|jq)zReturn an error.TN)rget_QUEUE_GET_TIMEOUTrEmptyHANGUP task_donerBZnext_valr(r(r)__iter__s zAudioStream.__iter__cCsZ ztd|jjdtIdH}Wn tjyYqw|tkr$dS|V|j q)z4Iterate through all items in the queue until HANGUP.TN) rfget_event_looprun_in_executorrrrrrrrrr(r(r) __aiter__&s  zAudioStream.__aiter__cCrn)z(Indicate if the audio stream has hungup.)rrorgr(r(r)hungup9 zAudioStream.hungupcCrn)z-Indicate in the input stream buffer is empty.)remptyrgr(r(r)r>rzAudioStream.emptycCs4|jo|j}|jduo|j o|j}|o|S)z;Indicate if the audio stream has hungup and been processed.N)rrris_aliver)rBZ input_doneZ output_doner(r(r)completeCs   zAudioStream.completecCs|jr|jSdS)z&Indicate if the ASR stream is running.F)rrrgr(r(r)runningNs zAudioStream.runningitemtimeoutcCs\|j!|jr td|tur|j|jj||dWddS1s'wYdS)zPut a new item into the queue.z?The audio stream has already been hungup. Cannot put more data.rN)rr RuntimeErrorrrrlrput)rBrrr(r(r)rUs "zAudioStream.putcs,t}t|d|j||IdHdS)z$Async put a new item into the queue.N)rfrwait_forrr)rBrrloopr(r(r)aput`s"zAudioStream.aputcCs|t|dS)zSend the hangup signal.N)rrrBrr(r(r)closeeszAudioStream.closecs|t|IdHdS)zAsync send the hangup signal.N)rrrr(r(r)acloseizAudioStream.aclose responseszrasr.StreamingRecognizeResponsecsZjrtdtjdddd fdd }tj|d _d j_jdS) zIDrain the responses from the provided iterator and put them into a queue.z,An ASR instance has already been registered.rPrrr!NcsD]<}|js q|jD]2}|jsq|jr2jjtt |jdj }j |qj sAjjqqdS)zConsume the ASR Generator.rN)rpresultsZ alternativesis_finalrrmrrlrrT transcriptrrro)responseresultrZ has_startedrrBr(r)workerts$      z$AudioStream.register..worker)targetTrr) rrrdBarrierThreadrdaemonstartrp)rBrrr(rr)registerms  zAudioStream.register)rN) r,r-r.r/rdrr[rrrar rrFrhrbytesrrStreamInputTyperrGrsrrrrrrrrr rr(r(r(r)rs2       rc @seZdZUdZdZeed<dZeed<edddZ e ed <ed d dZ e ed <ed d dZ e ed<ed dedeeefdeeeffddZed ddZd!ddZ d"dedeededefddZdS)#RivaASRzNA runnable that performs Automatic Speech Recognition (ASR) using NVIDIA Riva.Znvidia_riva_asrr~zA Runnable for converting audio bytes to a string.This is useful for feeding an audio stream into a chain andpreprocessing that audio to create an LLM prompt.rIr8z7The number of audio channels in the input audio stream.rKaudio_channel_countTz\Controls whether or not Riva should attempt to filter profanity out of the transcribed text.profanity_filterz]Controls whether Riva should attempt to correct senetence puncuation in the transcribed text.enable_automatic_punctuationrWvaluesr!cC t}|Sz4Validate the Python environment and input arguments.r*r=r_r(r(r)_validate_environmentzRivaASR._validate_environment&riva.client.StreamingRecognitionConfigc Cs4t}|jd|j|j|j|jd|j|j|jddS)z)Create and return the riva config object.Tr8)r^r_rZmax_alternativesrrr`)Zinterim_resultsconfig) r*ZStreamingRecognitionConfigZRecognitionConfigr^r_rrrr`rAr(r(r)rszRivaASR.configriva.client.ASRServicec C8t}z||jWSty}ztd|d}~ww);Connect to the riva service and return the a client object.z5Error raised while connecting to the Riva ASR server.N)r*Z ASRServicerV Exception ValueErrorrBrCr'r(r(r) _get_servicezRivaASR._get_serviceNinputrkwargscKs|js|}|j||jd}||g}|jsl|jj|jjd}Wdn1s/wY|ri|j sZz ||j g7}Wn t j yOYq6w|j |j r;tdt|d|S|jrdS)z3Transcribe the audio bytes into a string with Riva.)Z audio_chunksZstreaming_configg?NzRiva ASR returning: %s r)rrZstreaming_response_generatorrrrr not_emptyrpr get_nowaitrrr_LOGGERdebugreprjoinstrip)rBrrrservicerZ full_responsereadyr(r(r)invokes2     zRivaASR.invoke)r!r)r!rr)r,r-r.r/r~rTr[rIrrrFrrsrrrErrrrGrr ASRInputTyper r ASROutputTyperr(r(r(r)rs@   &   rc @s0eZdZUdZdZeed<dZeed<edddZ eed <ed d dZ e eed <e d de deeefdeeeffddZed e dedefddZd"ddZ d#dede ededefddZ d#deede ede edeefddZ d#deede ede edeed ffd d!Zd S)$RivaTTSz?A runnable that performs Text-to-Speech (TTS) with NVIDIA Riva.Znvidia_riva_ttsr~z_A tool for converting text to speech.This is useful for converting LLM output into audio bytes.rIzEnglish-US.Female-1zThe voice model in Riva to use for speech. Pre-trained models are documented in [the Riva documentation](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/tts/tts-overview.html).rK voice_nameNzThe directory where all audio files should be saved. A null value indicates that wave files should not be saved. This is useful for debugging purposes.rtTrrr!cCrrrrr(r(r)r rzRivaTTS._validate_environmentvcCs,|rt|}|jdddt|S|S)NT)parentsexist_ok)pathlibPathmkdirrTabsolute)r=rdirpathr(r(r)_output_directory_validator's   z#RivaTTS._output_directory_validator"riva.client.SpeechSynthesisServicec Cr)rz5Error raised while connecting to the Riva TTS server.N)r*ZSpeechSynthesisServicerVrrrr(r(r)r0rzRivaTTS._get_servicerrrcKsd|t|gS)zDPerform TTS by taking a string and outputting the entire audio file.)r transformiter)rBrrrr(r(r)r:szRivaTTS.invokec ks|}t|j|j\}}t|D],}td||j||j|j |j j |jd}|D]} t t | j} |r:|| | Vq+q|rM|td|dSdS)zHPerform TTS by taking a stream of characters and streaming output bytes.zRiva TTS chunk: %s)textrr`r^Zsample_rate_hzzRiva TTS wrote file: %sN)rrrtr_rrrZsynthesize_onlinerr`r^rDrraudioZwriteframesrawr) rBrrrrrrrrresprr(r(r)rCs0    zRivaTTS.transformc  stttd fdd }dttffdd d fdd d fd d }|}|} z td IdH}Wn tj j yVYq=w |t ur`n|Vq>|IdH|IdHdS)zGIntercept async transforms and route them to the synchronous transform.r!Ncs.2z 3dHW}|q6tdS)z#Produce input into the input queue.N) put_nowait_TRANSFORM_ENDr)r input_queuer(r) _producerus  z%RivaTTS.atransform.._producerc3s> zjdd}Wn tjyYqw|tkrdS|Vq)zIterate over the input_queue.TrrN)rrrrr)rr(r)_input_iterator{sz+RivaTTS.atransform.._input_iteratorcs*D]}|qtdS)z!Consume the input with transform.N)rrrr)r  out_queuerBr(r) _consumers z%RivaTTS.atransform.._consumercsdIdHdS)z"Coroutine that wraps the consumer.N)rr()r rr(r)_consumer_cororz*RivaTTS.atransform.._consumer_coroTrrr) rfget_running_looprrr r create_taskrr exceptions TimeoutErrorrr) rBrrrr r ZproducerZconsumerrXr()r r rrrr rBr) atransformjs.    zRivaTTS.atransform)r!rr)r,r-r.r/r~rTr[rIrrrtr rrErrrrrrrr TTSOutputTyperr rrrrr(r(r(r)rsd     &     * r)r!r")Nr/rfloggingrrr|rdrenumrtypingrrrrrrr r r r r rZlangchain_core.messagesrrZlangchain_core.prompt_valuesrZlangchain_core.runnablesrrZpydanticrrrrrrr#r&Zriva.client.proto.riva_asr_pb2r%protoZ riva_asr_pb2Zrasr getLoggerr,rrrrr*r+rrrTr0rHr\rafloatZ Wave_writerrrrrZStreamOutputTyperrrrrrrZ NVIDIARivaASRZ NVIDIARivaTTSZNVIDIARivaStreamr(r(r(r)sx 8      '!"    h  $