o Zh,@sNddlZddlmZmZmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZdZd Zd Zd Zd Zd ZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"eeegZ#edgdedgdedgde!dgde"dgdedgdedgdedegdedegdedegde degdi Z$d efd!d"Z%Gd#d$d$e Z&dS)%N)AnyDictListOptional)UUID)BaseCallbackHandler) LLMResult guard_import) import_pandasZ token_usageZ total_tokensZ prompt_tokensZcompletion_tokensrun_id model_nameZgoodbadZneutralsuccessZfailureiipromptresponsecontextdurationZfeedback llm_statusfiddler z$123e4567-e89b-12d3-a456-426614174000returncCs tdddS)zLImport the fiddler python package and raise an error if it is not installed.rzfiddler-client)Zpip_namer rre/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/callbacks/fiddler_callback.pyimport_fiddler8s rcseZdZdedededededdf fdd Zedefd d Z d d ede ede dede ede e ee fddfddZde ee fde ede de fddZdede ddfddZdede ddfddZZS)!FiddlerCallbackHandlerurlorgprojectmodelapi_keyrNc stt|_t|_||_||_||_||_ ||_ |j t |_ i|_i|_i|_|jj|||d|_|j|jvrhtd|jdz |j|jWntyg}z td|jd|d}~ww|jjj|j dd}tt|jD]3}|j|jtkr|jjj|j|_ t!|j|_"qy|j|jt#kr|jjj|j|_ t$t%g|j|_"qy|j |j&|jvrO|j |j'|jvrtd |j d |jdz|jj(|j|j d |j i|d Wnty}ztd |j d|d|d}~ww|jj)j*|d |jj+j,t-t.t/gtt0t1t2t3t4t5g|j6d} td|j d |jdz|jj7|j|j |j | dWdStyN}ztd|j d|d|d}~wwdS)a Initialize Fiddler callback handler. Args: url: Fiddler URL (e.g. https://demo.fiddler.ai). Make sure to include the protocol (http/https). org: Fiddler organization id project: Fiddler project name to publish events to model: Fiddler model name to publish events to api_key: Fiddler authentication token )Zorg_idZ auth_tokenzadding project z.This only has to be done once.zError adding project z(:{e}. Fiddler integration will not work.Nr)Zmax_inferred_cardinalityzadding dataset z to project train) project_id dataset_idZdatasetinfozError adding dataset z: z#.Fiddler integration will not work.) dataset_infor%Z model_taskfeaturestargetZ metadata_colscustom_featuresz adding model )r$r%Zmodel_id model_infozError adding model )8super__init__rfdlr pdrrr r!r" DataFrame _dataset_dictZ_dfrun_id_promptsZrun_id_responserun_id_starttimeZ FiddlerApifiddler_clientZget_project_namesprintZ add_project ExceptionZ DatasetInfoZfrom_dataframerangelencolumnsnameFEEDBACKZDataTypeCATEGORYZ data_typeFEEDBACK_POSSIBLE_VALUESZpossible_values LLM_STATUSSUCCESSFAILUREZget_model_namesZget_dataset_namesZupload_datasetZ ModelInfoZfrom_dataset_infoZ ModelTaskZLLMPROMPTCONTEXTRESPONSERUN_ID TOTAL_TOKENS PROMPT_TOKENSCOMPLETION_TOKENS MODEL_NAMEDURATIONr*Z add_model) selfrrr r!r"er'ir+ __class__rrr-?s     zFiddlerCallbackHandler.__init__c Cs|jjddtgd|jjdtdd|jjddtgd|jjdtdd|jjdd ttgd gd id |jjd dttgd|jjddttgdgS)a[ Define custom features for the model to automatically enrich the data with. Here, we enable the following enrichments: - Automatic Embedding generation for prompt and response - Text Statistics such as: - Automated Readability Index - Coleman Liau Index - Dale Chall Readability Score - Difficult Words - Flesch Reading Ease - Flesch Kincaid Grade - Gunning Fog - Linsear Write Formula - PII - Personal Identifiable Information - Sentiment Analysis zPrompt EmbeddingZ embedding)r: enrichmentr9z Prompt CF)r:Z source_columncolumnzResponse Embeddingz Response CFzText StatisticsZtextstat statistics)Zautomated_readability_indexZcoleman_liau_indexZdale_chall_readability_scoreZdifficult_wordsZflesch_reading_easeZflesch_kincaid_gradeZ gunning_fogZlinsear_write_formula)r:rOr9configZPIIZpiiZ SentimentZ sentiment)r.Z EnrichmentrAZ TextEmbeddingrC)rJrrrr*sLz&FiddlerCallbackHandler.custom_featuresr prompt_responsesrrr token_usage_dictc Cst|j|}|jt|j|t|tt|g|t|g|t |g|t |g|i}|rC| D]\} } t | t r>| g|n| || <q0z*|jddkrX|j|j|j|WdS|jdd} |jj|j|j| ddWdSty} ztd| dWYd} ~ dSd} ~ ww) z+ Publish events to fiddler rrrecords)Zorient)eventz$Error publishing events to fiddler: z. continuing...N)r8r2r/r0rArCrDstrrIr>rHitems isinstanceintshaper4Zpublish_events_batchr r!to_dictZ publish_eventr6r5) rJr rTrrr rUZ prompt_countdfkeyvalueZdf_dictrKrrr_publish_eventss4         z&FiddlerCallbackHandler._publish_events serializedpromptskwargscKs,|t}||j|<ttd|j|<dS)N)rDr2r[timer3)rJrbrcrdr rrr on_llm_start&s z#FiddlerCallbackHandler.on_llm_startrc Ks|}|t}ttd|j|}d}i}t|jtr0dd|jD}|j t d}dd|D}| |||t ||dS)NrerScSs$i|]\}}|tttfvr||qSr)rErFrG).0kvrrr 5s z5FiddlerCallbackHandler.on_llm_end..cSsg|] }|jddjqS)r)Z generationstext)rhZ llmresultrrr <sz5FiddlerCallbackHandler.on_llm_end..) flattenrDr[rfr3rZZ llm_outputdictrYgetrHrar?) rJrrdZflattened_llmresultr Z run_durationr rUrTrrr on_llm_end-s* z!FiddlerCallbackHandler.on_llm_enderrorcKsF|t}ttd|j|}||dgt|j||tdS)NrerS)rDr[rfr3rar8r2r@)rJrrrdr rrrr on_llm_errorIs z#FiddlerCallbackHandler.on_llm_error)rSN)__name__ __module__ __qualname__rXr-propertylistr*rrr[rrrrargrrq BaseExceptionrs __classcell__rrrMrr>sXwK *  r)'rftypingrrrruuidrZlangchain_core.callbacksrZlangchain_core.outputsrZlangchain_core.utilsr Z#langchain_community.callbacks.utilsr Z TOKEN_USAGErErFrGrDrHZGOODBADZNEUTRALr?r@ZDEFAULT_MAX_TOKENZDEFAULT_MAX_DURATIONrArCrBrIr;r>r=r1rrrrrrsR