o Zh@s dZddlmZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZdd lmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l"m&Z'dd l"m(Z)ddl*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6ddl4m7Z8ddl9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKddlLmMZMmNZNmOZOddlPmQZRdd lSm&ZTddlSmUZUmVZVe rddlWZXeYeZZ[eegeeIe#ffeee\gefe#eIfZ]eegeeIe#ffefZ^Gddde_Z`Gd d!d!e\ZaGd"d#d#e\Zb $ddd+d,Zcdd/d0ZdGd1d2d2eGZedd4d5Zfdd;d<Zgdd?d@ZhddBdCZiddKdLZjddRdSZkddUdVZlddXdYZmddcddZnddfdgZoddidjZpdddddkddtduZqddddvddydzZrdd{dd~dZsddddvdddZtddddvdddZudd{dddZv   ddddZwGdddeGddZxejyGdddZzdddZ{dddZ|dZ}dddddddddddZ~dddddddddddZdZee_edde~_dS)z>Utilities for running language models or Chains over datasets.) annotationsN)datetimetimezone) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnioncast)warn_deprecated) Callbacks)BaseLanguageModel) BaseMessagemessages_from_dict) ChatResult LLMResult)RunnableRunnableConfigRunnableLambdaconfig)utils)EvaluatorCallbackHandlerwait_for_all_evaluators)LangChainTracer)Client) get_git_infoget_langchain_env_var_metadata)EvaluationResult RunEvaluator) run_evaluator) as_runnableis_traceable_function)DatasetDataTypeExampleRun TracerSession)LangSmithError) HTTPError) TypedDict)Chain)load_evaluator) EvaluatorTypePairwiseStringEvaluatorStringEvaluator) evaluation)name_generationprogressc@seZdZdZdS)InputFormatErrorz(Raised when the input format is invalid.N)__name__ __module__ __qualname____doc__r;r;^/var/www/html/lang_env/lib/python3.10/site-packages/langchain/smith/evaluation/runner_utils.pyr6Nsr6c@s$eZdZdZd ddZd ddZdS) TestResultz1A dictionary of the results of a single test run.return pd.DataFramecCs.|}dd|jD}|jddj|ddS)zReturn quantiles for the feedback scores. This method calculates and prints the quantiles for the feedback scores across all feedback keys. Returns: A DataFrame containing the quantiles for each feedback key. cSs6g|]}|ds|ds|dvs|dr|qS)inputs.outputs.>outputinput reference) startswith).0colr;r;r< esz5TestResult.get_aggregate_feedback..all)include)Zaxis) to_dataframecolumnsZdescribeZdrop)selfdfZto_dropr;r;r<get_aggregate_feedbackXs  z!TestResult.get_aggregate_feedbackc CsHzddl}Wnty}ztd|d}~wwg}g}|dD]z\}}|d}|d}t|tr>dd|D} n |durEi} nd|i} id d|d D| } d |vrxt|d trr| d d|d Dn|d | d <| id d|D|d|d|dd|| ||q"|j||dS)z#Convert the results to a dataframe.rNzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.resultsfeedbackrBcSi|] \}}d||qS)rAr;rFkvr;r;r< z+TestResult.to_dataframe..cSrS)r@r;rTr;r;r<rWrXrCrDcSrS)z reference.r;rTr;r;r<rWrXcSsi|] }d|j|jqS)z feedback.)keyZscore)rFfr;r;r<rWrXErrorexecution_timerun_id)errorr\r])index) pandas ImportErroritemsget isinstancedictupdateappendZ DataFrame) rNpdeindicesrecords example_idresultrRZoutput_rBrr;r;r<rLosV       zTestResult.to_dataframeN)r>r?)r7r8r9r:rPrLr;r;r;r<r=Us r=cs,eZdZdZdfdd Zdd d ZZS) EvalErrorz"Your architecture raised an error.r[ BaseExceptionkwargsrr>Nonec stjdd|i|dS)Nr[r;)super__init__)rNr[rq __class__r;r<rtszEvalError.__init__namestrcCs*z||WStytd|dw)Nz%'EvalError' object has no attribute '')KeyErrorAttributeError)rNrwr;r;r< __getattr__s   zEvalError.__getattr__)r[rprqrr>rr)rwrxr>r)r7r8r9r:rtr| __classcell__r;r;rur<rosro llm_or_chain_factoryMODEL_OR_CHAIN_FACTORY dataset_namerxr>MCFcsZt|tr)|jj}|jdur#jjj}td|d|d|dfddSt|tr0|St|tr=|fddSt|rt |rRt t t |fd dSz|}Wn&t y}t t |}t|}td |d t|fd dYSwt t |t|tr|St t t |rt t t |fd dSt|tsfddSS|S)zForgive the user if they pass in a chain without memory instead of a chain factory. It's a common mistake. Raise a more helpful error message as well.Na$Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called. This will safegaurd against information leakage between dataset examples. For example: def chain_constructor(): new_memory = z(...) return z*(memory=new_memory, ...) run_on_dataset("z", chain_constructor, ...)cSNr;r;)chainr;r<z(_wrap_in_chain_factory..crrr;r;)lcfr;r<rrcrrr;r; runnable_r;r<rrzWrapping function z as RunnableLambda.crrr;r;)wrappedr;r<rrcrrr;r;rr;r<rrcstSr)rr;) constructorr;r<r)rdr.rvr7Zmemory ValueErrorrrcallabler%r$r r TypeErrorinspect signatureloggerinfor)rrZ chain_classZ memory_class_modelZ user_funcsigr;)rrrrrr<_wrap_in_chain_factorysV                 rinputsDict[str, Any]cCs,|stdg}d|vr%t|dtstdt|dj|dg}n]d|vrLt|dtr;tdd|dDsGtdt|dj|d}n6t|dkr{tt | }t|trc|g}nt|trttd d|Drt|}ntd |td |t|dkr|d Std t|d)zGet prompt from inputs. Args: inputs: The input dictionary. Returns: A string prompt. Raises: InputFormatError: If the input format is invalid. Inputs should not be empty.promptz"Expected string for 'prompt', got promptscs|]}t|tVqdSrrdrxrFir;r;r<   z_get_prompt..z,Expected list of strings for 'prompts', got rKcsrrrrr;r;r<r z)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got rz)LLM Run expects single prompt input. Got z prompts.) r6rdrxtyper7listrIlennextitervalues)rrZprompt_r;r;r< _get_promptsD       rc@seZdZUdZded<dS)ChatModelInputzVInput for a chat model. Parameters: messages: List of chat messages. zList[BaseMessage]messagesNr7r8r9r:__annotations__r;r;r;r<rs  rrecCs|std|}d|vr|d|d<nt|dkr&tt||d<d|vrS|d}t|tr?t dd|Dr?|g}t|dkrOt |d|d<|Stdtd |) zGet Chat Messages from inputs. Args: inputs: The input dictionary. Returns: A list of chat messages. Raises: InputFormatError: If the input format is invalid. rrrCrKcsrr)rdrerr;r;r<r8rz _get_messages..rzGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got ) r6copypoprrrrrdrrIr)rZ input_copyZ raw_messagesr;r;r< _get_messages$s0   r first_exampler( input_mapperOptional[Callable[[Dict], Any]]rrc Cs|r+||j}t|ts't|trtdd|Ds)td|dt|ddSdSzt|jWdStyTz t|jWYdStyStd|jdww)NcsrrrdrrFmsgr;r;r<rTrz>_validate_example_inputs_for_language_model..zWhen using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages. Got: of type .zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.) rrdrxrrIr6rrr)rrZ prompt_inputr;r;r<+_validate_example_inputs_for_language_modelLs8     rrr.cCs|r2||j}t|j|}t|ts!td|dt|d|r0td|jd|dS|j}t|j|}t |dkrLt |jdkrLdS|r[td|jd|dS) zt|trd\}}d}nd}|}t|tr|jnd}t|tr%|jnd}t||||djr7t|djnd||}|Sd}|S)zt|tr|St|ttfr!t|tst|}t||d}|j} nBt|tjrRd|i|} t|j fi| }|j j} t|tj rQ|j pF|}|j pK|}|j pP|}nt|rZt|Stdt|t|tr|jrz|durztd| d|dtjj||||||| gd} | St|trtd| d td| d ) N)rrzUnknown evaluator type: zPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r)rrrtagszRun evaluator for z is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead? See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)rdr"r0rxr/valuesmith_eval_configZ EvalConfigZ get_kwargsZevaluator_typeZSingleKeyEvalConfigrrrrrun_evaluator_decrrr2Zrequires_reference smith_evalStringRunEvaluatorChainfrom_run_and_data_typer1NotImplementedError) rrrrrrrrZ evaluator_Z eval_type_tagrqr#r;r;r<_construct_run_evaluatorsZ            r2Tuple[Optional[str], Optional[str], Optional[str]]cCs(t||}t||}t||}|||fSr)rrr)rrrrrrrr;r;r< _get_keysHs    rList[RunEvaluator]c Csg}d\}}} |js|jr!tdd|jDr!t||||\}}} |jD]} t| |j|||| ||} || q$|jp.)rrrzUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.) evaluatorscustom_evaluatorsanyrrrrgrdr"r2rrrrrr)rrrrrrrrrrrr#rZcustom_evaluatorr;r;r<rTsV        rr callbacksrmetadatarrrrrrOptional[Dict[str, Any]]Union[str, BaseMessage]c s|dur7||}t|tst|tr/tdd|Dr/|j|t||p$g|p'iddIdHStd|dzt|}|j|t||pDg|pGiddIdH}W|Stywt|} |jd i| dt||phg|pkidiIdH}Y|Sw) aAsynchronously run the language model. Args: llm: The language model to run. inputs: The input dictionary. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map inputs to the expected format. Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Ncsrrrrr;r;r<rrz_arun_llm..rrrrz%Input mapper returned invalid format 3 Expected a single string or list of chat messages.rr;) rdrxrrIainvokerr6rr) rrrrrrprompt_or_messagesr llm_output llm_inputsr;r;r< _arun_llmsP  rrrrUnion[Chain, Runnable]Union[dict, str]c s|dur|n||}t|tr;t|tr;t|dkr;|jr;tt|}|j|t ||p.g|p1iddIdH}|St |p?g||pCid} |j|| dIdH}|S)z%Run a chain asynchronously on inputs.NrKrrrrr) rdr.rerrrrrrr rrrrrrZinputs_valrBrunnable_configr;r;r< _arun_chains,    r)rr'Union[dict, str, LLMResult, ChatResult]c st|trdnd}d}z8t|tr(t||j|d|d||ddIdH}n|}t||j|d|d||ddIdH}|}W|Styp}z t|d|j d |jd t |t |d }WYd}~|Sd}~ww) aAsynchronously run the Chain or language model. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map the input to the expected format. Returns: A list of outputs. LLMr.Nrrrr failed for example with inputs  r[) rdrrrrcr Exceptionrridreprro) rrrr chain_or_llmrmrBrrir;r;r<_arun_llm_or_chainsJ   rc Cs|dur5||}t|tst|tr-tdd|Dr-|j|t||p#g|p&idd}|Std|dzt|}|j|t||pBg|pEidd}W|Stylt|} |jd i| dt||pcid i}Y|Sw) a Run the language model on the example. Args: llm: The language model to run. inputs: The input dictionary. callbacks: The callbacks to use during the run. tags: Optional tags to add to the run. input_mapper: function to map to the inputs dictionary from an Example Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Ncsrrrrr;r;r<rNrz_run_llm..rrz'Input mapper returned invalid format: rr)rrr;) rdrxrrIinvokerr6rr) rrrrrrrrZ llm_promptsrr;r;r<_run_llm0sL  rUnion[Dict, str]c Cs|dur|n||}t|tr7t|tr7t|dkr7|jr7tt|}|j|t ||p-g|p0idd}|St |p;g||p?id} |j|| d}|S)zRun a chain on inputs.NrKrrr) rdr.rerrrrrrrrr;r;r< _run_chainns*   rc Cst|trdnd}d}z2t|tr$t||j|d|d||dd}n|}t||j|d|d||dd}|}W|Styo}z&t|j} t |d|j d |jd | d | t |d }WYd}~|Sd}~ww) a Run the Chain or language model synchronously. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. Returns: Union[List[dict], List[str], List[LLMResult], List[ChatResult]]: The outputs of the model or chain. rr.Nrrrrrrz Error Type: z , Message: r ) rdrrrrcrr rr7rrr ro) rrrrrrmrBrriZ error_typer;r;r<_run_llm_or_chainsN    rclientr project_nameproject_metadatadataset_versionOptional[Union[str, datetime]]1Tuple[MCF, TracerSession, Dataset, List[Example]]c Csbt||}|j|d}t|j|j|d} | std|ddd| D} | r,t| nd} | r4| nd} z'|p:i}t} | rHi|d| i}| |d<|j ||j|rWd |ini|d }Wn1t tt fy}z"d t |vrp|t }d |d |d|d}td|d|d}~ww|jd|j}td|d|d|d|jdd|||| fS)N)r)Z dataset_idZas_ofzDataset z has no example rows.cSsg|]}|jr|jqSr;) modified_at)rFexr;r;r<rHsz%_prepare_eval_run..gitrr)Zreference_dataset_idZ project_extrarzalready exists z+ run_on_dataset( ... project_name="z - z", # Update since z already exists ) z Test project z/ already exists. Please use a different name: z/compare?selectedSessions=z)View the evaluation results for project 'z' at: z View all tests for Dataset z at: T)flush)rZ read_datasetrZ list_examplesr rmax isoformatrZcreate_projectr,r+rxuuiduuid4urlprint)rrrrrrr wrapped_modeldatasetrrZmax_modified_atZinferred_versionZgit_infoprojectriuidZ example_msgZcomparison_urlr;r;r<_prepare_eval_runsl    r)c@s*eZdZUdZded<ded<ded<dS) _RowResultz5A dictionary of the results for a single example row.z Optional[List[EvaluationResult]]rRzOptional[float]r\rr]Nrr;r;r;r<r*s  r*F)totalc@seZdZUdZded<ded<ded<ded <d ed <d Zd ed<d>ddZd?ddZd@ddZdAd!d"Z dBdCd&d'Z e ( dDdEdrecCsi}t|j|D]M\}}tt|t|ji}|j|dg|d|dd|t|j<t|t r?|j |t|jd<n ||t|jd<|j rU|j |t|jd<q|S)NrRr\r])rCrRr\r]r[rBrD) ziprr r*rcrxr rrdror[r)rNr/r0rQrrBZ row_resultr;r;r<_merge_test_outputss  z(_DatasetRunContainer._merge_test_outputsrunsDict[str, Run] List[dict]c s|j}|sgSfdd|jD}g}tjX}|D]L}z+|||j}t|tr-|}|t t||j |j j fi|d|j jdWqtyg}ztdt|d|WYd}~qd}~wwWd|S1sswY|S)Ncsg|] }t|jqSr;)rxr rFrr4r;r<rH0sz>_DatasetRunContainer._run_batch_evaluators..)r]Z project_idzError running batch evaluator z: )r.r concurrentfuturesThreadPoolExecutorrdr!rergr submitrZcreate_feedbackr'r r rr^r ) rNr4rZ runs_listaggregate_feedbackexecutorZ evaluatorrmrir;r8r<_run_batch_evaluators,s>    z*_DatasetRunContainer._run_batch_evaluators,Tuple[Dict[str, _RowResult], Dict[str, Run]]c Csi}i}|jD]d}tt|dD]Z}t|tr3|j}|D]\\}}}|t|i d|iqqt|t rj|j } | rH| j rH| j | j nd} | rQt| jnd} |t|ji | | | d| |t|j<qqttttf||fS)NrrR)r\r]run)r-r rrdrZlogged_eval_resultsrb setdefaultrxrfrZ latest_runend_time start_time total_secondsr rlrr*) rNr0all_runsccallbackZ eval_results_rlrVrAr\r]r;r;r<_collect_metricsEs<   z%_DatasetRunContainer._collect_metrics-List[Union[dict, str, LLMResult, ChatResult]]r=cCsXtdt|\}}d}|jrtd||}|||}t|jj ||dS)Nz#Waiting for evaluators to complete.zRunning session evaluators.)rrQZaggregate_metrics) rrrrJr.r?r3r=r'rw)rNr/r0rFr=rQr;r;r<_collect_test_resultsbs     z*_DatasetRunContainer._collect_test_resultsFverboseboolc Cs||}|r.z |}t|Wnty-}ztdt|WYd}~nd}~wwz|jj|j j t t jdW|Sty\}ztdt|WYd}~|Sd}~ww)Nz$Failed to print aggregate feedback: )rCzFailed to close project: )rLrP_display_aggregate_resultsr rrr rZupdate_projectr'r rnowrutc)rNr/rMrQZ agg_feedbackrir;r;r<finishts&   z_DatasetRunContainer.finishrrxrrrrr3rrrrrconcurrency_levelintrr revision_idrOptional[Union[datetime, str]]c  s|pt}| r| s i} | d| it|||| | d\} } }p%gjdp-iD]\}}d|d|q0djdi| rM| d<t|} t | ||| j pZt j t |d| |tt|fdd |D}|| |||r|jd Sdd S) NrV)rrrrzgit:=rrc sBg|]}ttj|jdtpg|jddgdqS))rrrlr)rrrlmax_concurrency)rrrYr)rrrwr rr7rrT progress_barr'rZ run_metadatarr;r<rHs*z0_DatasetRunContainer.prepare..)rr'r%rr-r.)r4Z random_namerfr)rrcrbrgrrrr'kvrr5ZProgressBarCallbackrr.)clsrrrrr3rrrTrrVrr%r&rrUrVr-r;rZr<preparesN  z_DatasetRunContainer.prepare)r/rr0r1r>re)r4r5r>r6)r>r@)r/rKr>r=)F)r/rrMrNr>r=)NNNrSNNN)rrrrxrrrrr3rrrrrrTrUrrrVrrrWr>r,) r7r8r9r:rr.r3r?rJrLrR classmethodr^r;r;r;r<r, s,       r,rNcCsDzddlm}|}|duodtt|vWSty!YdSw)Nr) get_ipythonZzmqshellF)ZIPythonr`rxrra)r`resr;r;r<_is_jupyter_environments  rbaggregate_resultsr?cCsTtrddlm}m}||d||dS|jdddd}tdt|dS) Nr)HTMLdisplayz

Experiment Results:

cSs|dS)Nz.2fr;)xr;r;r<rrz,_display_aggregate_results..right)Z float_formatjustifyz Experiment Results:)rbIPython.displayrdreZ to_stringr$)rcrdreZformatted_stringr;r;r<rOs   rOaThe input_mapper argument is deprecated and will be removed in a future release. Please add a RunnableLambda to your chain to map inputs to the expected format instead. Example: def construct_chain(): my_chain = ... input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x} return input_mapper | my_chain run_on_dataset(..., llm_or_chain_factory=construct_chain) (See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)rS)r3rrTrrrMrVOptional[Client]rWrTrUrMrVrqrc s| dd} | rtdtdd| durtd} | dd} | r)tdddd| r8tdd | d dd |p.rtru)rrrvr rcrrr,r^r2rr-rZget_executor_for_configrrxryrzrr%rR)rrrr3rrTrrrMrVrqrr/r>r;r}r<run_on_dataset8sh        r~a1 Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to add to the project. Useful for storing information the test variant. (prompt version, model version, etc.) client: LangSmith client to use to access the dataset and to log feedback and run traces. verbose: Whether to print progress. tags: Tags to add to each run in the project. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see :func:`arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain_openai import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = smith_eval.RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", smith_eval.RunEvalConfig.Criteria("helpfulness"), smith_eval.RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = smith_eval.RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) zrun_on_dataset(zawait arun_on_dataset()r~)rrrrxr>r)rrr>rx)rrr>re)rr(rrr>rr)rr(rr.rrr>rr)rr(rrrrr>rr) rrrrr3rrr'r>r)rrrrr>r)rrrrr>r)rrrrr>r)rrrrrrxrr'rrrrrrrrr>r") rrrrrrrrr>r)rrrrxrr'rrrrrrr>r)rrrrrrrrrrrrr>r)rrrrrrrrrrrrr>r) rr(rrrrrrr>r)rrrrrrrrrrrrr>r)rrrrrrrrrrrrr>rr)rrrrxrrrrxrrrrrrr>r)r>rN)rcr?r>rr)rrjrrxrrr3rrrWrTrUrrrrrMrNrVrrqrr>r)r: __future__rconcurrent.futuresr9 dataclassesryrloggingr!rrtypingrrrrr r r r r Zlangchain_core._apirZ langchain_core.callbacks.managerrZlangchain_core.language_modelsrZlangchain_core.messagesrrZlangchain_core.outputsrrZlangchain_core.runnablesrrrrrrrwZ!langchain_core.tracers.evaluationrrZ langchain_core.tracers.langchainrZlangsmith.clientrZ langsmith.envrr Zlangsmith.evaluationr!r"r#rZlangsmith.run_helpersr$r%Zlangsmith.schemasr&r'r(r)r*Zlangsmith.utilsr+requestsr,Ztyping_extensionsr-Zlangchain.chains.baser.Zlangchain.evaluation.loadingr/Zlangchain.evaluation.schemar0r1r2Zlangchain.smithr3rZlangchain.smith.evaluationrr4r5r`rh getLoggerr7rrerrr r6r=rorrrrrrrrrrrrrrrrrrrrr)r* dataclassr,rbrOrvr|r~Z_RUN_ON_DATASET_DOCSTRINGreplacer;r;r;r<s ,                I =2 ( ! %      C GE%>C%= = C FMk