o +if@sdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlmZmZddlmZmZmZddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(ddl)m*Z+ddl,Z,ddl,m-Z.dd l/m0Z0ddl1Z1dd l1m2Z3dd l1m4Z5dd l1m6Z7dd l8m9Z:erddl;ZZ?e =d@e7AdcddZBddddZCdcddZDe'ejEeFfZGdeddZHeFeIeJeKfZLdZMdfdgd$d%ZNdhd)d*ZOdid-d.ZPdjd0d1ZQdkd2d3ZRdld4d5ZSdmd8d9ZTdnd;d<ZUdodAdBZVedCdDGdEdFdFZWGdGdHdHZX CdpdqdOdPZYdrdSdTZZdUZ[dVZ\dWZ]dsd[d\Z^dtd_d`Z_dtdadbZ`dS)uzThe LangSmith Client.) annotationsN) dataclassfield)Empty PriorityQueueQueue) TYPE_CHECKINGAnyCallable DefaultDictDictIterableIteratorListMappingOptionalSequenceTupleTypeUnioncast)parse)adapters)Retry)env)schemas)utils) evaluatorzurllib3.connectionpoolurlstrreturnboolcCsVzt|jdd}t|}|dkp|dp|dWStjy*YdSw)zCheck if the URL is localhost. Parameters ---------- url : str The URL to check. Returns ------- bool True if the URL is localhost, False otherwise. :rz 127.0.0.1z0.0.0.0z::F) urllib_parseurlsplitnetlocsplitsocket gethostbyname startswithgaierror)rr%ipr,I/var/www/html/corbot_env/lib/python3.10/site-packages/langsmith/client.py _is_localhost<s  r. url_or_tokenapi_urlTuple[str, str]cCsjz t|||fWStyYnwt|}|jd}t|dkr-|d}||fSt d|)z*Parse a public dataset URL or share token./zInvalid public dataset URL: ) uuidUUID ValueErrorr#urlparsepathr&lenls_utilsLangSmithUserError)r/r0 parsed_url path_parts token_uuidr,r,r-_parse_token_or_urlQs      r@cCs8zt|jdd}|dWStyYdSw)zCheck if the URL is langchain hosted. Parameters ---------- url : str The URL to check. Returns ------- bool True if the URL is langchain hosted, False otherwise. r"rz langchain.comF)r#r$r%r&endswith Exception)rr%r,r,r-_is_langchain_hostedds    rCrcCsTtdgddddd}tjd}ttt|ddk}|r#d |d <td i|S) zGet the default retry configuration. If urllib3 version is 1.26 or greater, retry on all methods. Returns ------- Retry The default retry configuration. )iiiii?F)totalstatus_forcelistbackoff_factorraise_on_redirectraise_on_statusurllib3.)Nallowed_methodsr,) dict importlibmetadataversiontuplemapintr&r) retry_paramsurllib3_versionuse_allowed_methodsr,r,r-_default_retry_config{s  r[r3objr depthrWc sBztkrz ttWWStytYWSwttjr)WStt j r4t WSdus=tt r@WStt rKdWSttttfr_fddtDWSttrpfddDWSgd}|D]O\}}t|rtt|rzt|}|r||dn|}t|WWSty}ztdtd |tWYd}~WSd}~wwqvtrtWStd rfd djD}ntd rt}ntWSfd d|DWSty }ztdtd |tWYd}~Sd}~ww)Nzutf-8csg|] }t|dqSrN_serialize_json).0xr]r,r- z#_serialize_json..cs i|] \}}|t|dqSr^r_rakvrcr,r- s z#_serialize_json..))model_dump_jsonT)jsonT)to_jsonF exclude_nonezFailed to serialize z to JSON: __slots__csi|] }|t|dqSN)getattr)raslot)r\r,r-rire__dict__cs2i|]\}}||urt|ddnt|qS)rNrc)r`reprrfr]r\r,r-ris ) _MAX_DEPTHrkloadsdumps BaseExceptionrt isinstancedatetime isoformatr5r6r_PRIMITIVE_TYPESbytesdecodesetlistrUrQitemshasattrcallablerqrBloggerdebugtype dataclasses is_dataclassasdictrovars) r\r]serialization_methodsattrrnmethodjson_stre all_attrsr,rur-r`s`                 r`sessionrequests.SessionNonecCstd|dS)zjClose the session. Parameters ---------- session : Session The session to close. zClosing Client.sessionN)rrclose)rr,r,r- close_sessions  rapi_key Optional[str]cCs|s t|r tddSdS)aVerify API key is provided if url not localhost. Parameters ---------- api_url : str The API URL. api_key : str or None The API key. Raises ------ LangSmithUserError If the API key is not provided when using the hosted service. z8API key must be provided when using hosted LangSmith APIN)rCr;r<r0rr,r,r-_validate_api_key_if_hostedsr float | NonecCsBtd}|dur dSt|}|dks|dkrtd||S)ziGet the tracing sampling rate. Returns ------- float The tracing sampling rate. LANGCHAIN_TRACING_SAMPLING_RATENrrNzELANGCHAIN_TRACING_SAMPLING_RATE must be between 0 and 1 if set. Got: )osgetenvfloatr;r<)sampling_rate_str sampling_rater,r,r-_get_tracing_sampling_rates rcCs>|dur|ntd}|dus|sdS|ddS)NLANGCHAIN_API_KEY"')rrstrip)rr,r,r- _get_api_keysrcCsD|dur|ntdd}|std|dddS)NLANGCHAIN_ENDPOINTzhttps://api.smith.langchain.comz!LangSmith API URL cannot be emptyrrr2)rrrr;r<rstrip)r0r_api_urlr,r,r- _get_api_urls rinputsDict[str, Any]cCtjddkr iS|S)NLANGCHAIN_HIDE_INPUTStruerenvirongetrr,r,r- _hide_inputs#routputscCr)NLANGCHAIN_HIDE_OUTPUTSrrrr,r,r- _hide_outputs)rrvalueID_TYPEvar uuid.UUIDc CsNzt|tjs t|WS|WSty&}z t|d||d}~ww)Nz* must be a valid UUID or UUID string. Got )rzr5r6r7r;r<)rrrr,r,r-_as_uuid/s rT)orderc@s0eZdZUded<ded<eddZded<dS) TracingQueueItemrpriorityactionF)comparer itemN)__name__ __module__ __qualname____annotations__rrr,r,r,r-r8s rc @s eZdZdZgdZ dddddddddddZdddZdddZedddZ edddZ edd!d"Z e e jd#d$dd'd(Z #  ddd2d3Z ddd7d8Zdd9ddd?d@ddCdDZdejjdEddPdQZddejjdRddUdVZe ddZd[Ze dd]d^Zd_d`ddddeZdddfddodpZ  dd_dqddvdwZddddddxdddZdddZ _ddddZ dddddddddddd dddZ!ddddddZ"dddddZ#dddZ$dddZ%dddZ& ddddZ' ddddddZ( ddddddZ)dddZ*dddZ+dddddZ,ddddddd„Z-ddd_dddÜdddʄZ.dddddd˜ddd̈́Z/dddЄZ0e12dѡddd_dҜdddքZ3ddלdddلZ4dddќddd܄Z5      ddddZ6e12dddddddZ7dejjdEdddZ8ddddddZ9e12dddddddZ: ddddddZ;ddddddddZdddZ?e12d    ddddZ@e12d    ddd d ZA   ddd d ZBdddd dddZCe12d     ddddZDdddZE    dÐdĐddZFdddddŐdd ZGdƐd!d"ZHdǐd$d%ZIdȐd)d*ZJdɐd.d/ZKddd_d0dʐd7d8ZL ddːd:d;ZMddd_d0d̐d=d>ZNdddddejOjPddd_d?d@ d͐dPdQZQdddddRdΐdSdTZRdϐdUdVZSddddWdАd\d]ZTdѐd^d_ZUdddd`dҐdcddZVdddedӐdhdiZWdԐdjdkZXddldՐdmdnZYd֐dodpZZdאdrdsZ[dؐdudvZ\ddwddd_ddddxdِddZ]ddwddd_ddddxdِddZ^dS(Clientz.Client for interacting with the LangSmith API.) __weakref__r0r retry_config timeout_msr_get_data_type_cached_web_url _tenant_idtracing_sample_rate_sampled_post_uuids tracing_queueNT)rrrweb_urlrauto_batch_tracingr0rrrOptional[Retry]r Optional[int]rrOptional[requests.Session]rr!r rc Cst|_t|_t||_t||j|_t|j|j|pt |_ |p$d|_ ||_ d|_ |r0|nt|_t|t|j|rQt|_tjtt|fdnd|_tj|j d}|jd||jd|tjdd|j |_!dS) a*Initialize a Client instance. Parameters ---------- api_url : str or None, default=None URL for the LangSmith API. Defaults to the LANGCHAIN_ENDPOINT environment variable or https://api.smith.langchain.com if not set. api_key : str or None, default=None API key for the LangSmith API. Defaults to the LANGCHAIN_API_KEY environment variable. retry_config : Retry or None, default=None Retry configuration for the HTTPAdapter. timeout_ms : int or None, default=None Timeout in milliseconds for the HTTPAdapter. web_url : str or None, default=None URL for the LangSmith web app. Default is auto-inferred from the ENDPOINT. session: requests.Session or None, default=None The session to use for requests. If None, a new session will be created. Raises ------ LangSmithUserError If the API key is not provided when using the hosted service. i'Ntargetargs) max_retrieszhttp://zhttps:// maxsize)"rrrrrrrr0rr[rrrrrequestsSessionrweakreffinalizerrr threadingThread_tracing_control_thread_funcrefstartrequests_adapters HTTPAdaptermount functools lru_cache_get_data_typer) selfr0rrrrrradapterr,r,r-__init__Qs2%      zClient.__init__rcCs|j}d|dS)zReturn an HTML representation of the instance with a link to the URL. Returns ------- str The HTML representation of the instance. z LangSmith Client) _host_url)rlinkr,r,r- _repr_html_s zClient._repr_html_cCsd|jdS)zReturn a string representation of the instance with a link to the URL. Returns ------- str The string representation of the instance. zClient (API URL: ))r0rr,r,r-__repr__szClient.__repr__cCs~|jr|j}|St|j}t|jrd}|S|jdr1|jddd}t|j |d}|S|j dr;d}|Sd}|S) zThe web host url.zhttp://localhostz/apirNr)r9zdev.zhttps://dev.smith.langchain.comzhttps://smith.langchain.com) rr#r8r0r.r9rArsplit urlunparse_replacer%r))rrr=new_pathr,r,r-rs    zClient._host_urlDict[str, str]cCs$ddtji}|jr|j|d<|S)zGet the headers for the API request. Returns ------- Dict[str, str] The headers for the API request. z User-Agentz langsmith-py/z x-api-key) langsmith __version__r)rheadersr,r,r-_headerss  zClient._headers"Optional[ls_schemas.LangSmithInfo]cCs<t|j|j|j}|dur|jdurd|_tttj |S)zGet the information about the LangSmith API. Returns ------- Optional[ls_schemas.LangSmithInfo] The information about the LangSmith API, or None if the API is not available. N) r _get_inforr0rrrr ls_schemas LangSmithInfo)rinfor,r,r-rs z Client.inforNrrrWc Csz|j|dddi|dd}t|tjdi|WStjy)YdStyH}zt d|dt |WYd}~dSd}~ww) Nz/infoAcceptapplication/json)rtimeoutzFailed to get info from z: r,) rr;raise_for_status_with_textrrrkr HTTPErrorryrwarningrt)rr0rresponserr,r,r-rs zClient._get_inforequest_methodrrequest_kwargsrstop_after_attemptretry_on'Optional[Sequence[Type[BaseException]]] to_ignorerequests.Responsec Csg|pgtjtjfR}g|pdR}d} t|D]R} zz|jj||fddi|} t| | WWStjy} z| dur| j dkrYtd|d|dt | | j d krkt d |d t | | j d kr}t d |d t | | j dkrt d|d t | | j dkrtd|d t | td|d|dt | td|d|dt | d} ~ wtjy} ztd|d|dt | | d} ~ wty%} z7t| j} t| dkr| dnd} | dd} d| dg| g| dd}td|d|d|| d} ~ ww|yM} z| durCtd| | WYd} ~ SWYd} ~ qd} ~ w|ym| d|kr\d| td}t|Yqwtd|d|d)aSend a request with retries. Parameters ---------- request_method : str The HTTP request method. url : str The URL to send the request to. request_kwargs : Mapping Additional request parameters. stop_after_attempt : int, default=1 The number of attempts to make. retry_on : Sequence[Type[BaseException]] or None, default=None The exceptions to retry on. In addition to: [LangSmithConnectionError, LangSmithAPIError]. to_ignore : Sequence[Type[BaseException]] or None, default=None The exceptions to ignore / pass on. Returns ------- Response The response object. Raises ------ LangSmithAPIError If a server error occurs. LangSmithUserError If the request fails. LangSmithConnectionError If a connection error occurs. LangSmithError If the request fails. r,NstreamFizServer error caused failure to  z in LangSmith API. rEzRate limit exceeded for z. izAuthentication failed for izResource not found for iz Conflict for z Failed to z#Connection error caused failure to z< in LangSmith API. Please confirm your LANGCHAIN_ENDPOINT. rNrzsession (project) rr3zPassing on exception %srFz in LangSmith API.)r;LangSmithConnectionErrorLangSmithAPIErrorrangerrequestr rr  status_codertLangSmithRateLimitErrorLangSmithAuthErrorLangSmithNotFoundErrorLangSmithConflictErrorLangSmithErrorr<ConnectionErrorrBrrr:replacejoinrrrandomtimesleep)rrrrrrr retry_on_ to_ignore_r idxrrmsgemsg sleep_timer,r,r-request_with_retriess,          "    zClient.request_with_retriesr9paramsOptional[Dict[str, Any]]cCs*|jd|j|||j|jdddS)Nrr)r0rr r)r/r0rr)rr9r0r,r,r-_get_with_retriesis zClient._get_with_retriesr0Optional[dict]Iterator[dict]ccs|r|ni}|dd}|dd|d< ||d<|j||d}|}|s+dS|EdHt||dkr:dS|t|7}q)a1Get a paginated list of items. Parameters ---------- path : str The path of the request URL. params : dict or None, default=None The query parameters. Yields ------ dict The items in the paginated list. offsetrlimitdTr4N)copyrr3rkr:)rr9r0params_r7r rr,r,r-_get_paginated_listvs   zClient._get_paginated_listpostruns)bodyrdata_keyr?r@c cs|r|ni} |j||j|tj|td|j|jddd}|}|s,dS||s3dS||EdH|d}|sCdS|dsJdS|d|d <q ) aGet a cursor paginated list of items. Parameters ---------- path : str The path of the request URL. body : dict or None, default=None The query body. request_method : str, default="post" The HTTP request method. data_key : str, default="runs" Yields ------ dict The items in the paginated list. Tdefaultrdatarr r2Ncursorsnextcursor) r:r/r0rkrxr`rrr) rr9r?rr@r;r  response_bodyrEr,r,r-_get_cursor_paginated_lists0       z!Client._get_cursor_paginated_list) description data_typedf pd.DataFramename input_keys Sequence[str] output_keysrJrKOptional[ls_schemas.DataType]ls_schemas.DatasetcCs:t}|j|dd|d|jd|f|||||dS)aUpload a dataframe as individual examples to the LangSmith API. Parameters ---------- df : pd.DataFrame The dataframe to upload. name : str The name of the dataset. input_keys : Sequence[str] The input keys. output_keys : Sequence[str] The output keys. description : str or None, default=None The description of the dataset. data_type : DataType or None, default=DataType.kv The data type of the dataset. Returns ------- Dataset The uploaded dataset. Raises ------ ValueError If the csv_file is not a string or tuple. F)indexrzdata.csv)rOrQrJrNrK)ioBytesIOto_csvseek upload_csv)rrLrNrOrQrJrKcsv_filer,r,r-upload_dataframes% zClient.upload_dataframerNrJrKrZ"Union[str, Tuple[str, io.BytesIO]]c Cs<||d}|r ||d<|r||d<|rt||d<t|trGt|d}d|i} |jj|jd|j|| d} Wd n1sAwYnt|t r]|jj|jd|j|d|id} nt d t | | } d | vrd | d vrt|tr{|n|d } | dd} t d| dtjdi| |j|dS)a|Upload a CSV file to the LangSmith API. Parameters ---------- csv_file : str or Tuple[str, BytesIO] The CSV file to upload. If a string, it should be the path If a tuple, it should be a tuple containing the filename and a BytesIO object. input_keys : Sequence[str] The input keys. output_keys : Sequence[str] The output keys. name : str or None, default=None The name of the dataset. description : str or None, default=None The description of the dataset. data_type : DataType or None, default=DataType.kv The data type of the dataset. Returns ------- Dataset The uploaded dataset. Raises ------ ValueError If the csv_file is not a string or tuple. )rOrQrNrJrKrbfilez/datasets/upload)rrDfilesNz"csv_file must be a string or tupledetailzalready existsrr2Dataset z already existsrrr,)r;get_enum_valuerzropenrr=r0rrUr7r rkr&rDatasetr_get_tenant_id) rrZrOrQrNrJrKrDffile_r result file_namer,r,r-rYsN(      zClient.upload_csvrun3Union[ls_schemas.Run, dict, ls_schemas.RunLikeDict]rQcCst|drtt|dr|}ntt|}d|vr!t|d<nt|dtr1t |d|d<d|vr=t |d|d<d|vrIt |d|d<| dsUt j |d<|S)z Transforms the given run object into a dictionary representation. Args: run (Union[ls_schemas.Run, dict]): The run object to transform. Returns: dict: The transformed run object as a dictionary. rQidrr start_time)rrrqrQrr5uuid4rzrr6rrrr{utcnow)rm run_creater,r,r-_run_transformHs   zClient._run_transformSequence[dict]cstt}|D]1}tt|di}|di}i|||d<|dit}fdd|DqdS)NextraruntimerScsi|] \}}|vr||qSr,r,rfrSr,r-riqz.Client._insert_runtime_env..)ls_envget_runtime_and_metricsrrQ setdefaultget_langchain_env_var_metadataupdater)r> runtime_envrs run_extrarwlangchain_metadatar,rxr-_insert_runtime_enves  zClient._insert_runtime_envFpatchIterable[dict]r list[dict]cCs|jdur t|S|r(g}|D]}|d|jvr%|||j|dq|Sg}|D]}t|jkrB|||j|dq,|S)Nro)rrrappendremover&add)rr>rsampledrmr,r,r-_filter_for_samplingts"   zClient._filter_for_sampling) project_name revision_idrrrun_typerrOptional[ID_TYPE]kwargsr c Ks|p |dt}i|||||d}||gsdS||}||g|dur4||ddd<|jdurX|ddurX|ddurX|jdurX|j t |dd |Si|j d d d }|j d |j d tj|td||jddtjfddS)ayPersist a run to the LangSmith API. Parameters ---------- name : str The name of the run. inputs : Dict[str, Any] The input values for the run. run_type : str The type of the run, such as tool, chain, llm, retriever, embedding, prompt, or parser. revision_id : ID_TYPE or None, default=None The revision ID of the run. **kwargs : Any Additional keyword arguments. Raises ------ LangSmithUserError If the API key is not provided when using the hosted service. session_name)rrNrrNrvrSrtrace_id dotted_ordercreaterr Content-Typer=/runsrArrCrr)popr;get_tracer_projectrrtrrrrputrrr/r0rkrxr`rr!) rrNrrrrrrsrr,r,r- create_runsP        zClient.create_run) pre_sampledrGOptional[Sequence[Union[ls_schemas.Run, ls_schemas.RunLikeDict, Dict]]]r~rc s~|s|sdSfdd|pgD}fdd|pgD}|rL|rLdd|D}g}|D]}|d|vrD||ddd|Dq+||q+|}|D]}|d rZ|d s_td qN|D]}|d rn|d sstd qb|r|||d } n |j|d dd } | ds| dsdS| djdj dt j | t dj dijddddtjfddS)a Batch ingest/upsert multiple runs in the Langsmith system. Args: create (Optional[Sequence[Union[ls_schemas.Run, RunLikeDict]]]): A sequence of `Run` objects or equivalent dictionaries representing runs to be created / posted. update (Optional[Sequence[Union[ls_schemas.Run, RunLikeDict]]]): A sequence of `Run` objects or equivalent dictionaries representing runs that have already been created and should be updated / patched. pre_sampled (bool, optional): Whether the runs have already been subject to sampling, and therefore should not be sampled again. Defaults to False. Returns: None: If both `create` and `update` are None. Raises: LangsmithAPIError: If there is an error in the API request. Note: - The run objects MUST contain the dotted_order and trace_id fields to be accepted by the API. Ncg|]}|qSr,rtrarmrr,r-rdz,Client.batch_ingest_runs..crr,rrrr,r-rdrcSsi|]}|d|qSror,rr,r,r-rirz,Client.batch_ingest_runs..rocSi|] \}}|dur||qSrpr,rfr,r,r-ri ryrrz:Batch ingest requires trace_id and dotted_order to be set.)r=rTrr=rz /runs/batchrArrr)rDr rr)r~rrrr;r<rrr/r0rkrxr`rrr!) rrr~r create_dicts update_dicts create_by_idstandalone_updatesrmr?r,rr-batch_ingest_runssd$       zClient.batch_ingest_runs)end_timeerrorrreventsrun_idrrOptional[datetime.datetime]rOptional[Dict]rrOptional[Sequence[dict]]c Ks0i|jddd}t|d|dd|dd|ddd} |j| gd d s*dS|dur4|| d <|dur<|| d <|durFt|| d <|durPt|| d<|durX|| d<|jdurz| ddurz| ddurz|jdurz|j t | dd| S|j d|j d| dt j| td||jddddS)atUpdate a run in the LangSmith API. Parameters ---------- run_id : str or UUID The ID of the run to update. end_time : datetime or None The end time of the run. error : str or None, default=None The error message of the run. inputs : Dict or None, default=None The input values for the run. outputs : Dict or None, default=None The output values for the run. events : Sequence[dict] or None, default=None The events for the run. **kwargs : Any Kwargs are ignored. rrrrN parent_run_idr)rorrrTrrrrrrr~r/runs/rorArrCr2)rrrrr|rrrrrrr/r0rkrxr`r) rrrrrrrrrrDr,r,r- update_run7sL            zClient.update_runls_schemas.RuncCs|j|jd}tt}i}t|dddD]}|jdur'td|j d||j ||||j <q| |j g|_ | D] \}}|||_ qA|S)aGLoad child runs for a given run. Parameters ---------- run : Run The run to load child runs for. Returns ------- Run The run with loaded child runs. Raises ------ LangSmithError If a child run has no parent. rcSs|jSrp)r)rr,r,r-sz)Client._load_child_runs..)keyNz Child run z has no parent) list_runs child_run_ids collections defaultdictrsortedrr;r"rorr child_runsr)rrmrtreemapr> child_runrchildrenr,r,r-_load_child_runss"   zClient._load_child_runsload_child_runscCsL|dt|d}tjdi|d|ji}|r$|jr$||}|S)a5Read a run from the LangSmith API. Parameters ---------- run_id : str or UUID The ID of the run to read. load_child_runs : bool, default=False Whether to load nested child runs. Returns ------- Run The run. rrrNr,)r3rrRunrkrrr)rrrr rmr,r,r-read_runs   zClient.read_run) project_idrrreference_example_idqueryfilterexecution_orderrrprrun_idsrrrrrrrpOptional[bool]rOptional[List[ID_TYPE]]Iterator[ls_schemas.Run]c  +s|dur|dur tdj|dj}|r|gnd||r |gnd||||| r+| nd| | d | } dd| D} fddjd | d d DEdHdS) aList runs from the LangSmith API. Parameters ---------- project_id : UUID or None, default=None The ID of the project to filter by. project_name : str or None, default=None The name of the project to filter by. run_type : str or None, default=None The type of the runs to filter by. reference_example_id : UUID or None, default=None The ID of the reference example to filter by. query : str or None, default=None The query string to filter by. filter : str or None, default=None The filter string to filter by. execution_order : int or None, default=None The execution order to filter by. Execution order is the position of the run in the full trace's execution sequence. All root run traces have execution_order 1. parent_run_id : UUID or None, default=None The ID of the parent run to filter by. start_time : datetime or None, default=None The start time to filter by. error : bool or None, default=None Whether to filter by error status. run_ids : List[str or UUID] or None, default=None The IDs of the runs to filter by. **kwargs : Any Additional keyword arguments. Yields ------ Run The runs. Nz3Only one of project_id or project_name may be givenr) rrreference_examplerrr parent_runrprrocSrrpr,rfr,r,r-riryz$Client.list_runs..c3*|]}tjdi|djiVqdSrNr,rrrrrr,r-   z#Client.list_runs..z /runs/queryr=)r?r)r7 read_projectror|rrI)rrrrrrrrrrprrr body_queryr,rr-rs04    zClient.list_runs)rrls_schemas.RunBasecCst|dr|jdur|j}n|dur|}n|dur!|j|dj}n t}|j|dj}t|d}|jd|d|d|jdS)adGet the URL for a run. Parameters ---------- run : Run The run. project_name : str or None, default=None The name of the project. project_id : UUID or None, default=None The ID of the project. Returns ------- str The URL for the run. session_idNrz/o/z /projects/p/z/r/z ?poll=true) rrrror;rrrrh)rrmrrr session_id_r,r,r- get_run_url s zClient.get_run_url)share_idrcCsnt|d}t||pttd}|jj|jd|d|j|d}t || d}|j d|dS) zGet a share link for a run.r)r share_tokenr/sharerrkr/public//r) rrr5rqrrr0rr;r rkr)rrrrun_id_rDr rr,r,r- share_run2s   zClient.share_runcC4|jj|jdt|dd|jd}t|dS)zDelete share link for a run.rrrrNrdeleter0rrr;r )rrr r,r,r- unshare_runB zClient.unshare_runcCsb|jj|jdt|dd|jd}t||}|dus$d|vr&dS|jd|ddS)Nrrrrrrr) rrr0rrr;r rkr)rrr rkr,r,r-read_run_shared_linkJs zClient.read_run_shared_linkcCs|t|d}|duS)zGet share state for a run.rN)rr)rrrr,r,r- run_is_sharedUszClient.run_is_sharedrOptional[List[str]]List[ls_schemas.Run]csV|t|d}jjjdt|ddj|d}t|fdd|DS)zGet shared runs.)rorrrrrr0c&g|]}tjdi|djiqSrr,rrrr,r-rdez+Client.list_shared_runs..) rrrr0rrr;r rk)rrrr0r r,rr-list_shared_runsZs  zClient.list_shared_runs dataset_name dataset_idrls_schemas.DatasetShareSchemac Cs|dur |dur td|dur|j|dj}|jj|jdt|dd|jd}t || }t t j i|d|jdt|d d d iS) N/Either dataset_id or dataset_name must be givenr /datasets/rrrrrrzresponse.share_token/d)r7 read_datasetrorrr0rrr;r rkrrDatasetShareSchemar)rrrr dr,r,r-read_dataset_shared_schemais(   z!Client.read_dataset_shared_schemacCs|dur |dur td|dur|j|dj}dt|i}|jj|jdt|dd|j|d}t || }t t ji|d|jd |d d iS) zGet a share link for a dataset.Nrrrrrrrrrr)r7rrorrrr0rrr;r rkrrrr)rrrrDr rr,r,r- share_datasets"  zClient.share_datasetcCr)z Delete share link for a dataset.rrrrNr)rrr r,r,r-unshare_datasetrzClient.unshare_datasetcCsX|jj|jdt|dd|jd}t|tjdi| |j d|ddS) zGet shared datasets.rr /datasetsrr)r _public_pathNr,) rrr0rrr;r rrgrkr)rrr r,r,r-read_shared_datasets   zClient.read_shared_dataset example_idsrList[ls_schemas.Example]csfi}|durdd|D|d<jjjdt|ddj|d}t|fd d|DS) zGet shared examples.NcSsg|]}t|qSr,)r)raror,r,r-rdsz/Client.list_shared_examples..rorr /examplesrcrr)rExamplerradatasetrr,r-rd)rrr0rrr;r rk)rrrr0r r,rr-list_shared_exampless  zClient.list_shared_examples) project_idsrN name_containsdataset_share_tokenr r (Iterator[ls_schemas.TracerSessionResult]c#sF|||d}t|d}fddjd|d|dDEdHdS)N)rorNr rcrrrTracerSessionResultrraprojectrr,r-rdr z/Client.list_shared_projects..rz/datasets/sessionsr4)rr<)rrr rNr r0rr,rr-list_shared_projectss    zClient.list_shared_projects)rJrSupsert project_extrareference_dataset_idrSrrrls_schemas.TracerSessionc Cs|jd}|}|ri|pid|i}|||d} i} |r"d| d<|dur*|| d<|jj|i|jdd itj| td d } t| t j d i| d |j iS)aCreate a project on the LangSmith API. Parameters ---------- project_name : str The name of the project. project_extra : dict or None, default=None Additional project information. metadata: dict or None, default=None Additional metadata to associate with the project. description : str or None, default=None The description of the project. upsert : bool, default=False Whether to update the project if it already exists. reference_dataset_id: UUID or None, default=None The ID of the reference dataset to associate with the project. Returns ------- TracerSession The created project. /sessionsrS)rNrvrJTrNrrrrArrDrr,) r0rr=rrkrxr`r;r r TracerSessionr) rrrJrSrrrendpointrvr?r0r r,r,r-create_projects(   zClient.create_project)rNrJrSrrc Cs|jdt|d}|}|ri|pid|i}||||r"|ndd} |jj|i|jdditj| tdd } t | t j d i| d |j iS) aUpdate a LangSmith project. Parameters ---------- project_id : UUID The ID of the project to update. name : str or None, default=None The new name to give the project. This is only valid if the project has been assigned an end_time, meaning it has been completed/closed. description : str or None, default=None The new description to give the project. metadata: dict or None, default=None project_extra : dict or None, default=None Additional project information. Returns ------- TracerSession The updated project. /sessions/rrSN)rNrvrJrrrrArrr,)r0rr|rrrrkrxr`r;r rrr) rrrNrJrSrrrrvr?r r,r,r-update_projects   zClient.update_projectrcCsj|jdur|jS|jdddid}|}t|tr0tjdi|dd|ji}|j|_|jSt d) Nrr8rNr4rrzNo projects foundr,) rr3rkrzrrrr tenant_idr;r")rr rktracer_sessionr,r,r-rh?s   zClient._get_tenant_id)rr)rr include_statsr"ls_schemas.TracerSessionResultcCsd}ddi}|dur|dt|d7}n |dur||d<ntd||d <|j||d }|}t|trTt|d krEtd |d t j di|d d|j iSt j di|d|j iS)a,Read a project from the LangSmith API. Parameters ---------- project_id : str or None, default=None The ID of the project to read. project_name : str or None, default=None The name of the project to read. Note: Only one of project_id or project_name may be given. include_stats : bool, default=False Whether to include a project's aggregate statistics in the response. Returns ------- TracerSessionResult The project. rr8rNNr2rrN'Must provide project_name or project_idr"r4rzProject not foundrr,) rr7r3rkrzrr:r;r rrr)rrrr"r9r0r rkr,r,r-rLs*    zClient.read_project)rcCs*z |j|dWdStjyYdSw)aLCheck if a project exists. Parameters ---------- project_name : str The name of the project to check for. project_id : str or None, default=None The ID of the project to check for. Returns ------- bool Whether the project exists. rFT)rr;r )rrrr,r,r- has_projectzs zClient.has_project'pd.DataFrame'csddl}|j||dd}g}g|D]X}d|jidd|jDdd|jp)iD}|jrF|jD]\}} | d |d |<q7||j rS|j |j  nd|j |j d |jre|j||q||d} d g} fd dtdtDD]} |j| dD]} | d| j idd| jpiDqq| r|| d}|j| ddd} || jddS)aRRead the record-level information from a test project into a Pandas DF. Note: this will fetch whatever data exists in the DB. Results are not immediately available in the DB upon evaluation run completion. Returns ------- pd.DataFrame A dataframe containing the test results. rNrN)rrr example_idcSi|] \}}d||qS)zinput.r,rfr,r,r-riz+Client.get_test_results..cSr))zoutputs.r,rfr,r,r-rir*avgz feedback.)execution_timerror9csg|] }||qSr,r,)rai batch_sizerr,r-rdsz+Client.get_test_results..rcSr))z reference.r,rfr,r,r-ris T) left_index right_indexrecords)orient)pandasrrrrrfeedback_statsrr~rrp total_secondsrror DataFrame set_indexrr: list_examplesmergejson_normalizeto_dict)rrrpdr>resultsrrowrgrhrkexample_outputsbatchexamplerLr,r.r-get_test_resultss^      zClient.get_test_resultsreference_dataset_namereference_free"Iterator[ls_schemas.TracerSession]c#si}|dur ||d<|dur||d<|dur||d<|dur,|dur'td||d<n|dur;j|dj}||d<|durC||d<fd d jd |d DEdHdS) a List projects from the LangSmith API. Parameters ---------- project_ids : Optional[List[ID_TYPE]], optional A list of project IDs to filter by, by default None name : Optional[str], optional The name of the project to filter by, by default None name_contains : Optional[str], optional A string to search for in the project name, by default None reference_dataset_id : Optional[List[ID_TYPE]], optional A dataset ID to filter by, by default None reference_dataset_name : Optional[str], optional The name of the reference dataset to filter by, by default None reference_free : Optional[bool], optional Whether to filter for only projects not associated with a dataset. Yields ------ TracerSession The projects. NrorNr zGOnly one of reference_dataset_id or reference_dataset_name may be givenreference_datasetrrEc3rrrrrr,r-r rz'Client.list_projects..rr4)r7rror<)rr rNr rrDrEr0r,rr- list_projectss4    zClient.list_projectscCs^|durt|j|dj}n|durtd|jj|jdt|d|jd}t |dS)a Delete a project from LangSmith. Parameters ---------- project_name : str or None, default=None The name of the project to delete. project_id : str or None, default=None The ID of the project to delete. Nrr$rrr) rrror7rrr0rrr;r )rrrr r,r,r-delete_projects zClient.delete_projectls_schemas.DataTypecCshtj|||d}|jj|jdi|jddi|d}t|tj di||j | dS) aCreate a dataset in the LangSmith API. Parameters ---------- dataset_name : str The name of the dataset. description : str or None, default=None The description of the dataset. data_type : DataType or None, default=DataType.kv The data type of the dataset. Returns ------- Dataset The created dataset. r\rrrrrdNr,) r DatasetCreaterr=r0rrkr;r rgrrh)rrrJrKr r r,r,r-create_dataset's"  zClient.create_datasetrrcCs,z |j||dWdStjyYdSw)anCheck whether a dataset exists in your tenant. Parameters ---------- dataset_name : str or None, default=None The name of the dataset to check. dataset_id : str or None, default=None The ID of the dataset to check. Returns ------- bool Whether the dataset exists. rMTF)rr;r )rrrr,r,r- has_datasetOs zClient.has_datasetcCsd}ddi}|dur|dt|d7}n |dur||d<ntd|j||d }|}t|trSt|d krAtd |d t j di|d |j | d St j di||j | d S)aYRead a dataset from the LangSmith API. Parameters ---------- dataset_name : str or None, default=None The name of the dataset to read. dataset_id : UUID or None, default=None The ID of the dataset to read. Returns ------- Dataset The dataset. rr8rNNr2rrN'Must provide dataset_name or dataset_idr4rrcr%rdr,) rr7r3rkrzrr:r;r rrgrrh)rrrr9r0r rkr,r,r-rfs6        zClient.read_datasetrcCshd}|durn|dur|j|dj}ntd||dt|dd}dd |jd D}|S) aw Download a dataset in OpenAI Jsonl format and load it as a list of dicts. Parameters ---------- dataset_id : str The ID of the dataset to download. dataset_name : str The name of the dataset to download. Returns ------- list The dataset loaded as a list of dicts. rNrrOr2rz /openai_ftcSsg|]}t|qSr,)rkrw)raliner,r,r-rdrz9Client.read_dataset_openai_finetuning..r)rror7r3rtextrr&)rrrr9r r r,r,r-read_dataset_openai_finetuningsz%Client.read_dataset_openai_finetuning) dataset_idsrKrdataset_name_containsrSrTIterator[ls_schemas.Dataset]c#sli}|dur ||d<|dur||d<|dur||d<|dur#||d<fddjd|d DEdHdS) zyList the datasets on the LangSmith API. Yields ------ Dataset The datasets. NrorKrNr c30|]}tjdi|jdVqdSrdNr,)rrgrrhrrr,r-r  z'Client.list_datasets..rr4r<)rrSrKrrTr0r,rr- list_datasetss  zClient.list_datasets)rrcCsX|dur |j|dj}|durtd|jj|jdt|d|jd}t |dS)aDelete a dataset from the LangSmith API. Parameters ---------- dataset_id : UUID or None, default=None The ID of the dataset to delete. dataset_name : str or None, default=None The name of the dataset to delete. Nrz&Must provide either dataset name or IDrrr) rror7rrr0rrr;r )rrrr r,r,r-delete_datasetszClient.delete_dataset)source_api_urlr token_or_urlr\c Cs|p|j}t||\}}t|dd}||}|p|j}|j|dr-td|ddSzEt| |}|j ||j |j p@t jjd}z|jdd |Dd d |D|jd WW~dStyr} z td |d | d} ~ ww~w)a<Clone a public dataset to your own langsmith tenant. This operation is idempotent. If you already have a dataset with the given name, this function will do nothing. Args: token_or_url (str): The token of the public dataset to clone. source_api_url: The URL of the langsmith server where the data is hosted. Defaults to the API URL of your current client. dataset_name (str): The name of the dataset to create in your tenant. Defaults to the name of the public dataset. placeholderrrrcz) already exists in your tenant. Skipping.N)rrJrKcSg|]}|jqSr,rrarr,r,r-rdz/Client.clone_public_dataset..cSr_r,rr`r,r,r-rdrarrrz)An error occurred while creating dataset z . You should delete it manually.)r0r@rrrNrNrrrr rLrJrKrDataTypekvcreate_examplesroryr) rr]r\rr? source_clientdsexamplesr rr,r,r-clone_public_datasetsF           zClient.clone_public_datasetcCs|j|d}|jS)Nr)rrK)rrr r,r,r-r-s zClient._get_data_typeprompt generation created_atls_schemas.ExamplecCs|jd|id|i|||dS),Add an example (row) to an LLM-type dataset.inputoutputrrrrrm)create_example)rrkrlrrrmr,r,r-create_llm_example1s zClient.create_llm_examplemessages:List[Union[Mapping[str, Any], ls_schemas.BaseMessageLike]] generations>Optional[Union[Mapping[str, Any], ls_schemas.BaseMessageLike]]c Csg}|D]}t|r|tttj|q|tt|qd}|dur;t|r6tttj|}ntt|}|jd|i|durHd|ind|||dS)z,Add an example (row) to a Chat-type dataset.Nrprqrr) r;is_base_message_likerconvert_langchain_messagerrBaseMessageLikerQrs) rrurwrrrm final_inputmessagefinal_generationsr,r,r-create_chat_exampleCs.      zClient.create_chat_examplec Cs|dur |j|dj}d}||}|tjjkrm|jdkr&td|jdzt |j }WntyAtd|jd|j wd|i}|j sLd}nzt |j } Wntygtd |jd |j wd | i}nk|tjj kr|jdkrtd|jd z dt|j i}Wntytd |jd|j w|j sd}n3z d t|j i}Wn(tytd|jd |j w|tjjkr|j }|j }ntd|d|j|||||dS)roNrllmz Run type z+ is not supported for dataset of type 'LLM'z2Error converting LLM run inputs to prompt for run z with inputs rpz7Error converting LLM run outputs to generation for run z with outputs rqz, is not supported for dataset of type 'chat'z9Error converting LLM run inputs to chat messages for run z=Error converting LLM run outputs to chat generations for run z Dataset type z not recognized.rr)rrorrrcrrr7r;get_prompt_from_inputsrrget_llm_generation_from_outputschatget_messages_from_inputs#get_message_generation_from_outputsrdrs) rrmrrrm dataset_typerkrrrlr,r,r-create_example_from_runks             zClient.create_example_from_run)rrrSequence[Mapping[str, Any]]/Optional[Sequence[Optional[Mapping[str, Any]]]]c sdur |dur tddur|j|djfddt||p&dgt|D}|jj|jdi|jddit j |t d d }t |dS) aCreate examples in a dataset. Parameters ---------- inputs : Sequence[Mapping[str, Any]] The input values for the examples. outputs : Optional[Sequence[Optional[Mapping[str, Any]]]], default=None The output values for the examples. dataset_id : Optional[ID_TYPE], default=None The ID of the dataset to create the examples in. dataset_name : Optional[str], default=None The name of the dataset to create the examples in. Returns ------- None Raises ------ ValueError If both `dataset_id` and `dataset_name` are `None`. Nz3Either dataset_id or dataset_name must be provided.rcsg|] \}}||dqS)rbr,)rain_out_rjr,r-rdsz*Client.create_examples..z/examples/bulkrrrAr)r7rrozipr:rr=r0rrkrxr`r;r )rrrrrrrhr r,rjr-res   zClient.create_examplesMapping[str, Any]Optional[Mapping[str, Any]]r(c Cs|dur |j|dj}|||d}|r||d<|r||d<tjd i|}|jj|jdi|jddi| d } t | | } tj d i| |j |d S) aCreate a dataset example in the LangSmith API. Examples are rows in a dataset, containing the inputs and expected outputs (or other reference information) for a model or chain. Parameters ---------- inputs : Mapping[str, Any] The input values for the example. dataset_id : UUID or None, default=None The ID of the dataset to create the example in. dataset_name : str or None, default=None The name of the dataset to create the example in. created_at : datetime or None, default=None The creation timestamp of the example. outputs : Mapping[str, Any] or None, default=None The output values for the example. exemple_id : UUID or None, default=None The ID of the example to create. If not provided, a new example will be created. Returns ------- Example The created example. Nrrbrmrorrrrrdr,)rror|r ExampleCreaterr=r0rrkr;r rrrh) rrrrrmrr(rDrBr rkr,r,r-rss.%     zClient.create_examplecCs:|dt|d}tjdi||j|dS)zRead an example from the LangSmith API. Parameters ---------- example_id : str or UUID The ID of the example to read. Returns ------- Example The example. /examples/r(rdNr,)r3rrrrkrrhrr(r r,r,r- read_example+ s  zClient.read_exampleOptional[Sequence[ID_TYPE]]inline_s3_urlsIterator[ls_schemas.Example]c#sxi}|dur ||d<n|durj|dj}||d<n |dur%||d<||d<fddjd|d DEdHdS) aRetrieve the example rows of the specified dataset. Parameters ---------- dataset_id : UUID or None, default=None The ID of the dataset to filter by. dataset_name : str or None, default=None The name of the dataset to filter by. example_ids : List[UUID] or None, default=None The IDs of the examples to filter by. Yields ------ Example The examples. Nr rrorc3rVrW)rrrrh)rarBrr,r-rc rXz'Client.list_examples..rr4)rror<)rrrrrr0r,rr-r9A s    zClient.list_examplesrbcCs\tj|||d}|jj|jdt|di|jddi|jddd}t ||S) aUpdate a specific example. Parameters ---------- example_id : str or UUID The ID of the example to update. inputs : Dict[str, Any] or None, default=None The input values to update. outputs : Mapping[str, Any] or None, default=None The output values to update. dataset_id : UUID or None, default=None The ID of the dataset to update. Returns ------- Dict[str, Any] The updated example. rbrr(rrTrmr) r ExampleUpdaterrr0rrrkr;r )rr(rrrrBr r,r,r-update_examplej s  zClient.update_examplecC2|jj|jdt|d|jd}t|dS)zDelete an example by ID. Parameters ---------- example_id : str or UUID The ID of the example to delete. rr(rNrrr,r,r-delete_example zClient.delete_example9Union[ls_schemas.Run, ls_schemas.RunBase, str, uuid.UUID]cCs*t|ttjfr|j||d}|S|}|S)azResolve the run ID. Parameters ---------- run : Run or RunBase or str or UUID The run to resolve. load_child_runs : bool Whether to load child runs. Returns ------- Run The resolved run. Raises ------ TypeError If the run type is invalid. r)rzrr5r6r)rrmrrun_r,r,r-_resolve_run_id s zClient._resolve_run_idrB5Union[ls_schemas.Example, str, uuid.UUID, dict, None]Optional[ls_schemas.Example]cCst|ttjfr||}|St|tjr|}|St|tr0tjdi||j| d}|S|j dur=||j }|Sd}|S)aBResolve the example ID. Parameters ---------- example : Example or str or UUID or dict or None The example to resolve. run : Run The run associated with the example. Returns ------- Example or None The resolved example. rdNr,) rzrr5r6rrrrQrrhr)rrBrmreference_example_r,r,r-_resolve_example_id s$      zClient._resolve_example_idr>DUnion[ls_evaluator.EvaluationResult, ls_evaluator.EvaluationResults]#List[ls_evaluator.EvaluationResult]cCsTt|tjr |g}|St|tr d|vr tttj|d}|Stdt|d)Nr>zInvalid evaluation result type z1. Expected EvaluationResult or EvaluationResults.)rz ls_evaluatorEvaluationResultrQrr TypeErrorr)rr>results_r,r,r-_select_eval_results s zClient._select_eval_results) source_inforrrls_evaluator.RunEvaluatorrr9Optional[Union[ls_schemas.Example, str, dict, uuid.UUID]]ls_evaluator.EvaluationResultc Cs@|j||d}|||}|j||d}|j|||d} | dS)a0Evaluate a run. Parameters ---------- run : Run or RunBase or str or UUID The run to evaluate. evaluator : RunEvaluator The evaluator to use. source_info : Dict[str, Any] or None, default=None Additional information about the source of the evaluation to log as feedback metadata. reference_example : Example or str or dict or UUID or None, default=None The example to use as a reference for the evaluation. If not provided, the run's reference example will be used. load_child_runs : bool, default=False Whether to load child runs when resolving the run ID. Returns ------- Feedback The feedback object created by the evaluation. rrBrr)rr evaluate_run_log_evaluation_feedback rrmrrrrrrevaluator_responser>r,r,r-r s! zClient.evaluate_runrc Csr||}|D]/}|p i}|jri|j|}|jr|jn|j}|j||j|j|j|j|j ||j t j j d q|S)N)scorercomment correctionr source_run_idfeedback_source_type)revaluator_info target_run_idrocreate_feedbackrrrrrrrFeedbackSourceTypeMODEL)rrrmrr>res source_info_rr,r,r-r s$  zClient._log_evaluation_feedback%Union[ls_schemas.Run, str, uuid.UUID]c sH|j||d}|||}|j||dIdH}|j|||d} | dS)a7Evaluate a run asynchronously. Parameters ---------- run : Run or str or UUID The run to evaluate. evaluator : RunEvaluator The evaluator to use. source_info : Dict[str, Any] or None, default=None Additional information about the source of the evaluation to log as feedback metadata. reference_example : Optional Example or UUID, default=None The example to use as a reference for the evaluation. If not provided, the run's reference example will be used. load_child_runs : bool, default=False Whether to load child runs when resolving the run ID. Returns ------- EvaluationResult The evaluation result object created by the evaluation. rrNrr)rr aevaluate_runrrr,r,r-r7 s!  zClient.aevaluate_runr) rrrrrrr feedback_ideagerrrrUnion[float, int, bool, None]r(Union[float, int, bool, str, dict, None]rUnion[dict, None]rUnion[str, None]r)Union[ls_schemas.FeedbackSourceType, str]rrrls_schemas.Feedbackc  Cst|tjs t|}|tjjkrtj|d} n|tjjkr%tj|d} ntd|| jdur4| jni| _| durId| jvrIdt | i| jd<| jrzd| jvrz| jd}t |drct |rc| }d|vrut t | jddd|d<|| jd<tj| pt||||||| tjtjjtjtjjd }|jd |jd | rd nd tj|j d dtdi|jddd|jdd| tjfdtjdi| S)aCreate a feedback in the LangSmith API. Parameters ---------- run_id : str or UUID The ID of the run to provide feedback on. key : str The name of the metric, tag, or 'aspect' this feedback is about. score : float or int or bool or None, default=None The score to rate this run on the metric or aspect. value : float or int or bool or str or dict or None, default=None The display value or non-numeric value for this feedback. correction : dict or None, default=None The proper ground truth for this run. comment : str or None, default=None A comment about this feedback. source_info : Dict[str, Any] or None, default=None Information about the source of this feedback. feedback_source_type : FeedbackSourceType or str, default=FeedbackSourceType.API The type of feedback source, such as model (for model-generated feedback) or API. source_run_id : str or UUID or None, default=None, The ID of the run that generated this feedback, if a "model" type. feedback_id : str or UUID or None, default=None The ID of the feedback to create. If not provided, a random UUID will be generated. eager : bool, default=False Whether to skip the write queue when creating the feedback. This means that the feedback will be immediately available for reading, but may cause the write to fail if the API is under heavy load, since the target run_id may have not been created yet. stop_after_attempt : int, default=10 The number of times to retry the request before giving up. rxzUnknown feedback source type N__runrrQz+feedback_source.metadata['__run']['run_id']) rorrrrrrfeedback_sourcerm modified_atPOST /feedbackz/eagerrTrmrAr)rrrrC)rrrr,)rzrrAPIAPIFeedbackSourcerModelFeedbackSourcer7rSrrrrQrFeedbackCreater5rqr{nowtimezoneutcr/r0rkrxr`rrr;r Feedback)rrrrrrrrrrrrrr _run_metafeedbackr,r,r-rf sl 4           zClient.create_feedback)rrrrcCsi}|dur ||d<|dur||d<|dur||d<|dur"||d<|jj|jdt|di|jdd itj|td d }t |dS) a[Update a feedback in the LangSmith API. Parameters ---------- feedback_id : str or UUID The ID of the feedback to update. score : float or int or bool or None, default=None The score to update the feedback with. value : float or int or bool or str or dict or None, default=None The value to update the feedback with. correction : dict or None, default=None The correction to update the feedback with. comment : str or None, default=None The comment to update the feedback with. Nrrrr /feedback/rrrrAr) rrr0rrrkrxr`r;r )rrrrrrfeedback_updater r,r,r-update_feedback s zClient.update_feedbackcCs*|dt|d}tjdi|S)zRead a feedback from the LangSmith API. Parameters ---------- feedback_id : str or UUID The ID of the feedback to read. Returns ------- Feedback The feedback. rrNr,)r3rrrrkrrr r,r,r- read_feedback s zClient.read_feedback)r feedback_keyrrOptional[Sequence[str]]1Optional[Sequence[ls_schemas.FeedbackSourceType]]Iterator[ls_schemas.Feedback]cksPd|i|}|dur||d<|dur||d<dd|jd|dDEdHdS) aList the feedback objects on the LangSmith API. Parameters ---------- run_ids : List[str or UUID] or None, default=None The IDs of the runs to filter by. feedback_key: List[str] or None, default=None The feedback key(s) to filter by. Example: 'correctness' The query performs a union of all feedback keys. feedback_source_type: List[FeedbackSourceType] or None, default=None The type of feedback source, such as model (for model-generated feedback) or API. **kwargs : Any Additional keyword arguments. Yields ------ Feedback The feedback objects. rmNrsourcecs |] }tjdi|VqdSNr,)rr)rarr,r,r-r4  z'Client.list_feedback..rr4rY)rrrrrr0r,r,r- list_feedback s zClient.list_feedbackcCr)zDelete a feedback by ID. Parameters ---------- feedback_id : str or UUID The ID of the feedback to delete. rrrNrrr,r,r-delete_feedback9 rzClient.delete_feedback) queue_idsrNr r$Iterator[ls_schemas.AnnotationQueue]ccsJ|durddt|Dnd||d}dd|jd|dDEdHdS)NcSs"g|] \}}t|d|dqS)z queue_ids[])rrar-id_r,r,r-rdR s"z1Client.list_annotation_queues..)idsrNr csrr)rAnnotationQueue)raqueuer,r,r-rY rz0Client.list_annotation_queues../annotation-queuesr4) enumerater<)rrrNr r0r,r,r-list_annotation_queuesI s  zClient.list_annotation_queues)rJqueue_idrls_schemas.AnnotationQueuecCsV|||d}|d|jddd|D|jd}t|tjdi|S)N)rNrJror=rcSrrpr,rfr,r,r-rin ryz2Client.create_annotation_queue..rkrr,) r/r0rrr;r rrrk)rrNrJrr?r r,r,r-create_annotation_queue^ s  zClient.create_annotation_queuecCst|j|gdS)N)r)rFr)rrr,r,r-read_annotation_queueu szClient.read_annotation_queue)rJcCs<|d|jdt|d||d|jd}t|dS)Nr/annotation-queues/r)rNrJr)r/r0rrr;r )rrrNrJr r,r,r-update_annotation_queuey s zClient.update_annotation_queuecCs2|jj|jdt|d|jd}t|dS)Nrrrr)rrr r,r,r-delete_annotation_queue s zClient.delete_annotation_queue List[ID_TYPE]cCsF|d|jdt|ddddt|D|jd}t|dS)Nr=rrrcSs&g|]\}}tt|d|dqS)zrun_ids[r)rrrr,r,r-rd rz7Client.add_runs_to_annotation_queue..r)r/r0rrrr;r )rrrr r,r,r-add_runs_to_annotation_queue s z#Client.add_runs_to_annotation_queue/Iterator[ls_schemas.RunWithAnnotationQueueInfo]ccs<dt|dd}dd|j|d|jidDEdHdS)Nrrrcsrr)rRunWithAnnotationQueueInforr,r,r-r rz9Client.list_runs_from_annotation_queue..rr4)rr<r)rrr9r,r,r-list_runs_from_annotation_queue s z&Client.list_runs_from_annotation_queue) evaluationconcurrency_levelrproject_metadataverbosetags input_mapperrllm_or_chain_factoryr Optional[Any]rrrrrOptional[Callable[[Dict], Any]]c sNzddlm} Wn tytdw| |||||||||| | d IdHS)a Asynchronously run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Optional evaluation configuration to use when evaluating concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to store with the project. verbose: Whether to print progress. tags: Tags to add to each run in the project. input_mapper: A function to map to the inputs dictionary from an Example to the format expected by the model to be evaluated. This is useful if your model needs to deserialize more complex schema or if your dataset has inputs with keys that differ from what is expected by your chain or agent. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the synchronous version, see client.run_on_dataset. Examples -------- .. code-block:: python from langsmith import Client from langchain.chat_models import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import RunEvalConfig # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", RunEvalConfig.Criteria("helpfulness"), RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() await client.arun_on_dataset( "", construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) await client.arun_on_dataset( "", construct_chain, evaluation=evaluation_config, ) r)arun_on_datasetzlThe client.arun_on_dataset function requires the langchainpackage to run. Install with pip install langchain) rrclientrrrrrrrrN)langchain.smithr  ImportError) rrrrrrrrrrr_arun_on_datasetr,r,r-r  s*x  zClient.arun_on_datasetc CsFzddlm} Wn tytdw| |||||||||| | d S)a Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of tasks to execute concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Metadata to store with the project. verbose: Whether to print progress. tags: Tags to add to each run in the project. input_mapper: A function to map to the inputs dictionary from an Example to the format expected by the model to be evaluated. This is useful if your model needs to deserialize more complex schema or if your dataset has inputs with keys that differ from what is expected by your chain or agent. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see `client.arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain.chat_models import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import RunEvalConfig # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", RunEvalConfig.Criteria("helpfulness"), RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() client.run_on_dataset( "", construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) client.run_on_dataset( "", construct_chain, evaluation=evaluation_config, ) r)run_on_datasetzkThe client.run_on_dataset function requires the langchainpackage to run. Install with pip install langchain) rrrr rrrrrrr)r rr ) rrrrrrrrrrr_run_on_datasetr,r,r-r5 s(y zClient.run_on_datasetrp)r0rrrrrrrrrrrrr!r r)r r)r r)r r)rrr0rrrWr r)rNNN)rrrrrrrrWrrrrr r)r9rr0r1r r)r9rr0r5r r6) r9rr?r5rrr@rr r6)rLrMrNrrOrPrQrPrJrrKrRr rS)rZr]rOrPrQrPrNrrJrrKrRr rS)rmrnr rQ)r>rur r)r>rrr!r r)rNrrrrrrrrrrr r r)NN)rrr~rrr!)rrrrrrrrrrrrrr r r)rmrr r)F)rrrr!r r)rrrrrrrrrrrrrrrrrprrrrrrr r r)rmrrrrrr r)rrrrr r)rrr r)rrr r)rrr r!)rrrrr r)rrrrr r)rrr r)rrr rS)rrrrr r) rrr rrNrr rr r)rrrJrrSr5rr!rr5rrr r)rrrNrrJrrSr5rr5rrr r)r r)rrrrr"r!r r#)rrrrr r!)rrrrr r')NNNNNN)r rrNrr rrrrDrrErr rF)rrrrr r)rrrJrrKrJr rS)rrrrr r!)rrrrr rS)rrrrr r) rSrrKrrrrTrr rU)rrrrr r)r]rr\rrrr r)rrr rJ)NNNN) rkrrlrrrrrrmrr rn) rurvrwrxrrrrrmrr rn)NNN) rmrrrrrrmrr rn) rrrrrrrrrr r r)NNNNN)rrrrrrrmrrrr(rr rn)r(rr rn)NNNT) rrrrrrrr!r r) r(rrr1rrrrr r)r(rr r)rmrrr!r r)rBrrmrr r)r>rr r) rmrrrrr1rrrr!r r)rrrmrrr1r r) rmrrrrr1rrrr!r r)rrrrrrrrrrrrrr1rrrrrrrr!rrWr r) rrrrrrrrrrr r)rrr r) rrrrrrrr r r)rrr r)rrrNrr rr r)rNrrJrrrr r)rrr r)rrrNrrJrr r)rrr r)rrrrr r)rrr r)rrrr rrrrWrrrr1rr!rrrr rrr r)_rrr__doc__rorrrpropertyrrr staticmethodrrrr/r3r<rIrrcrdr[rYrtrrrrrrrrrrrrrrrrrrr rrrrhr;xor_argsrr&rCrHrIrLrNrrRrZr[rirrtrrrersrr9rrrrrrrrrrrrrrrrrrrrrrr rr,r,r,r-r?s F    |&87NT b K%S&     : 1 .G:)."  @*S4<-'!5 4t()rr9rrr8blockList[TracingQueueItem]cCsg}z6|j|dd}r|||j|dd}r6|||r*t||kr*W|S|j|dd}sW|SW|StyBY|Sw)Ng?)rr g?)rrr:r)rr8r next_batchrr,r,r-_tracing_thread_drain_queue s"   rr rAc Csrdd|D}dd|D}z z |j||ddWn ty"YnwW|D]}|q&dS|D]}|q1w)NcSg|] }|jdkr|jqS)rrrraitr,r,r-rd r*z0_tracing_thread_handle_batch..cSr)r~rrr,r,r-rd r*T)rr~r)rrB task_done)r rrArr~_r,r,r-_tracing_thread_handle_batch s   rrrrls_schemas.BatchIngestConfigcCsFtjdtttd}|s |Sz |js|WS|jWSty"|YSw)Nr9) size_limitscale_up_nthreads_limitscale_up_qsize_triggerscale_down_nempty_trigger)rBatchIngestConfig_AUTO_SCALE_UP_NTHREADS_LIMIT_AUTO_SCALE_UP_QSIZE_TRIGGER_AUTO_SCALE_DOWN_NEMPTY_TRIGGERbatch_ingest_configry)rdefault_configr,r,r-_ensure_ingest_config s r- client_refweakref.ref[Client]c Cs|}|dur dSz|jstd|jtjddWdSWnty6}z td|WYd}~dSd}~ww|j}|dus@Jt|j}|d}|d}|d}g}d} t rt || t |kr|D] } | ss|| qht ||kr||krt jtt|fd } || | t||d } rt||| t rt || t |ksft||d d } rt||| t||d d } sdSdS) Nz no info: T)r_flush#Error in tracing control thread: %sr#r$r%rDrr8Fr8r)rprintsysstderrryrrrr-r main_threadis_alive getrefcountr:rqsizer_tracing_sub_thread_funcrrrrrr) r.r rrr+r#r$r% sub_threadsnum_known_refsthread new_threadrr,r,r-r s^            rc Cs|}|dur dSz|jsWdSWnty*}z td|WYd}~dSd}~ww|j}|dus4Jt|j}|dd}d}t rn||dkrnt ||d}r^d}t |||n|d7}t rn||dksMt ||dd }rt |||t ||dd }swdSdS) Nr1r#r9rr&r2rNFr3) rryrrrr-rrr7r8rr)r.r rrr+r#seen_successive_empty_queuesrr,r,r-r;3 sL        r;)rrr r!)r/rr0rr r1)r r)r)r\r r]rWr r )rrr r)r0rrrr r)r r)rrr r)r0rrrr r)rrr r)rrr r)rrrrr r)r9T)rrr8rWrr!r r)r rrrrArr r)rrr r")r.r/r r)ar __future__rrrr{rrRrUrkloggingrr&r'r5rr'r5rrrrrrrtypingrr r r r r rrrrrrrrrurllibrr#rrr urllib3.utilrrrrzrrrr;langsmith.evaluationrrr4r= getLoggerrr addFilterFilterPoolFullWarningr.r@rCr6rrr[rWrr!r}rvr`rrrrrrrrrrrrr)r(r*r-rr;r,r,r,r-s D               8          2