o /ifQ@s(ddlmZmZmZddlmZmZddlmZddl m Z m Z m Z ddl Z ddlZddlZddlZddlZddlZddlmZddlmZmZddlmZmZmZdd lmZmZmZdd l m!Z!m"Z"dd l#m$Z$m%Z%dd l&m'Z'dd l(m)Z)ddgZ*dee$e%fde+fddZ,eGdddZ-dS))ListOptionalUnion) dataclassfield)Console)Progress SpinnerColumn TextColumnN) BaseMetric)Api Endpoints)convert_test_cases_to_goldensconvert_goldens_to_test_cases)convert_convo_goldens_to_convo_test_cases) APIDatasetCreateDatasetHttpResponseDatasetHttpResponse)GoldenConversationalGolden) LLMTestCaseConversationalTestCase) is_confident)BaseSynthesizercsvjson test_casesubjectcCs,t|tst|tstd|ddSdS)Nz Provided `z9` must be a list of LLMTestCase or ConversationalTestCase) isinstancerr TypeError)rrr Q/var/www/html/corbot_env/lib/python3.10/site-packages/deepeval/dataset/dataset.pyvalidate_test_case_type s  r"c@sheZdZUeeed<eeed<eddZe e dfed<eddZ e e dfed<egddZ ee ed<egddZeeed <gggfd ee e efdeedeefd d Zd dZedee e effddZejd ee e effddZde e effddZddZdeefddZ     dNde de de dee d ee d!e d"ee d#e fd$d%Z   dOde d&e d'e d(ee d)ee d*ee f d+d,ZdPd-e d.eefd/d0ZdQd-e d2efd3d4Z  5 6 7  1dRd8eee d9ed:e d;e defd?d@Z!  A B C 6 7dSdDee d9edEe dFe dGe d;e dt||_| |q||_ ||_ dSNz test cases)r) r"r$r%r(r)rrlen _dataset_rankappendrr,r-)selfr.r$r%rllm_test_casesconversational_test_casesr r r!__init__6s$       zEvaluationDataset.__init__c Cs6|jjd|jd|jd|jd|jd|jd S)Nz (test_cases=z , goldens=z, conversational_goldens=z , _alias=z, _id=)) __class____name__r.r$r%r(r)r3r r r!__repr__PszEvaluationDataset.__repr__returncCs |j|jSN)r,r-r:r r r!r.Xs zEvaluationDataset.test_casescCst|ts tdg}g}|D]4}t|tst|tsq|j|_|j|_t|tr4t ||_ | |qt|trCt ||_ | |q||_ ||_ dS)Nz'test_cases' must be a list)rlistrrrr(_dataset_aliasr) _dataset_idr0r1r2r,r-)r3r.r4r5rr r r!r.\s(         rcCslt|dd|j|_|j|_t|tr!t|j|_ |j |dSt|t r4t|j |_ |j |dSdSr/) r"r(r?r)r@rrr0r,r1r2rr-)r3rr r r! add_test_casews     zEvaluationDataset.add_test_casecCs t|jSr=)iterr.r:r r r!__iter__s zEvaluationDataset.__iter__metricscCs.ddlm}t|jdkrtd||j|S)Nr)evaluatezLNo test cases found in evaluation dataset. Unable to evaluate empty dataset.)deepevalrEr0r. ValueError)r3rDrEr r r!rEs  zEvaluationDataset.evaluate; file_pathinput_col_nameactual_output_col_nameexpected_output_col_namecontext_col_namecontext_col_delimiterretrieval_context_col_nameretrieval_context_col_delimiterc  szddl} Wn tytdwdd| jdtfdd} | |} | | |} | | |} | | |dd}fd d | | |d dD}fd d | | |d dD}t| | |||D]\}}}}}|t|||||d qWdS)ab Load test cases from a CSV file. This method reads a CSV file, extracting test case data based on specified column names. It creates LLMTestCase objects for each row in the CSV and adds them to the Dataset instance. The context data, if provided, is expected to be a delimited string in the CSV, which this method will parse into a list. Args: file_path (str): Path to the CSV file containing the test cases. input_col_name (str): The column name in the CSV corresponding to the input for the test case. actual_output_col_name (str): The column name in the CSV corresponding to the actual output for the test case. expected_output_col_name (str, optional): The column name in the CSV corresponding to the expected output for the test case. Defaults to None. context_col_name (str, optional): The column name in the CSV corresponding to the context for the test case. Defaults to None. context_delimiter (str, optional): The delimiter used to separate items in the context list within the CSV file. Defaults to ';'. retrieval_context_col_name (str, optional): The column name in the CSV corresponding to the retrieval context for the test case. Defaults to None. retrieval_context_delimiter (str, optional): The delimiter used to separate items in the retrieval context list within the CSV file. Defaults to ';'. Returns: None: The method adds test cases to the Dataset instance but does not return anything. Raises: FileNotFoundError: If the CSV file specified by `file_path` cannot be found. pd.errors.EmptyDataError: If the CSV file is empty. KeyError: If one or more specified columns are not found in the CSV file. Note: The CSV file is expected to contain columns as specified in the arguments. Each row in the file represents a single test case. The method assumes the file is properly formatted and the specified columns exist. For context data represented as lists in the CSV, ensure the correct delimiter is specified. rNz>Please install pandas to use this method. 'pip install pandas'dfcol_namecSs"||jvr ||jS|gt|Sr=)columnsvaluesr0)rQrRr'r r r!get_column_datas  zGEvaluationDataset.add_test_cases_from_csv_file..get_column_datar&cg|] }|r |ngqSr split).0context)rNr r! szBEvaluationDataset.add_test_cases_from_csv_file..crVr rW)rYretreival_context)rPr r!r[s  input actual_outputexpected_outputrZretrieval_contextr=)pandasModuleNotFoundError DataFramestrread_csvziprAr)r3rIrJrKrLrMrNrOrPpdrUrQinputsactual_outputsexpected_outputscontextsretrieval_contextsr_r`rarZrbr )rNrPr!add_test_cases_from_csv_filesZ%        z.EvaluationDataset.add_test_cases_from_csv_fileinput_key_nameactual_output_key_nameexpected_output_key_namecontext_key_nameretrieval_context_key_namec Cszt|d }t|}Wdn1swYWnty+td|dtjy:td|dw|D]1} || vsG|| vrKtd| |} | |} | |} | |} | |}|t| | | | |dq=dS)a Load test cases from a JSON file. This method reads a JSON file containing a list of objects, each representing a test case. It extracts the necessary information based on specified key names and creates LLMTestCase objects to add to the Dataset instance. Args: file_path (str): Path to the JSON file containing the test cases. input_key_name (str): The key name in the JSON objects corresponding to the input for the test case. actual_output_key_name (str): The key name in the JSON objects corresponding to the actual output for the test case. expected_output_key_name (str, optional): The key name in the JSON objects corresponding to the expected output for the test case. Defaults to None. context_key_name (str, optional): The key name in the JSON objects corresponding to the context for the test case. Defaults to None. retrieval_context_key_name (str, optional): The key name in the JSON objects corresponding to the retrieval context for the test case. Defaults to None. Returns: None: The method adds test cases to the Dataset instance but does not return anything. Raises: FileNotFoundError: If the JSON file specified by `file_path` cannot be found. ValueError: If the JSON file is not valid or if required keys (input and actual output) are missing in one or more JSON objects. Note: The JSON file should be structured as a list of objects, with each object containing the required keys. The method assumes the file format and keys are correctly defined and present. rNz The file z was not found.z is not a valid JSON file.z7Required fields are missing in one or more JSON objectsr^) openrloadFileNotFoundErrorJSONDecodeErrorrGgetrAr)r3rIrprqrrrsrtfile json_listjson_objr_r`rarZrbr r r!add_test_cases_from_json_files>      z/EvaluationDataset.add_test_cases_from_json_filealias overwritec Cst|jdkrt|jdkrtdtrs|j}|t|jt||||jd}z |j ddd}Wnt yB|j ddd}Ynwt }|j tjj|d}|rqt|dd}|j} t} | d | d | d t| dSdStd ) NrzgUnable to push empty dataset to Confident AI, there must be at least one test case or golden in dataset)rrr$conversationalGoldensT)by_alias exclude_none)endpointbodylink)ru?✅ Dataset successfully pushed to Confidnet AI! View at [link=]z[/link]z5To push dataset to Confident AI, run `deepeval login`)r0r.r$rGrextendrrr% model_dumpAttributeErrordictr post_requestr DATASET_ENDPOINTvaluerrrprint webbrowserrv Exception) r3rrr$ api_datasetrapiresultresponserconsoler r r!push4sR  zEvaluationDataset.pushT"auto_convert_goldens_to_test_casesc Cs4trt}ttddtdddz}|jd|ddd }t}|jt j j d |id }t |d |d |dd}||_ |j|_|r`t|j||j} t|j||j} |j| |j| n|j|_|j|_t} t| |d} |j||j|jd| ddWddS1swYdStd)Nzrgb(106,0,255))stylez([progress.description]{task.description}F) transientzPulling [rgb(106,0,255)]'z''[/rgb(106,0,255)] from Confident AI...d)totalr)rparamsr$r datasetId)r$rrz.2fz [rgb(25,227,160)]Done! (zs)) descriptionz6Run `deepeval login` to pull dataset from Confident AI)rr rr r add_tasktime perf_counter get_requestr rrrr(rr)rr$rr%r,rr-formatupdatetasksrr) r3rrrprogresstask_id start_timerrr4r5end_time time_takenr r r!pull]sf   "4zEvaluationDataset.pullFrm synthesizermax_goldens_per_contextnum_evolutionsenable_breadth_evolve source_files_show_indicatorc C<ddlm}|dur |}|j|j||||||ddS)Nr Synthesizer)rmrrrrr)deepeval.synthesizerrr$rgenerate_goldens) r3rmrrrrrrrr r r!r z"EvaluationDataset.generate_goldensrdocument_pathsmax_goldens_per_document chunk_size chunk_overlapc Cr)Nrr)rrrrrr)rrr$rgenerate_goldens_from_docs) r3rrrrrrrrr r r!rrz,EvaluationDataset.generate_goldens_from_docs file_type directoryc Csf|tvrtddddtDt|jdkr!td|tjdd|}tj |s9t |tj ||}|d krjt |d }d d |jD}t j||d dWdn1sdwYn?|dkrt |d dd,}t|}|gd|jD]}||j|j|jd|j|jgqWdn1swYtd|d|S)Nz4Invalid file type. Available file types to save as: z, css|]}|VqdSr=r )rYtyper r r! sz,EvaluationDataset.save_as..rzVNo synthetic goldens found. Please generate goldens before attempting to save data as z %Y%m%d_%H%M%S.rwcSs&g|]}|j|j|j|j|jdqS)r_r`rarZ source_filer)rYgoldenr r r!r[sz-EvaluationDataset.save_as..)indentrr\)newliner|zEvaluation dataset saved at !)valid_file_typesrGjoinr0r$datetimenowstrftimeospathexistsmakedirsrvrdumprwriterwriterowr_r`rarZrr) r3rr new_filenamefull_file_pathr{ json_datarrr r r!save_assP        zEvaluationDataset.save_as)NNrHNrH)NNNr=)T)NrrFNT)NrrrrF)$r9 __module__ __qualname__rr__annotations__rrr(rrfr)r,rr-rr6r;propertyr.setterrArCr rError~boolrrrintrrrr r r r!r#+s       e B)>    r#).typingrrr dataclassesrr rich.consoler rich.progressrr r rrrrrrdeepeval.metricsr deepeval.apir r deepeval.dataset.utilsrrrdeepeval.dataset.apirrrdeepeval.dataset.goldenrrdeepeval.test_caserrdeepeval.utilsr%deepeval.synthesizer.base_synthesizerrrrfr"r#r r r r!s4