o
    g׬                 
   @   s  d dl mZ d dlmZ d dlmZ d dlmZmZ d dl	m
Z
 d dlZd dlmZmZmZ d dlmZmZ d d	lmZ d dlZd dlZd dlZd d
lmZ d dlmZ d dlZd dlmZmZmZ d dlmZ d dl m!Z! d dl"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z. d dl-m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4Z4d dl5m6Z6 d dlm7Z7m8Z8 d dl9m:Z: d dl;m<Z< d dl=m>Z> d dl?m@Z@ d dlmAZA d dlBmCZC d d lDmEZE d d!lFmGZG ejHIejHJejHKeL d d"lMmNZN d dlZd d#lOmPZP eP ZQd d"lRmNZS d dlTZTd d$lUmVZVmWZW d d%l mXZX d d&lYmZZZ d dl[Z[d dl\Z\d dl=Z=d dl]Z]e]^ Z_d'Z`d(Zad)Zbd*Zcd+d, Zddd-d.Zed/d0 Zfdd2d3Zge
d4d5 Zhe
d6d7 Zie
d8d9 Zje
d:d; Zke
d<d= Zle
d>d? Zmd@dA ZndBdC Zoe<dDejpdE< e( Zqe<dFZre<dGZse<dHZte<dIZue<dJZve<dKZwe<dLZxe7dMdNdOdPZydQdR Zzze*eyg dSeqe*{ dTZ|W n* e}y Z~ ze_dUe~  ez re*eyg dSeqe*{ dTZ|W Y dZ~[~ndZ~[~ww dVdW ZdXdY ZdZd[ Zd\d] Zd^d_ Zd`da Zdbdc Zddde Zdfdg Zdhdi ZddkdlZdmdn Zdodp Zdqdr Zdsdt Zdudv Zdwdx Zdydz Zd{d| Zd}d~ Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdededefddZdd Zdd Zdd Zdd Ze
dd Ze
dd Ze
dd Ze
dd Zdd Ze
dd Ze
dd Ze
dd Ze
dd Ze
dd Ze
dd Zdd Zdd Zdd ZdS )    )render)login_required)User)JsonResponseHttpResponse)csrf_exemptN)	timedeltadatetimetimezone)timesleep)
ChatOpenAI)List)dumps)BaseMessagemessage_to_dictmessages_from_dict)glob)floor)ChatPromptTemplatePromptTemplate)create_stuff_documents_chain)OpenAIEmbeddings)ElasticsearchStore)ConversationalRetrievalChain)ConversationBufferMemory)ConversationBufferWindowMemory)HumanMessage)	AIMessage)HTTPBasicAuth)ElasticsearchNotFoundError)OrderedDict)config)OpenAI)Image)BytesIO)	send_mail)SystemMessage)ChatMessage)Document)DocumentLoader)authenticatelogin)redirect)reversez https://scicdev.crm.dynamics.comz$6e5989aa-4fd1-4015-8648-8f75609d607bz(mUv8Q~1u3JtjdAdw.ov1sRfjzTm3yS.f~cC6wcQSz$95f69e7d-1811-4ab9-9b5a-eba95d3eba9bc                 C   sP   d| d}d| |t d}tj||d}|jdkr | dS td|j )	Nz"https://login.microsoftonline.com/z/oauth2/tokenclient_credentials)
grant_type	client_idclient_secretresource)data   access_tokenzFailed to retrieve token: )base_urlrequestspoststatus_codejsonget	Exceptiontext)r2   r3   	tenant_id	token_urlpayloadresponse rD   3/var/www/html/AlliBotV5_chatbot/AlliBotV50/views.py	get_tokenJ   s   
rF   c                 C   s   d}g }| r| d|  d |r| d| d |r%| d| d |r,d|nd}|r7|d| 7 }tj||d	}|jd
krG| S td|j d|j )Nz7https://scicdev.crm.dynamics.com/api/data/v9.2/contactszemailaddress1 eq ''zadx_identity_username eq 'ztrit_tnapassword eq 'z and  z	?$filter=)headersr6   zFailed to fetch data:  - )appendjoinr9   r=   r;   r<   r>   r?   )emailusernamepasswordrI   urlfilter_queryfilter_expressionrC   rD   rD   rE   #get_data_by_email_username_passwordX   s   
rS   c                 C   s   d| v sd| v r
dS dS )Nzscic.comriskeducationTFrD   )rM   rD   rD   rE   is_valid_emails   s   rU   contactsc                 C   sX   d| d|  d}d|i}t j|||d}|jdkr td d S td|j d	|j )
Nz/https://scicdev.crm.dynamics.com/api/data/v9.2/()trit_tnapassword)rI   r<      zPassword updated successfully.zFailed to update password: rJ   )r9   patchr;   printr>   r?   )	record_idnew_passwordrI   entityrP   rB   rC   rD   rD   rE   update_passwordz   s   
r`   c                 C   s6   | j d}| j d}|rt| d||dS tdS )NnamerM   z
index.html)	user_namemailidr-   )sessionr=   r   r.   )requestra   rM   rD   rD   rE   chatbot   s
   rf   c              
   C   s  | j dkr| jd }| jd }| jd }td| td| td|  zDt|r\tttt	}d| dd}t
||||d	}t|d
 dkrS|| jd< || jd< n	t| dddiW S td| d ttW S  ty } ztd| W  Y d }~S d }~ww t| dS )NPOSTrN   rO   rM   username%s Bearer application/jsonAuthorizationzContent-Type)rM   rN   rO   rI   valuer   ra   
login.htmlerrorzInvalid credentialszUser z4 authenticated successfully. Redirecting to chatbot.An error occurred: )methodrg   r\   loggerinforU   rF   r2   r3   r@   rS   lenrd   r   r.   rf   r>   r   )re   rN   rO   rM   tokenrI   recorderD   rD   rE   
login_view   s2   








rx   c              
   C   s   | j dkrq| jd}| jd }zGtttt}d| dd}t|||d}t|d d	krOd
}|| 	d7 }t
d| td|d|gdd t| dddiW S t| dddiW S  typ } ztd| W  Y d }~S d }~ww t| dS )Nrg   rN   rM   ri   rj   rk   rM   rN   rI   rm   r   z.Click the link below to reset your password:

z/reset-password/messagezPassword Reset Requestztna@scic.comF)subjectrz   
from_emailrecipient_listfail_silentlyzforgot_password.htmlz'Password reset link sent to your email.ro   User not found.rp   )rq   rg   r=   rF   r2   r3   r@   rS   rt   build_absolute_urir\   r'   r   r>   r   )re   rN   rM   ru   rI   rv   rz   rw   rD   rD   rE   forgot_password_view   s6   



r   c           	   
   C   s   | j dkrt| jd}| jd}| jd }zDt|rVtttt}d| dd}t|||d}t	|d	 d
krQ|d	 d
 d }t
|||dd t| dddiW S tdW S tdW S  tys } ztd| W  Y d }~S d }~ww t| dS )Nrg   rN   r^   rM   ri   rj   rk   ry   rm   r   	contactidrV   )r_   rn   rz   z%Password has been reset successfully.r   rp   zreset_password.html)rq   rg   r=   rU   rF   r2   r3   r@   rS   rt   r`   r   r   r>   )	re   rN   r^   rM   ru   rI   rv   r]   rw   rD   rD   rE   reset_password_view   s,   




r   c                 C   s   | j d= td}td|dS )Nra   r-   zSignout successful)rz   redirect_url)rd   r/   r   )re   	login_urlrD   rD   rE   signout
  s   r   c           
   
   C   s>  | j dkr| jd }| jd }td| td|  zhtjddddd	d
|iid	d|iigiiid}g }|d d d dkr}|d d D ]7}|d }|dddkrVd}n|dddkrad}nd}|d|d|d|d|d}|| qEt	d|iW S  t
y }	 ztd|	 W  Y d }	~	S d }	~	ww d S )Nrg   rN   useridrh   notificationsqueryboolmusttermusername.keywordzuser_id.keywordindexbodyhitstotalrm   r   _sourcere      Approved   DeclinedPendinguser_idrz   
created_at)ra   r   rz   r   re   userlistrp   )rq   rg   r\   rr   rs   essearchr=   rK   r   r>   r   )
re   rN   userIdrC   r   hitsourcerequest_staformatted_docrw   rD   rD   rE   get_statuslist  sP   





r   c                 C   sZ   dddd| iidd|iigiidddd	iigd
}t j||d}dd |d d D }|S )Nr   r   r   r   zcandidate_name.keywordi'  lastmodified_atorderasc)r   sizesortr   c                 S   s   g | ]}|d  qS )r   rD   ).0r   rD   rD   rE   
<listcomp>\      z4get_messages_by_user_and_session.<locals>.<listcomp>r   r   r   )rN   ind_namecustomerr   rC   messagesrD   rD   rE    get_messages_by_user_and_sessionI  s   


r   c                 C   sP   t  }| D ]}|dd}tj|}tj|d }|| qd|}|S )Nr   rH   r   z, )setr=   ospathbasenamesplitextaddrL   )	meta_dataunique_sourcesitemr   filenamename_without_extunique_sources_strrD   rD   rE   unique_sources_with_pages_  s   
r   
SECRET_KEYOPENAI_API_KEYes_index_namechatname_indextextlensindexcompareindex
formsindexriea_org_indexriea_token_indexz^https://elastic:8oKIqy312EBsAPzWT64NUzji@scic-elasticsearch.es.us-central1.gcp.cloud.es.io:443i,  T)request_timeoutretry_on_timeoutc                  C   s*   t dd} tjd| d}|jdkrdS dS )Nelastic8oKIqy312EBsAPzWT64NUzjiz9https://scic-elasticsearch.es.us-central1.gcp.cloud.es.io)authr6   TF)r   r9   r=   r;   )basicrC   rD   rD   rE   check_elastic_status  s
   

r   docx_new_pagepdf_new_pageadditional_commercial_formsdocx_pdf_pagebotcoach_indexhtml_unstructured1excel_json_indpdf_json_indiso_forms_pdf_full_pdfiso_forms_pdf_pagewisejson_otherses_connection
index_name	embeddingstrategyException occured due to %sc                 C   sT   ddd|iid}t j| |d}d}|d d D ]}|d }d	|v r'|d	}q|S )
N  r   session_id.keyword)r   r   r   rH   r   r   chat_id_name)r   r   r=   )r   
session_idr   rC   r   r   r   rD   rD   rE   get_chat_id_names_by_session_id  s   

r   c	                 C   s`   | ||||t t d t t d tjtt|ddd|||d}	tj||	d}
|
d}|S )N  contentTensure_asciicandidate_name	modelnamerN   r   user_promptr   r   historyzhuman tokenis_safeflagged_categoriesr   _id)	roundr   r<   r   r   r   r   r   r=   )r   rN   r   r   r   
user_tokenr   r   r   docresdocument_idrD   rD   rE   add_user_message_to_es  s$   
r   c	                 C   sX   | ||||t t d t t d tjtt|dddd|||d}	tj||	d}
|
S )Nr   user)r   roleTr   r   r   )r   r   r<   r   r   r)   r   r   )r   rN   r   r   r   r   r   r   r   r   r   rD   rD   rE   add_chat_message_to_es  s"   r   c                 C   T   | ||||t t d t t d tjtt|ddd|dd
}tjt|d}|S Nr   r   Tr   null)
r   r   rN   r   answerr   r   r   zassistant tokenfeedbackr   )	r   r   r<   r   r   r   r   r   r   r   rN   r   r   assistant_tokenr   r   r   rD   rD   rE   add_assistant_message_to_es      r  c                 C   r   r   )	r   r   r<   r   r   r(   r   r   r   r   rD   rD   rE   add_system_message_to_es  r  r  c                 C   s8   ddd|iidddiigd}t j| |d}|d	 d	 S )
Nr   matchr   r   r   r   )r   r   r   r   r   r   )r   r   r   rC   rD   rD   rE   search_by_session_id  s   r  c                 C   sz   t | drd| jv rtj| jd }|| jd< t| tr*| dd| dddS | jt | dr9t	
| jdS ddS )Nmetadatar   page_contentrH   r  r  )hasattrr  r   r   r   
isinstancedictr=   r  r<   r   )r   	file_namerD   rD   rE   document_to_dict  s   



r  c           	   	   C   sR   dd |D }| ||t t d t t d tj|dd|d}tj||d}|S )Nc                 S      g | ]}t |qS rD   )r  r   r   rD   rD   rE   r   5  r   z)add_document_ai_to_es.<locals>.<listcomp>r   Tr   )rN   r   r   r   r   r   r   r   )r   r   r<   r   r   r   )	rN   r   docs1r   orderidfiledocuments_dictsr   r   rD   rD   rE   add_document_ai_to_es3  s   r  c                 C   s4   t t  d }dd|ii}tj| ||d d S )Nr   r   r   r   idr   )intr	   now	timestampr   update)r   doc_idcurrent_timestampupdate_bodyrD   rD   rE   update_lastmodified_atG  s   r  c                 C   s*   t | |}|D ]}|d }t| | qd S )Nr   )r  r  )r   r   	documentsr   r  rD   rD   rE   update_create_at_with_sessionidU  s
   
r!     r#  c                 C   sX   t | }||t j t }|j|dd | W  d    S 1 s%w   Y  d S )NBMPformat)r%   open	thumbnailLANCZOSr&   savegetvalue)
image_pathr   imgbufferrD   rD   rE   resize_image^  s   $r/  c                 C   s   t | dS )Nutf-8)base64	b64encodedecode)
image_datarD   rD   rE   encode_imagee  s   r5  c                   C   s   t   S N)r	   utcnow	isoformatrD   rD   rD   rE   get_current_iuser_idso_dateh  s   r9  c                 C   s   | j ||d d S )N)r   document)r   )r   r   
chat_entryrD   rD   rE   store_chat_historyk  s   r<  c                 C   s2   t j| |d}|d r|d }|d |d fS dS )Nr   r  foundr   r   chat_history)NN)r   r=   )	indexnamer   rC   	chat_datarD   rD   rE   retrieve_chat_datan  s
   rB  c                 C   sB   ||t t d t t d t|t|g| d}t||| d S )Nr   )rN   r   r   r   r   r-  r   r   img_message_to_dictr<  )base64_imager   rz   rC   r   index_name_imgr   r;  rD   rD   rE   chat_hist_imgz  s   rG  c                 C   s   t | tr| j}t |trddd |D }dd|idS t | tr<| j}t |tr5ddd |D }dd|idS td	t|  )
N c                 S       g | ]}|d  dkr|d qS typer?   rD   r   partrD   rD   rE   r          z'img_message_to_dict.<locals>.<listcomp>humanr   rK  r5   c                 S   rI  rJ  rD   rL  rD   rD   rE   r     rN  aizUnexpected message type: )r  r   r   listrL   r   
ValueErrorrK  )rz   r   rD   rD   rE   rD    s   



rD  c                 C   sd   t | trdd | D S | d}|dkrt| d d dS |dkr+t| d d dS td	| )
Nc                 S   r  rD   )img_message_from_dict)r   mrD   rD   rE   r     r   z)img_message_from_dict.<locals>.<listcomp>rK  rO  r5   r   r   rQ  zGot unexpected message type: )r  rR  r=   r   r   rS  )message_dictmessage_typerD   rD   rE   rT    s   

rT  c                 C   s(   | j |dd|iiddd}|d d S )Nr  r   r   r   r   r   r   )r   )r   r   r   r   rD   rD   rE   retrieve_chat_history  s   
rY  c                 C   s@   | |t t d t t d t|t|gd}t||| d S )Nr   )rN   r   r   r   r   rC  )r   r   rz   rC   r   r   r;  rD   rD   rE   chat_hist_txt  s   rZ  c                 C   B   t | d}t| dW  d    S 1 sw   Y  d S Nrbr0  r'  r1  r2  readr3  )r,  
image_filerD   rD   rE   encode_image1     $ra  c                 C   r[  r\  r^  )r  r`  rD   rD   rE   avi_to_base64  rb  rc  c                 C   s   t | O}d}||t j |jdkr4|d}| dd}|j|dd t	|  |W  d    S | d|j
  d|j
  }|| |W  d    S 1 sWw   Y  d S )	N)   rd  r$  RGB.bmp.jpegJPEGr%  .)r%   r'  r(  r)  r&  convertreplacer*  r   removelower)r,  r-  max_sizeconverted_pathrD   rD   rE   resize_and_convert_image  s   


$
$rp  c              
   C   s  t j| \}}d}d}z| dkrt| }n| dkr%t| }nx| dkr1t| }nq| dv r=t| }nj| dkrIt	| }nc| dkrUt
| }n\| d	krat| }nU| d
krmt| }nN| dkryt| }nG| dkrt| }n@| dkrt| }n9td| W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS  ty } zd}W Y d }~||fS d }~ww )N   rH   z.xlsxz.csvz.tsv)z.docz.docxz.pdfz.pptxz.xmlz.jsonz.txtz.mdz.htmlzUnsupported file type: r   )r   r   r   rm  document_loaderexcel_file_loadercsv_file_loadertsv_file_loaderdoc_docx_file_loaderpdf_file_loaderpptx_file_loaderxml_file_loaderjson_file_loadertext_file_loadermarkdown_file_loaderhtml_file_loaderrS  r>   )
file_name1_extension_file_name1respr  rw   rD   rD   rE   add_documents  sl   
r  c                 C   r[  r\  r^  )	file_pathr  rD   rD   rE   encode_file  rb  r  c                 C   s   |  dd }tdd| ddddd	t|  id
gd}td t|}td t|||g|| td tddd	t|  id
gdS )N/r?   zExtracted text in the z imagerJ  	image_urlrP   data:image/jpeg;base64,rK  r  r   zimg up enteredzdict conv completedzcheck erro in im up)splitr   r  rr   rs   message_to_dict_imgindex_chat_message)r  	sessionidrN   r@  	candidater   rz   
image_convrD   rD   rE   image_upload  s&   
	

r  c              
   C   s   zt j|| d}|d d }W n tjy   g }Y nw |dd|ii t j|| | |tt d tt d ||ddd	d
 d S )Nr=  r   r   r5   r   r   )r   rN   r   r   r   r   Tr   doc_as_upsertr  )r   r=   elasticsearchr!   rK   r  r   r   )r  rN   new_contentr@  r  r   existing_messagesrD   rD   rE   r  !  s0   
r  c                 C   s|   g }| D ]7}|j dkr|d|jd q|j dkr-|j}d| dg}|d|d q|j dkr;|d|jd q|S NrO  rP  rQ  r?   rJ  system)rK  rK   r   )hist	hist_listr   ai_histai_hist_contentrD   rD   rE   message_to_dict_listB  s   


r  c                 C   s`   | j dkr| j}d|dS | j dkr!| j}d| dg}d|dS | j dkr.| j}d|dS d S r  )rK  r   )r  contr  rD   rD   rE   r  O  s   





r  c           	      C   s  g }| D ]}t |trE|ddkrt|d}|| q|d ddkrD|dg }|rDt |trDt|d dd}|| qt |tr|D ]=}|ddkrbt|d}|| qL|ddkrvt|d}|| qL|ddkrt|d}|| qLq|S )	NrK  rO  r5   rQ  r   r?   rH   r  )r  r  r=   r   rK   rR  r   r(   )	his_listcon_listihuman_contentai_dataai_conlt
ai_contentsys_contentrD   rD   rE   message_from_dict_list]  s2   



r  c                 C   s   t j|ddddd| iigiiid}g }|d d d	 d
kr=|d d D ]}|d }|d D ]}|d d }|| q.q$|rA|S d S )Nr   r   r   r  r   r   r   r   rm   r   r   r   r5   r   )r   r   rK   )r  r@  rC   contentsr   r:  rz   r   rD   rD   rE   retrieve_chat_contentw  s(   
r  c                 C   s@   t dd}d}tj||d}| }| t|dg }||}|S )Ngpt-4omodel
similaritysearch_typer   )r   dbr   r   invoke)past_messages
user_queryllmr  retrieved_dataformatted_datacombined_messagesrC   rD   rD   rE   query_and_combine  s   
r  c                 C   sh   ddddd|iidd|iigiii}t j| |d}|d }|d	kr.td
| d| d d S td d S )Nr   r   r   r  r   r   r   deletedr   zSuccessfully deleted z documents with session_id 'z' and user_prompt 'upload1'.zJNo documents found with the provided session ID and user_prompt 'upload1'.)r   delete_by_queryr\   )r   r   condition_checkr   delete_responsedeleted_countrD   rD   rE   delete_document_content  s   

r  stringr  returnc                 C   s   t |}t|| }|S )z.Returns the number of tokens in a text string.)tiktokenencoding_for_modelrt   encode)r  r  encoding
num_tokensrD   rD   rE   num_tokens_from_string  s   
r  c                 C   sD   t | trtdd t|  D S t | tr tdd | D S | S )z1Convert dictionary to a tuple that can be hashed.c                 s   s     | ]\}}|t |fV  qd S r6  dict_to_tuple)r   kvrD   rD   rE   	<genexpr>  s    z dict_to_tuple.<locals>.<genexpr>c                 s   s    | ]}t |V  qd S r6  r  )r   r  rD   rD   rE   r        )r  r  tuplesorteditemsrR  )drD   rD   rE   r    s
   

r  c                 C   s   |  dd}|S )N"z\")rk  )r?   
fixed_textrD   rD   rE   fix_syntax_error  s   r  c                 C   st  d }t jtdddd| iigdd|iidddddiiiigd	iid
d d }|rd}|D ]_}|d }	|d }
|
d|krd}|
dd}|
dd}|| }|| }|| }td|  td|  td|  d|||||| ddi}t jt|	|d td|  |	} nq-|s|||||| | ddd}t jt|d
}td|  |d }|S td|   |S )Nr   r   r  orgnamerN   must_notexistsfield)r   shouldr   r   Fr   r   Tinput_tokensr   output_tokenszinput_overall_tokens %szoutput_overall_tokens %soverall_tokens %sr   rq  )rN   mailr  r  overall_tokensr  limits_tokenr  zDocument updated for username:   )rN   r  r  r  r  r  r  assigned_tokensz#New document created for username: z$No document found for organization: )	r   r   user_details_indexr=   rr   rs   r  r\   r   )org_namerN   r  r  rc   
corp_orgiduser_tokensusername_foundr   r  
doc_sourceinpoutpinput_overall_tokensoutput_overall_tokensr  updated_docnew_docresultrD   rD   rE   user_org_tokens  s   

%
r  c                 C   s   t jtddd|iiidd d }|rZ|D ]
}|d }|d }q|dd	}|d
d	}|dd	}	| | }
|| }|
| }t||	 d }d||
|||di}t jt||d}d S d S )Nr   r  organization_namer   r   r   r   r  r   r  r  d   r   )r  r  r  r  
percenatger  )r   r   org_details_indexr=   r   r  )r  r  r  org_toksr   r  r  r  r  r  r  r  r  
percentage
update_docr   rD   rD   rE   	org_token:  sD   		

r  c              
   C   s   z+t  }|jjd| d}| }|d d d }|d d d }| |r(|fW S i fW S  tyH } ztd|  di fW  Y d	}~S d	}~ww )
zg
    Check user input using the Moderation API.
    Returns a tuple (is_safe, flagged_categories).
    zomni-moderation-latest)r  inputresultsr   flagged
categorieszModeration API failed: TN)r$   moderationscreateto_dictr>   r\   )r?   clientrC   ar  r  rw   rD   rD   rE   moderate_inputh  s   r   c           (      C   s  z| j dkr| jd}|dkr| jd}| jd}| jd}| jd}| jd}| jd	}tt| t|d
}t|\}	}
t||||t|d
|	|
	}|	rd}t	g d|d}t
dddd}td tjtdd|iidd}|rt|d d dkrdd |d d D }ng }t|}|rtdt|D ]%}t|| tkr|j|| j qt|| tkr|j|| j qtjtd
dtjddid|d|idd}||}|d }t|d
}d d!d"d#d|iigiii}tjt|d$}d}d}|d d D ]}|d% d&d}|d% d'd}qtd&| td(| |dkr|dkr||| |  dk rtd)| td*t tjtd d!d"d#d|iigiiid$}|d d+ d, dkr||d d d d- }td.| tjt|d/d0d1did2id3 tjt|d$}t d4|d5W S t!|||||d
 t"||| t#|||||}|d6 }g }|D ]	} |$| j% qt&|}!z5t'd7(d8d |d6 D }"t'|}#t'|}$t'|!}%d9|# d:|$ d:|" d:|% d:| d:| }&t)*|& W n t+y }' zt,-d;|'  W Y d }'~'nd }'~'ww t ||d5W S t!|||||d
 t"||| t#|||||}|d6 }g }|D ]	} |$| j% q/t&|}!z5t'd7(d<d |d6 D }"t'|}#t'|}$t'|!}%d9|# d:|$ d:|" d:|% d:| d:| }&t)*|& W n t+y }' zt,-d;|'  W Y d }'~'nd }'~'ww t ||d5W S t d=||
d>W S W d S W d S  t+y }' zt,-d;|'  t d?t.|'iW  Y d }'~'S d }'~'ww )@Nrg   operational_typeaction1rN   r   r   r  rM   rz   zgpt-4-0125-previewa  Answer the question in your own words as truthfully as possible from the context given to you. If there is any MCQ question explain why the choice is correct.
                    If you do not know the answer to the question, simply respond with "I don't know. Can you ask another question".
                    Response must be in and around 200 words. it must be in paragraph manner and it must not exceed 4 paragraphs.
                    Give a line of space between the paragraphs.
                    If questions are asked where there is no relevant context available, simply respond with "That's a great question! I'm not sure of the answer right now.  Can you ask your question a different way? I am excited to assist you further!"
                    Context: {context}
            
            
                    {chat_history}
                    Human: {question}
                    Assistant:contextr?  question)input_variablestemplater?  Tr   
memory_keyreturn_messages
output_keyrq  r  lastmodified_at:ascr   r   r   r   r   c                 S      g | ]}t |d  d qS r   r   r<   loadsr   r:  rD   rD   rE   r         zallibot.<locals>.<listcomp>r  r     search_kwargsprompt)r  	retrievermemorycombine_docs_chain_kwargsreturn_source_documentsr   r   r   r   r   r   r  r  oupif satisfiedr  r   rm   r   
userdoc_idscript  
                                                if (ctx._source.containsKey('limits_token')) {
                                                    // If limits_token exists, update its value
                                                    ctx._source.limits_token = params.limits_token;
                                                } else {
                                                    // If limits_token doesn't exist, create it and set the value
                                                    ctx._source['limits_token'] = params.limits_token;
                                                }
                                            r  r   paramsr  "Your token limit has been reached.rC   r   source_documentsrH   c                 S      g | ]	}|j d  d qS r  
_lc_kwargsr(  r  rD   rD   rE   r         zOpython3 /var/www/html/AlliBotV5_chatbot/AlliBotV50/Quesgen_answer_relevancy.py rH  r   c                 S   r&  r'  r)  r  rD   rD   rE   r   4  r*  jThe Question you asked violates AlliBot usage policies.Don't repeat this from blocking of AlliBot account.rC   r   r   ro   )/rq   rg   r=   rd   r!  r   r  r   r   r   r   r   r   r   rt   r   rangerK  r   chat_memoryadd_user_messager   r   add_ai_messager   from_llmr   r  as_retrieverr  r\   r  r   r  r  r  rK   r  r   r   rL   r   popenr>   rr   	exceptionstr)(re   action_typerN   r   r  r  rc   r   "input_token_cost_gpt4_0125_previewr   r   document_id_userr  r  r  r  r  session_chat_histr  qaformatted_responser   #output_token_cost_gpt4_0125_previewr   rC   r  r  r   r  org_user_docidr   r   sorunique_entries_mod_strjson_str	user_quesanswer_promptunique_entries_strcmdrw   rD   rD   rE   allibot  s   













((
  CrE  c           5   
   C   s&  z| j dkr| jd}|dkrI| jd}| jd}| jd}| jd}| jd}| jd	}tt| t|d
}t|\}	}
t||||t|d
|	|
	 |	r@d}t	dddd}t
d tjtdd|iiddd}|rt|d d dkrdd |d d D }ng }t|}|rtdt|D ]%}t|| tkr|j|| j qt|| tkr|j|| j qtjtdd|iidd}|rt|d d dkrdd |d d D }ng }g }|r4|D ]I}|d drt|d dnd}|d d}t|ts*zt|trt|ni }W n tjtfy)   i }Y nw |t ||d qt!tg dt"t!# d }|j$d!did"%|}g }|D ]}|t |j&d# qO|| }t' }|(d$di}|d }|j)|||d%}|j*j+j,d
d&|d'gd(d)}|j-d j.j}t|d
} d*d+d,d-d|iigiii}!tjt/|!d.}d}"d}#|d d D ]}$|$d/ d0d}"|$d/ d1d}#q|"|#|  |  dk r!t0d2| t0d3t/ tjt/d*d+d,d-d|iigiiid.}|d d4 d5 dkr|d d d d6 }%t0d7|% tj1t/|%d8d9d:did;id< tjt/|!d.}t2d=|d>W S t3||||| d
 t4|| | t5|||| | t2||d>W S t2d?||
d@W S |dAkr| j67dB}&| jd}'| jd&}|&smt2dCdDdEdFdGW S t8j9:t8j9;t<dH }(t8j9=|(st8>|( |&D ].})t8j9:|(|)j?}*t@|*dI}+|)A D ]},|+B|, qW d    n	1 sw   Y  qtC|(dJ }-tDt|'dK tE|-d \}.}/|-d FdLdM })|/r|/D ]}dN|) dO|j&i|_&qtG||'|/tdK|)  |-D ]}0t8H|0 q|.dPkrtt|&dQ }1t2dR|1dEW S dS}1t2dT|1dEW S |dUkr| j67dB}&| jd}'| jd&}|&s>t2dCdDdEdFdGW S t8j9:t8j9;t<dH }(t8j9=|(sVt8>|( |&D ].})t8j9:|(|)j?}*t@|*dI}+|)A D ]},|+B|, qlW d    n	1 sw   Y  qXtDt|'dV tC|(dJ }-tE|-d \}2}3|-d FdLdM })|3r|3D ]}dW|) dO|j&i|_&qtG||'|3tdV|)  |-D ]}0t8H|0 q|2dPkrtt|&dQ }1t2dR|1dEW S dX}1t2dT|1dEW S W d S W d S  tIy }4 ztJKdY|4  t2dCt|4iW  Y d }4~4S d }4~4ww )ZNrg   r  r  rN   r   rz   r   r  rM   r  a;  Answer the question in your own words as truthfully as possible from the context given to you.
                    You are a helpful assistant. Include the filename within your response where relevant.
                    Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                    Give a line of space between the paragraphs.
    
                    If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.
    
                    If questions are asked where there is no relevant context available, simply respond with:
                    "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"
    
                    Context: {context}  
    
                    {chat_history}  
                    Human: {question}  
                    Assistant:r?  Tr   r  r   r  r  r   r   r   r   r   r   r   c                 S   r  r  r  r  rD   rD   rE   r   x  r  zdoccompare.<locals>.<listcomp>r  c                 S   r  r  r  r  rD   rD   rE   r     r  r  rH   r  r	  r   r   r  r  r  r  r  r   r   r   Fr  r   streamr   r   r   r   r   r   r  r  r  r  r   rm   r   r  r  t  
                                            if (ctx._source.containsKey('limits_token')) {
                                                // If limits_token exists, update its value
                                                ctx._source.limits_token = params.limits_token;
                                            } else {
                                                // If limits_token doesn't exist, create it and set the value
                                                ctx._source['limits_token'] = params.limits_token;
                                            }
                                        r  r!  r  r#  r$  r+  r,  action2filesro   No files uploadedstatusrz     rP  z/upload_fileswb+/*upload1r  r  context of the first uploaded 	 documentrq   uploads completesuccess$unable to process upload1 documents.failedaction3upload2zcontext of the second uploaded $unable to process upload2 documents.r   )Lrq   rg   r=   rd   r!  r   r  r   r   r   r   r   r   rt   r   r-  rK  r   r.  r/  r   r   r0  	doc_indexr5  r  r  r<   r  JSONDecodeError	TypeErrorrK   BaseDocumentr   
embeddingsExactRetrievalStrategyr2  get_relevant_documentsr  r$   load_memory_variablesr&  chatcompletionsr  choicesrz   r  r\   r  r   r  r  r  FILESgetlistr   r   rL   dirname__file__r  makedirsra   r'  chunkswriter   r  r  r  r  rl  r>   rr   r4  )5re   r6  rN   r   r   r  r  rc   input_token_cost_gpt4or   r   r  r  r  r  r9  r  	doc_itemsuploaded_document_docsr  r  r  primary_docsprimary_docs_textr   combined_docsr  memory_variablesr?  formatted_templaterC   assistant_responseoutput_token_cost_gpt4or   r  r  r   r  rM  	conver_id
upload_dirr  r  destinationchunk	files_dirind_response1r  fres_msgind_response2docs2rw   rD   rD   rE   
doccompareK  s  






$
















   |'r  c           ;   
      s|  z| j dkr| jd}| jd}tdd}tjtjt	d  tj
 s0t  |dkrW| jd}| jd	}| jd
}| jd}| jd}| jd}	tt| t|d}
t|\}}t||||t|
d||	 |rNzt d r|r fdd|D }|rg }g }|D ]}| drtd t|||t|}td ||g}td|  |t| ||j qt|\}}|dd }dd |D }tdd| ddd| dd| dgd}t|}t |||gt| d}|tt!|d || qW n t"y2 } ztd |  W Y d }~nd }~ww t d }|D ]}t#| q;|r\t$d! t%|t}|rZ|}t&|}ng }td|dgd}d"} t'j(|| d#}!d$d |!D }t|d}"|r||"g |g }#n||g }#||#}$t)|g}%|%t|$ t |||%t| t|$jd}&d%d&d'd(d|iigiii}'t*j(t+|'d)}d*}(d*})|d+ d+ D ]}*|*d, d-d*}(|*d, d.d*})q|(|)|& |
  d*k r;t,d/| t,d0t+ t*j(t+d%d&d'd(d|iigiiid)}|d+ d1 d2 d*kr,|d+ d+ d* d3 }+t,d4|+ t*j-t+|+d5d6d7d*id8id9 t*j(t+|'d)}t.d:|d;W S t/||||$j|&d t0|
|&|	 t1|	||
|&| t.|$j|d;W S g },t2 }-|D ]}.t3|.}/|/|-vrx|,|. |-4|/ qct5d*t6|,D ]}t |||,| gt| qd<t7t2|}0t|0d}&d%d&d'd(d|iigiii}'t*j(t+|'d)}d*}(d*})|d+ d+ D ]}*|*d, d-d*}(|*d, d.d*})q|(|)|& |
  d*k r/t,d/| t,d0t+ t*j(t+d%d&d'd(d|iigiiid)}|d+ d1 d2 d*kr |d+ d+ d* d3 }+t,d4|+ t*j-t+|+d5d6d7d*id8id9 t*j(t+|'d)}t.d:|d;W S t/||||0|&d t0|
|&|	 t1|	||
|&| t.|0|d;W S t.d=||d>W S |d?kr| j8d@}|sot.dAdBdCdDdEW S |D ].}tj |j9}1t:|1dF}2|; D ]}3|2<|3 qW d    n	1 sw   Y  qqzAt dG t dH  t dI  t dJ  }4g d}5g }4|5D ]}6|4=ttj dK|6  q|4D ]}7t>|7}8qdL}9W n t"y } zd*}9W Y d }~nd }~ww |9dLkrt?t6|dM }:t.dN|:dCW S dO}:t.dP|:dCW S W d S W d S  t"y= } zt@dQ|  t.dAt?|iW  Y d }~S d }~ww )RNrg   r  rM   r  r  z
/image_pthr  rN   r   rz   zfile_names[]r   r  rT  c                    s   g | ]} d  | qS )r  rD   )r   r   r|  rD   rE   r         zimgtotext.<locals>.<listcomp>.jpgrg  .pngrf  zentered initial stagezissue with upload statuszresponse got %sr  r  c                 S   s   g | ]}|j qS rD   rG  r   r   rD   rD   rE   r     s    r?   z
Filename: rJ  z2Here is the text content of the uploaded document z:

r   zCould you please clarify the specific operations you'd like to perform on the uploaded documents? If you need further assistance or have additional questions, feel free to askz(Exception occured in the image upload %sr   r  r  c                 S   s&   g | ]}|j  d krd|j dqS )z{"text": " "}r?   rJ  )r  stripr  rD   rD   rE   r     s
    
r   r   r   r   r   r   r   r   r  r  r  r  r   rm   r   r  r  r   r  r!  r  r#  r$  z

r+  r,  rL  rM  ro   rN  rO  rQ  rR  rS  /*.jpg/*.jpeg/*.png/*.bmp*rq  rX  rY  zUnable to process image.r[  r   )Arq   rg   r=   rd   r   r   r   rL   rl  rm  r  rn  rk  r!  r   r  r   r   r   rm  endswithrr   rs   r  imtext_indexr  rK   r  r   r  r  r   r  r   r>   rl  r   r  r  r  r   r  r   r  r\   r  r   r  r  r  r   r  r   r-  rt   rR  rj  ra   r'  ro  rp  extendrp  r5  r4  );re   r6  rc   r  rN   r   r   rM  r  r  rq  r   r   a_with_pathai_type_convimg_only_respr  	encodeimgrC   r  r  r  r  rz   img_doc_contstatic_contentrw   r  r  ret_chatconverted_chat
user_inputr  r  
ai_messageall_chatimg_responser   rz  r   r  r  r   r  unique_dataseenr  
dict_tupleonly_files_responser  r}  r~  image_filessupported_formatsextr`  processed_image_pathind_responser  rD   r  rE   	imgtotextt  s  





























0"
    !'r  c                 C   s4   t  }|jjjdd| dgdd}|jd jj}|S )Nr  r   rH  FrI  r   )r$   rg  rh  r  ri  rz   r   )rx  r  rC   ry  rD   rD   rE   get_resptable  s   
r  c           A      C   s  z| j dkr| jd}|dkrw| jd}| jd}| jd}| jd}| jd}| jd	}tt| t|d
}t|\}	}
t||||t|d
|	|
	 |	rnd}t	dddd}t
jtdd|iiddd}|rt|d d dkrdd |d d D }ng }t|}|rtdt|D ]I}t|| tkr|j|| j qt|| tkr|j|| j qt|| tkr|j|| j qt|| tkr|j|| j qt
jtdd|iidd}|rt|d d dkrdd |d d D }ng }g }|rX|D ]K}|d drt|d dnd}|d d}t|tsMzt|tr:t|ni }W n tjtfyL   i }Y nw | t!||d qt"t
g dt#t"$ d}|j%d d!id"&|}g }|D ]}| t!|j'd# qs|| }t( }|)d$di}|d }|j*|||d%}|j+j,j-d
d&|d'gd(d)}|j.d j/j}t|d
} d*d+d,d-d|iigiii}!t
jt0|!d.}d}"d}#|d d D ]}$|$d/ d0d}"|$d/ d1d}#qt1d0|" t1d1|# |"|#|  |  dk rOt1d2| t1d3t0 t
jt0d*d+d,d-d|iigiiid.}|d d4 d5 dkr@|d d d d6 }%t1d7|% t
j2t0|%d8d9d:did;id< t
jt0|!d.}t3d=|d>W S t4||||| d
 t5|| | t6|||| | t3||d>W S t3d?||
d@W S |dAkr| j78dB}&| jd}'| jd&}|&st3dCdDdEdFdGW S t9j:;t9j:<t=dH }(t9j:>|(st9?|( |&D ].})t9j:;|(|)j@}*tA|*dI}+|)B D ]},|+C|, qW d    n	1 sw   Y  qtD|(dJ }-tEt|'dK tD|(dL tD|(dM  tD|(dN  tD|(dO  }.|.rg dP}/g }.|/D ]}0|.FtDt9j:;|(dQ|0  qtGtHdRd
dSdT}1|.D ]W}2t1dU|2 tI|2}3t1dV|3 tJ|3}4|3KdWsQ|3KdXrTdY}5n|3KdZr]d[}5ndY}5|1Ltd\d]td^d\d_d`dadb|5 dc|4 iddgd]g}6de}7|6j}8t1df|6j q1ntM|-d \}7}8|-d Ndgdh })t|8tOs|8g}8|8rtP|8D ].\}}t|trdi|) dj| dk|-d id|8|< qtQ|drdi|) dl|j'i|_'qtR||'|8tdK|)  |-D ]}9t9S|9 q|7dekrtt|&dm }:t3dn|:dEW S do}:t3dp|:dEW S |dqkr| j78dB}&| jd}'| jd&}|&s2t3dCdDdEdFdGW S t9j:;t9j:<t=dH }(t9j:>|(sJt9?|( |&D ].})t9j:;|(|)j@}*tA|*dI}+|)B D ]},|+C|, q`W d    n	1 stw   Y  qLtD|(dJ }-tEt|'dr tD|(dL tD|(dM  tD|(dN  tD|(dO  }.|.rg dP}/g }.|/D ]}0|.FtDt9j:;|(dQ|0  qtGtHdRd
dSdT}1|.D ]G}2tI|2}3tJ|3}4|3KdWs|3KdXrdY}5n|3KdZrd[}5ndY}5|1Ltd\d]td^d\d_d`dadb|5 dc|4 iddgd]g}6de};|6j}<qntM|-d \};}<|-d Ndgdh })t|<tOs+|<g}<|<rktP|<D ].\}}t|trOdi|) dj| dk|-d id|<|< q2tQ|dr_di|) dl|j'i|_'q2tR||'|<tdr|)  |-D ]}9t9S|9 qm|;dekrtt|&dm }:t3dn|:dEW S ds}:t3dp|:dEW S |dtkr| jd}| jd}| jd}| jd}| jd	}g du}=dv}tt| t|d
}t|\}	}
tT||||t|d
|	|
	 |	rdw}t	dddd}t
jtdd|iidd}|rt|d d dkrdxd |d d D }ng }g }|ra|D ]K}|d dr(t|d dnd}|d d}t|tsVzt|trCt|ni }W n tjtfyU   i }Y nw | t!||d q|}tUVdy|  |)d$di}|d }|j*||dz}td{D ]g}>tUVd||>  tW|}d}|v rtUVd~|  d}|Nd}dh  }|Xdd}d|v r|Xdd}nd|v r|Xdd}nd|v r|Xdd}|=D ]}?d|? d |v r|Xd|? d |?}q nqt|d
} d*d+d,d-d|iigiii}!t
jt0|!d.}d}"d}#|d d D ]}$|$d/ d0d}"|$d/ d1d}#qtUVd|"  tUVd|#  tUVd|   |"|#|  |  dk rt1d2| t1d|  t
jt0d*d+d,d-d|iigiiid.}|d d4 d5 dkr|d d d d6 }%t1d7|% t
j2t0|%d8d9d:did;id< t
jt0|!d.}t3d=|d>W S tUVd|   tY||||| d
 t5|| | tUVd|   t6|||| | t3||d>W S t3d?||
d@W S W d S W d S  tZy }@ ztU[d|@  t3dCt|@iW  Y d }@~@S d }@~@ww )Nrg   r  r  rN   r   rz   r   r  rM   r  a+  Answer the question in your own words as truthfully as possible from the context given to you.
                    You are a helpful assistant. Include the filename within your response where relevant.
                    Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                    Give a line of space between the paragraphs.

                    If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.

                    If questions are asked where there is no relevant context available, simply respond with:
                    "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"

                    Context: {context}  

                    {chat_history}  
                    Human: {question}  
                    Assistant:r?  Tr   r  r  r  r   rF  r   r   c                 S   r  r  r  r  rD   rD   rE   r     r  z formscompare.<locals>.<listcomp>r  c                 S   r  r  r  r  rD   rD   rE   r     r  r  rH   r  r	  r   r   r  r   r  rG  r  r  r   rH  FrI  r   r   r   r   r   r   r  r  r  r  r   rm   r   r  r  rK  r  r!  r  r#  r$  r+  r,  rL  rM  ro   rN  rO  rQ  rR  z/formsupload_filesrS  rT  rU  r  r  r  r  r  r  r   rd  )openai_api_keyr  
max_tokensr`  r  rg  r  z
image/jpegr  z	image/pngz?You are a useful bot that is especially good at OCR from imagesr   r?   rJ  r  rP   zdata:z;base64,r  rq  zimg outr  r  rV  z image: r   rW  rX  rY  rZ  r[  r\  r]  r^  action4)zPolicy NumberzPolicy Effective DatezPolicy Issue Datez"Schedule of Forms and Endorsementsz)Common Policy Declarations and ConditionsDeclarationsCoverageEndorsementsa  Extract the following information from each uploaded document and return the output strictly in a table format only, with column names as the document names and row fields as: Policy Number, Policy Effective Date, Policy Issue Date, Schedule of Forms and Endorsements, Common Policy Declarations and Conditions, Declarations, Coverage, and Endorsements.

                **Ensure the following**:  
                1. The Policy Number format must strictly follow the pattern of two alphabetic characters, a space, followed by two digits, a space, repeated thrice (e.g., AG 00 02 30 06). The Policy Number is typically found in the header or footer of the document. If the policy number field appears empty, perform a thorough search across the document, including all sections (header, footer, body text, metadata, etc.), to ensure the policy number is identified and extracted wherever it exists.
                2. If any field is missing or not found in a document, include only the data from the fields that are present. Use as much detail as possible, up to 100 words, in each field that is available. Leave missing fields blank in the table.  

                **For Specific Fields**:  
                - **Schedule of Forms and Endorsements**: Get data based on the list of policy forms, schedules, and endorsements by line of business.  
                - **Declarations**: Include line of coverage-specific declarations (general liability, business auto, business income, contractors equipment, etc.).  
                - **Coverages**: Provide a comparison of coverages offered by the underlying policy (general liability, business auto, business income, contractors equipment, etc.).  
                - **Endorsements**: List the name of each endorsement included in the document with a summary of how each endorsement affects the underlying policy.  

                **Response Format**:  
                - Provide the extracted data in a table format as described above.  
                - Include an "Overall Summary" section directly under the table. This summary must summarise the table data's important observations across the documents, titled as **summary**.ak  Answer the question in your own words as truthfully as possible from the context given to you.  
                    You are a helpful assistant. Include the filename within your response where relevant.  
                    Extract the following information from each uploaded document and return the output strictly in a table format only.  

                    **Table Format Requirements**:  
                    - The table must have a header row with "Field Name" in the first column, followed by the uploaded document names as subsequent column headers (e.g., "Uploaded Document 1 Name" and "Uploaded Document 2 Name").  
                    - Row fields should include the **actual values** for the following fields: Policy Number, Policy Effective Date, Policy Issue Date, Schedule of Forms and Endorsements, Common Policy Declarations and Conditions, Declarations, Coverage, and Endorsements.  

                    **Ensure the following**:  
                    1.The Policy Number must follow one of the patterns below:
                    - Two alphabetic characters, a space, followed by two digits, a space, repeated thrice (e.g., AG 00 02 30 06).
                    - Three alphabetic characters, a space, followed by two digits, a space, repeated twice (e.g., CNI 90 22 11 22).
                    - Two alphabetic characters, a space, another two alphabetic characters, a space, followed by two digits, a space, and repeated once (e.g., CG DS 01 10 01).
                    - Two alphabetic characters, a space, one alphabetic character, a space, followed by three digits, a space, then two digits, a space, and two digits (e.g., IL U 002 05 10).
                      The Policy Number is usually found in the header or footer of the document. Ensure to search these sections carefully when extracting the number.
                    2. If any field is missing or not found in a document, include only the data from the fields that are present. Use as much detail as possible, up to 100 words, in each field that is available. Leave missing fields blank in the table.  
                    3. Perform a comparison for each field across all uploaded documents to ensure consistency.

                    **For Specific Fields**:  
                    - **Schedule of Forms and Endorsements**: Get data based on the list of policy forms, schedules, and endorsements by line of business.  
                    - **Declarations**: Include line of coverage-specific declarations (general liability, business auto, business income, contractors equipment).  
                    - **Coverages**: Provide a comparison of coverages offered by the underlying policy (general liability, business auto, business income, contractors equipment).  
                    - **Endorsements**: List the name of each endorsement included in the document with a summary of how each endorsement affects the underlying policy.  

                    **Response Format**:  
                    - Provide the extracted data in a table format as described above.  
                    - Include an "Overall Summary" section directly under the table. This summary must summarise the table data's important observations across the documents, titled as **summary**.  

                    **Additional Note**:  
                    If questions are asked where there is no relevant context available, simply respond with:  
                    "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"

                    Context: {context}  

                    {chat_history}  
                    Assistant:c                 S   r  r  r  r  rD   rD   rE   r   	  r  combined_docs%sr  r?  r  z	attempt%sz
Field Namezassistant_response if cond %sz```z**summary:**z**Summary**z**Summary:**zOverall Summary:z**assigned_tokens %sr  output_token_cost_gpt4o %srz  z$output_token_cost_gpt4o to sys es %sz!output_token_cost_gpt4o org es %sr   )\rq   rg   r=   rd   r!  r   r  r   r   r   r   r   rt   r   r-  rK  r   r.  r/  r   r   r0  r)   r(   forms_indexr5  r  r  r<   r  r`  ra  rK   rb  r   rc  rd  r2  re  r  r$   rf  r&  rg  rh  r  ri  rz   r  r\   r  r   r  r  r  rj  rk  r   r   rL   rl  rm  r  rn  ra   r'  ro  rp  r   r  r  r   r#   rp  ra  r  r  r  r  rR  	enumerater
  r  rl  r   rr   rs   r  rk  r  r>   r4  )Are   r6  rN   r   r   r  r  rc   rq  r   r   r  r  r  r  r9  r  rr  rs  r  r  r  rt  ru  r   rv  r  rw  r?  rx  rC   ry  rz  r   r  r  r   r  rM  r{  r|  r  r  r}  r~  r  r  r  r  chainr`  r  image	mime_typemsgr  r  r  r  r  r  
field_listattemptfierw   rD   rD   rE   formscompare  sV  






&












0"

%



0"
%




%

&











        4 er  c              
   C   sJ  | j dkrzt| jd}| jd}ddddd|iigiii}tjt|d}d	}|d
 d
 D ]D}|d dd}|d dd}|d dd}	|	dkrOd}q.|r]|s]|dkrZd	}q.d}q.|rr|rr|| }
|
dk sm|
dkrpd}q.d	}q.td|iW S  ty } zt	d|  tdt
|iddW  Y d }~S d }~ww tddiddS )Nrg   r  rN   r   r   r   r   r   Tr   r   r  r   r  r  Fr  activate_convzcheck token index %sro   r  rR  zMethod not allowed  )rq   rg   r=   r   r   r  r   r>   rr   rs   r5  )re   r  rN   r   rC   r  r   r  r  token_limitssub_resrw   rD   rD   rE   activate_query_conversationk
  sJ   

 r  c           
   	   C   sR  z| j dkrtd | jd}| jd}| jd}| jd}td td|  td	|  tj|d
d|iiddd}|d d d }td|  |dkr|d d D ]#}|d }d|d v r~|d d }	|	|kr~tj	||d|iddd q[t
dddW S t
ddddd W S W d S  ty   td!| d"| d# Y d S w )$Nrg   zthumbs flagr   r@  r   thumbs_flagr   zindex_name%szsession_id%sr   r   r   rX  r   r   r   rm   ztotal_docs%sr   r   r   r   Tr  r  rY  zDocument updatedrO  failurezNo documents foundi  rR  z)Error: No documents found for session_id z and answer 'z'.)rq   r\   rg   r=   r   rr   rs   r   r   r  r   r!   )
re   r   r   r   r  rC   
total_docsr   r  r  rD   rD   rE   update_thumbs_flag
  sN   


.r  c              
   C   s  | j dkrz| jd}| jd}| jd}| jd}d}tt| t|d}d}td	d
dd}tj	t
dd|iidd}	|	rWt|	d d dkrWdd |	d d D }
ng }
g }|
r|
D ]F}|d drqt|d dnd}|d d}t|tszt|trt|ni }W n tjtfy   i }Y nw |t||d q_|}td|  |ddi}|d	 }|j||d}t|}td| t|d}dddd d!|iigiii}tj	t|d"}d}d}|d d D ]}|d# d$d}|d# d%d}qtd&|  td'|  td(|  ||| |  dk rstd)| td*| tj	tdddd d!|iigiiid"}|d d+ d, dkre|d d d d- }td.| tjt|d/d0d1did2id3 tj	t|d"}td4|d5S t||d5S td6d7id8d9S ):Nrg   r   r   r  rM   aI  Analyze the information provided from both ISO forms and generate a letter emphasizing the key similarities, differences, and insights. Ensure the letter clearly reflects the comparison, maintaining adherence to the formats specified in the ISO forms. Include the filename(s) of the ISO forms within your response where relevant.r  a  Analyze the information provided from both ISO forms and generate a letter emphasizing the key similarities, differences, and insights. Ensure the letter clearly reflects the comparison, maintaining adherence to the formats specified in the ISO forms. Include the filename(s) of the ISO forms within your response where relevant.  

        Context: {context}  

        {chat_history}  
        Assistant:r?  Tr   r  r  r  r  r   r   c                 S   r  r  r  r  rD   rD   rE   r     r  z'compforms_lettergen.<locals>.<listcomp>r  rH   r  r	  r  r  r  ry  zgpt-4o-minir   r   r   r   rN   r   r   r  r  r  r  r  r  rz  r   rm   r   r  r  a  
                                if (ctx._source.containsKey('limits_token')) {
                                    // If limits_token exists, update its value
                                    ctx._source.limits_token = params.limits_token;
                                } else {
                                    // If limits_token doesn't exist, create it and set the value
                                    ctx._source['limits_token'] = params.limits_token;
                                }
                            r  r!  r  r#  r$  ro   Invalid request methodr  rR  )rq   rg   r=   rd   r!  r   r  r   r   r   r  rt   r5  r  r  r<   r  r`  ra  rK   rb  rr   rs   rf  r&  r  r\   r  r  r   )re   r   r  r  rc   r   rq  r  r  r  rr  rs  r  r  r  rv  rw  r?  rx  ry  rz  r   rC   r  r  r   r  rD   rD   rE   compforms_lettergen
  s   

	

$








r  c              
   C   s   | j dkrN| jd}|stddiddS ztjjjdd|d	}|j}t	
|d
}td|iW S  tyM } ztdt|iddW  Y d }~S d }~ww tddiddS )Nrg   rz   ro   zNo input text providedrQ  rR  ztts-1alloy)r  voicer  r0  audior  r  r  )rq   rg   r=   r   openair  speechr  r   r1  r2  r3  r>   r5  )re   
input_textrC   audio_contentbase64_encoded_audiorw   rD   rD   rE   text_to_speech{  s$   
 r  c              
   C   s   | j dkrOd| jv rG| jd }z| }t }|jjjd|j|fdd}td|iW S  t	yF } ztdt
|idd	W  Y d }~S d }~ww tdd
idd	S tddidd	S )Nrg   r  z	whisper-1r?   )r  r  response_formattranscriptionro   r  rR  zNo file providedrQ  r  r  )rq   rj  r_  r$   r  transcriptionsr  ra   r   r>   r5  )re   
audio_fileaudio_bytesr  r  rw   rD   rD   rE   speech_to_text  s$   


 r  c              
   C   s  zPg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|D ]}tj
|d d tjd}||  krJ|krQn q2|| q2|rG|dkrFttd	d
 |D }	i }
g }|	d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}td|  |d d D ]}td|  |d dkrd|d d d i}ni|d dkrd|d d d d 	dd i}nR|d d!kr#d"|d d d v r|d d d d"d  }d#|i}n-t|d d$krd|d d v rd|d d d d 	dd i}n
d%|d d d i}|| qqqd&d' |D }tt|}|||dd(}
||
 qqn|d)krttd*d
 |D }	i }
g }|	d d d D ]}tjtdd|iidd}|rt|d d dkrd+d' |d d D }g }i }|D ]}d#|i}|| qtjtdd|iidd}|rt|d d dkrtd,d
 |d d D }|D ]!}t|tkrd%|ji}nt|tkrd|ji}|| qtt|}|||d)d(}
||
 qbnF|d-krttd.d
 |D }	i }
g }|	d d d D ]}td/| tjtdd|iidd}|rDt|d d dkrDd0d' |d d D }g }i }|D ]}d#|i}|| qJtjtdd|iidd}|rt|d d dkrtd1d
 |d d D }|dd  D ].}t|tkrd%|ji}nt|t krd|ji}nt|tkrd|ji}|| qtt|}|||d-d(}
||
 qn{ttd2d
 |D }	i }
g }|	d d d D ]c}tjtdd|iidd}|r@t|d d dkr@td3d
 |d d D }g }i }|D ]!}t|tkrd%|ji}nt|tkr(d|ji}|| qtt|}|||d4d(}
||
 qd|i}ndg i}t!d|iW S  t"yu } ztd5|  t!d6t#|id7d8W  Y d }~S d }~ww )9N
X-UsernameX-Candidate-Name
image_logsr   hourminutesecondmicrosecondr   r   tzc                 s       | ]}|d  V  qdS r   NrD   r   r  rD   rD   rE   r    r  z!today_messages.<locals>.<genexpr>r  r  r   r  r  r   r   r   zretrived data %sr5   r   zsecond retrived %srK  rQ  	assistantr?   r  rq  r  rP   r  rH   rO  	Filename:r  r   r   c                 S      g | ]}|r|qS rD   rD   r  rD   rD   rE   r     r   z"today_messages.<locals>.<listcomp>r   r?  r   
app_switchdoc_logsc                 s   r  r  rD   r  rD   rD   rE   r    r  c                 S      g | ]}|d  d qS r   r   rD   r  rD   rD   rE   r     r  c                 s   "    | ]}t |d  d V  qdS r   r   Nr  r  rD   rD   rE   r        	forms_logc                 s   r  r  rD   r  rD   rD   rE   r  '  r  id_userc                 S   r  r  rD   r  rD   rD   rE   r   2  r  c                 s   r  r  r  r  rD   rD   rE   r  A  r  c                 s   r  r  rD   r  rD   rD   rE   r  _  r  c                 s   r  r  r  r  rD   rD   rE   r  i  r  alli_logztoday sec error %sro   r  rR  )$rI   r=   r   r  r   r	   r  r
   utcrk  fromtimestamprK   rR  r"   fromkeysr   r   rr   rs   r  r  rt   r   r   r_  r   rK  r   r   r   r\   r  r(   r   r>   r5  )re   trN   r   testcurrent_datetimestart_of_todayr   lastmodified_at_datetimesession_ids	yest_dictr  r  r  tot_histimtexr:  jr  r   chat_id_namesresult1session_hist1chat_hsession_histformatted_yest_dictrw   rD   rD   rE   today_messages  sv  


"
$"

,





0





5



 r	  c              
   C   s  zPg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|t
dd }|t
dd }|D ]}	tj|	d	 d
 tjd}
||
  krX|kr_n q@||	 q@|rG|dkrFttdd |D }i }g }|d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}|d d D ]}|d dkrd|d d d i}ni|d dkrd|d d d d 	dd i}nR|d d!kr#d"|d d d v r|d d d d"d  }d#|i}n-t|d d$krd|d d v rd|d d d d 	dd i}n
d%|d d d i}|| qqqd&d' |D }tt|}|||dd(}|| qn|d)krttd*d |D }i }g }|d d d D ]}tjtdd|iidd}|rt|d d dkrd+d' |d d D }g }i }|D ]}d#|i}|| qtjtdd|iidd}|rt|d d dkrtd,d |d d D }|D ]!}t|tkrd%|ji}nt|tkrd|ji}|| qtt|}|||d)d(}|| qbnF|d-krttd.d |D }i }g }|d d d D ]}td/| tjtdd|iidd}|rDt|d d dkrDd0d' |d d D }g }i }|D ]}d#|i}|| qJtjtdd|iidd}|rt|d d dkrtd1d |d d D }|dd  D ].}t|tkrd%|ji}nt|tkrd|ji}nt|tkrd|ji}|| qtt|}|||d-d(}|| qn{ttd2d |D }i }g }|d d d D ]c}tjtdd|iidd}|r@t|d d dkr@td3d |d d D }g }i }|D ]!}t|tkrd%|ji}nt|tkr(d|ji}|| qtt|}|||d4d(}|| qd|i}ndg i}t d|iW S  t!yu } zt"#d5|  t d6t$|id7d8W  Y d }~S d }~ww )9Nr  r  r  r   r  rq  days)microsecondsr   r   r  c                 s   r  r  rD   r  rD   rD   rE   r    r  z%yesterday_messages.<locals>.<genexpr>r  r  r   r  r  r   r   r   r5   r   rK  rQ  r  r?   r  r  rP   r  rH   rO  r  r  r   r   c                 S   r  rD   rD   r  rD   rD   rE   r     r   z&yesterday_messages.<locals>.<listcomp>r  r  c                 s   r  r  rD   r  rD   rD   rE   r    r  c                 S   r  r  rD   r  rD   rD   rE   r     r  c                 s   r  r  r  r  rD   rD   rE   r    r  r  c                 s   r  r  rD   r  rD   rD   rE   r    r  r  c                 S   r  r  rD   r  rD   rD   rE   r     r  c                 s   r  r  r  r  rD   rD   rE   r     r  c                 s   r  r  rD   r  rD   rD   rE   r  >  r  c                 s   r  r  r  r  rD   rD   rE   r  I  r  r  zyesterday sec error %sro   r  rR  %rI   r=   r   r  r   r	   r  r
   r  rk  r   r  rK   rR  r"   r  r   r   r  r  rt   r   r   r_  r   rK  r   r   r   r\   r  r(   r   r>   rr   rs   r5  )re   yrN   r   r  r  r  start_of_yesterdayend_of_yesterdayr   r  r  r  r  r  r  r   r  r:  r  r  r   r  r  r  r  r  r  rw   rD   rD   rE   yesterday_messages  sv  


"
$"

(





/





5



 r  c              
   C   s  zPg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|t
dd }|t
dd }|D ]}	tj|	d	 d
 tjd}
||
  krX|k r_n q@||	 q@|rG|dkrFttdd |D }i }g }|d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}|d d D ]}|d dkrd|d d d i}ni|d dkrd|d d d d 	dd i}nR|d d!kr#d"|d d d v r|d d d d"d  }d#|i}n-t|d d$krd|d d v rd|d d d d 	dd i}n
d%|d d d i}|| qqqd&d' |D }tt|}|||dd(}|| qn|d)krttd*d |D }i }g }|d d d D ]}tjtdd|iidd}|rt|d d dkrd+d' |d d D }g }i }|D ]}d#|i}|| qtjtdd|iidd}|rt|d d dkrtd,d |d d D }|D ]!}t|tkrd%|ji}nt|tkrd|ji}|| qtt|}|||d)d(}|| qbnF|d-krttd.d |D }i }g }|d d d D ]}td/| tjtdd|iidd}|rDt|d d dkrDd0d' |d d D }g }i }|D ]}d#|i}|| qJtjtdd|iidd}|rt|d d dkrtd1d |d d D }|dd  D ].}t|tkrd%|ji}nt|tkrd|ji}nt|tkrd|ji}|| qtt|}|||d-d(}|| qn{ttd2d |D }i }g }|d d d D ]c}tjtdd|iidd}|r@t|d d dkr@td3d |d d D }g }i }|D ]!}t|tkrd%|ji}nt|tkr(d|ji}|| qtt|}|||d4d(}|| qd|i}ndg i}t d|iW S  t!yu } zt"#d5|  t d6t$|id7d8W  Y d }~S d }~ww )9Nr  r  r  r   r  rq  r
     r   r   r  c                 s   r  r  rD   r  rD   rD   rE   r    r  z"previous_7_days.<locals>.<genexpr>r  r  r   r  r  r   r   r   r5   r   rK  rQ  r  r?   r  r  rP   r  rH   rO  r  r  r   r   c                 S   r  rD   rD   r  rD   rD   rE   r     r   z#previous_7_days.<locals>.<listcomp>r  r  c                 s   r  r  rD   r  rD   rD   rE   r    r  c                 S   r  r  rD   r  rD   rD   rE   r     r  c                 s   r  r  r  r  rD   rD   rE   r    r  r  c                 s   r  r  rD   r  rD   rD   rE   r    r  r  c                 S   r  r  rD   r  rD   rD   rE   r     r  c                 s   r  r  r  r  rD   rD   rE   r    r  c                 s   r  r  rD   r  rD   rD   rE   r  *  r  c                 s   r  r  r  r  rD   rD   rE   r  5  r  r  zprevious 7 days error %sro   r  rR  r  )re   srN   r   r  r  r  r  start_of_seven_days_agor   r  r  r  r  r  r  r   r  r:  r  r  r   r  r  r  r  r  r  rw   rD   rD   rE   previous_7_daysp  sv  


"
$"

)





9





6



 r  )NNNN)rV   )r"  )django.shortcutsr   django.contrib.auth.decoratorsr   django.contrib.auth.modelsr   django.httpr   r   django.views.decorators.csrfr   r<   r	   r   r
   r   r   langchain.chat_modelsr   r   sysiotypingr   langchain.load.dumpr   r  langchain_core.messagesr   r   r   r   mathr   langchain.promptsr   r   "langchain.chains.combine_documentsr   langchain_openair    langchain_community.vectorstoresr   langchain.chainsr   langchain.memoryr   r   langchain_core.messages.humanr   langchain_core.messages.air   r9   requests.authr   r    r!   collectionsr"   decoupler#   r  r$   PILr%   r&   django.core.mailr'   langchain_core.messages.systemr(   langchain_core.messages.chatr)   r   rK   rL   rl  rm  langchain_core.documentsr*   Loader_functions_utilsr+   rr  langchain.docstore.documentrb  r  django.contrib.authr,   r-   r.   django.urlsr/   r1  bcryptLogUtilsgetRootLoggerrr   r8   r2   r3   r@   rF   rS   rU   r`   rf   rx   r   r   r   r   r   r   environrc  r   r   r  r_  r  r  r  r   r   rd  r  r>   exr4  r   r   r   r  r  r  r  r  r  r!  r/  r5  r9  r<  rB  rG  rD  rT  rY  rZ  ra  rc  rp  r  r  r  r  r  r  r  r  r  r  r5  r  r  r  r  r  r  r   rE  r  r  r  r  r  r  r  r  r  r	  r  r  rD   rD   rD   rE   <module>   sd  


	
*
(


5


	
#!	d.

 L
  *
  O
     "
>
5
 

 e a