o
    3"h                     @   s  d dl mZ d dlmZ d dlmZ d dlmZmZ d dl	m
Z
 d dlZd dlmZmZmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlZd dlZd dlZd dlmZ d dlmZ d dl Z d dl!Z!d dl"Z"d dl#m$Z$m%Z%m&Z& d dl'm'Z' d dl(m)Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl7m9Z9 d dl:m;Z; d dl<m=Z= d dl>Z>d dl?m@Z@ d dl"mAZAmBZB d dlCmDZD d dlEmFZF d d l1mGZG d d!lHmIZI d d"lmJZJ d d#lKmLZL d d$lMmNZN d d%lOmPZP ejQRejQSejQTeU d d&lVmWZW d dlZd dlXZYd d'lZm[Z[ e[ Z\d d&l]mWZ^ d dl_Z_d d(l`maZambZb d d)l mcZc d d*ldmeZe d d+lfmgZgmhZh d dlfZid dljZjd dlfZid dl1Z1d dlkZkelem nd, ejQTejQTejQoeUZpejQSepd-d.Zqejreqd/d0 ejQSeqd1em nd, d2Zsd3d4 ZteFd5Zud6Zvd7d8 Zwd9d: Zxdd<eyd=eyd>eyd?eyd@eyf
dAdBZzdd<eyd?eyd@e{fdCdDZ|dEdF Z}dGeyfdHdIZ~dJeyd@eyfdKdLZdMdN Ze
dOdP Ze
dQdR Ze
dSdT Ze
dUdV Ze
dWdX Ze
dYdZ Ze
d[dR Zd\d] Zd^d_ ZeFd`ejda< eledb dcejdb< e0 ZeFddZeFdeZeFdfZeFdgZeFdhZeFdiZeFdjZeAdkdld/dmZdndo Zze4eg dpee4 dqZW n# ey Z ze rye4eg dpee4 dqZW Y dZ[ndZ[ww drds Zdtdu Zdvdw Zdxdy Zdzd{ Zd|d} Zd~d Zdd Zdd Zdd ZdddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdeydeyd@efddZdd Zdd Zdd Zdd Ze
dd Ze
ddĄ ZddƄ Ze
ddȄ Ze
ddʄ Ze
dd̄ Ze
dd΄ Ze
ddЄ Ze
dd҄ ZddԄ Zddք Zdd؄ Ze
ddڄ Ze
dd܄ ZdS )    )render)login_required)User)JsonResponseHttpResponse)csrf_exemptN)	timedeltatimezonedatetime)timesleep)
ChatOpenAI)ChatGoogleGenerativeAI)genai)HttpOptions)List)dumps)BaseMessagemessage_to_dictmessages_from_dict)glob)floor)ChatPromptTemplatePromptTemplate)create_stuff_documents_chain)OpenAIEmbeddings)AsyncOpenAI)ElasticsearchStore)ConversationalRetrievalChain)ConversationBufferMemory)ConversationBufferWindowMemory)HumanMessage)	AIMessage)HTTPBasicAuth)ElasticsearchNotFoundError)OrderedDict)config)OpenAI)Image)BytesIO)	send_mail)SystemMessage)ChatMessage)Document)DocumentLoader)authenticatelogin)redirect)reverse)ThreadPoolExecutoras_completedz%Y-%m-%dZAlliBot30_applogT)exist_okzcaa_allibot_bot-log-z.logc           	      C   s   t  j}tj|jj}|j}|j	d }t
 d}| d| d|  dd|dd|dd	| d
}ttd}|| W d    n1 sJw   Y  t|dd d S )N__name__z%Y-%m-%d %H:%M:%Sz - z>8sz - [Z25sz : Z3dz] - 
a )end)inspectcurrentframef_backospathbasenamef_codeco_filenamef_lineno	f_globalsr
   nowstrftimeupperopenLOG_FILEwriteprint)	levelmessageframefilenamelinenomodule	timestampZlog_linef rV   H/var/www/html/corporate_allibots_3/corporate_allibot31/allibot3/views.py
custom_logO   s   

2rX   Zorg_auth_indexz$https://chatbot.scic.com/CAA-AlliBotc                 C   s8   d}ddd| igigi}ddi}t j|||d}| S )Nzhttps://generativelanguage.googleapis.com/v1/models/gemini-2.0-flash-001:generateContent?key=AIzaSyA3NBj38uVck1-DusRC-sPdSGT0Lg_ZJ6QcontentspartstextzContent-Typezapplication/json)urljsonheaders)requestspostr]   )user_promptZ
gemini_urlbodyr^   requestrV   rV   rW   !text_generation_google_gemini_apij   s   rd   c                 C   s,   |   dd }ddddd}||dS )N.
image/jpeg	image/pngz	image/bmp)ZjpgZjpegZpngZbmpzapplication/octet-stream)lowersplitget)	file_pathextZmime_maprV   rV   rW   get_mime_type_from_extension   s   rn   caa_allibot_file_doc_index
session_iddocument_extract	file_name
index_namereturnc                 C   s0   t dddd}| ||d}|j||d}|d S )aW  
    Inserts a document into the specified Elasticsearch index.

    Args:
        session_id (str): The session identifier.
        document_extract (str): The extracted text content.
        index_name (str): The Elasticsearch index name (default: caa_allibot_file_doc_index).

    Returns:
        str: The ID of the inserted document.
    ^https://elastic:8oKIqy312EBsAPzWT64NUzji@scic-elasticsearch.es.us-central1.gcp.cloud.es.io:443,  TZrequest_timeoutZretry_on_timeout)rp   rq   rr   indexdocument_id)r$   ry   )rp   rq   rr   rs   esdocresponserV   rV   rW   insert_document   s   r   c                 C   sX   t dddd}dddd| iiii}|j||d	}|d
i d
g }|r*|d d S dS )a  
    Retrieves the document with the given session_id from the specified index.

    Args:
        session_id (str): The session identifier to search for.
        index_name (str): The Elasticsearch index name.

    Returns:
        dict: The matching document or None if not found.
    ru   rv   Trw   querytermrp   valuery   rb   hitsr   _sourceN)r$   searchrk   )rp   rs   r|   r   r~   r   rV   rV   rW   get_document_by_session_id   s   
r   c                 C   s"   t jdd}|jjd| d}|jS )N'AIzaSyA3NBj38uVck1-DusRC-sPdSGT0Lg_ZJ6Q)api_keygemini-2.0-flashmodelrY   )r   ClientmodelsZcount_tokensZtotal_tokens)r[   clientZtokens_counterrV   rV   rW   gemini_token_counter   s
   r   
image_pathc                 C   s   d}t jdtddd}t| }t| d}t| d}W d    n1 s*w   Y  | 	dd	 }|j
j|d
dd| did||digdgd}|jd jjd j S )Ngemini-2.0-flash-001r   Zv1)api_version)r   Zhttp_optionsrbutf-8/rf   userr[   Extracted text in the   image, remove if any ** and ###Zinline_data)	mime_typedata)rolerZ   r   r   )r   r   r   rn   rJ   base64	b64encodereaddecoderj   r   Zgenerate_content
candidatescontentrZ   r[   strip)r   r   r   r   Zimg_fileZimage_base64rQ   r~   rV   rV   rW   generate_content_with_image   s(   r   	xlsx_pathc                 C   s   t | }|jddS )zNExtracts the reference table from an XLSX file and returns it as a CSV string.Fry   )pdZ
read_excelZto_csv)r   dfrV   rV   rW   extract_reference_table   s   
r   c                 C   st   | j d}| j d}| j d}tdd |r1tdd| d| d|  t| d|||d	S tdd
 tdS )Nname	unique_idcompanyCodeinfozentered chatbot functionz'User logged in, rendering chatbot page  zAlliBot_index.html)Z	user_namer   r   z.User not logged in, redirecting to login page.r1   )sessionrk   rX   r   r2   )rc   r   r   r   rV   rV   rW   chatbot   s   

r   c              
   C   s  | j dkr| jd }| jd }| jd }td| td| tdd|  tdd|  ztjtdd	d
dd|iidd|iigiiid}|d d d dkr|d d D ]n}|d }d|v r|d |krtd| d|v ru|d | jd< nd | jd< d|v r|d | jd< nd | jd< td| jd  || jd< || jd< tdd| d tt	  W S t
| dddi  W S t
| dddi  W S n	t
| dddiW S W n ty } ztd"| W  Y d }~S d }~ww t
| dS )#NPOSTusernamepassworduserIdr   username%s z	userID%s r   boolmustr   username.keyworduser_id.keywordr   r   totalr   r   r   sourcer   ZRiea_companyzsession comanr   r   zUser z2 authenticated successfully. Redirecting to index.zAlliBot_login.htmlerrorzInvalid credentialsZerror1Zerror2Zerror3An error occurred: )methodr   rM   rX   r|   r   organ_user_indr   r2   r   r   	Exceptionr   )rc   r   r   r   r~   hitr   erV   rV   rW   
login_view  s`   












 #
r   c           
   
   C   s>  | j dkr| jd }| jd }td| tdd|  zhtjdddd	d
d|iid
d|iigiiid}g }|d d d dkr}|d d D ]7}|d }|dddkrVd}n|dddkrad}nd}|d|d|d|d|d}|| qEtd|iW S  t	y }	 zt
d|	 W  Y d }	~	S d }	~	ww d S )Nr   r   useridr   r   notificationsr   r   r   r   r   r   r   r   r   r   r   r   rc      Approved   DeclinedPendinguser_idrO   
created_atr   r   rO   r   rc   userlistr   )r   r   rM   rX   r|   r   rk   appendr   r   r   
rc   r   r   r~   r   r   r   Zrequest_staZformatted_docr   rV   rV   rW   get_statuslistK  sP   





r   c                 C   s   | j   tdS )Nr1   )r   flushr2   )rc   rV   rV   rW   logout_view  s   
r   c           
   
   C   s>  | j dkr| jd}| jd}d}t | }zgtjtddddd	|iidd
|iigiiid}|d d d dkrx|d d d d }|d}|rotd| d}||7 }td| td|d|gdd t	| dddiW S t	| dddiW S t	| dddiW S  t
y }	 ztd|	 W  Y d }	~	S d }	~	ww t	| dS )Nr   r   r   z/reset-password/r   r   r   r   r   r   r   r   r   r   r   r   mail
user_emailz.Click the link below to reset your password:

rO   zPassword Reset Requestztna@scic.comF)subjectrO   
from_emailrecipient_listfail_silentlyzAlliBot_forgot_password.htmlz'Password reset link sent to your email.r   zEmail not found for this user.User not found.r   )r   r   rk   custom_domainr|   r   r   rM   r+   r   r   r   )
rc   r   r   Z
reset_pathZ	reset_urlr~   Z	user_datar   rO   r   rV   rV   rW   forgot_password_view  sR   






r   c              
   C   s   | j dkrv| jd}| jd}| jd}zEtjtddddd	|iidd
|iigiiid}|d d d dkrX|d d d d }tjt|dd|iid t| dddiW S tdW S  t	yu } ztd| W  Y d }~S d }~ww t| dS )Nr   r   r   new_passwordr   r   r   r   r   r   r   r   r   r   r   r{   r}   r   ry   idrb   zAlliBot_reset_password.htmlrO   z%Password has been reset successfully.r   r   )
r   r   rk   r|   r   r   updater   r   r   )rc   r   r   r   r~   r   r   rV   rV   rW   reset_password_view  sD   


	

r   c                 C   s&   | j d= | j d= td}td|dS )Nr   r   r1   zSignout successful)rO   redirect_url)r   r3   r   )rc   Z	login_urlrV   rV   rW   signout  s   r   c           
   
   C   s0  | j dkr| jd }| jd }td| zhtjdddddd	|iidd
|iigiiid}g }|d d d dkrv|d d D ]7}|d }|dddkrOd}n|dddkrZd}nd}|d|d|d|d|d}|| q>td|iW S  ty }	 zt	d|	 W  Y d }	~	S d }	~	ww d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rc   r   r   r   r   r   r   rO   r   r   r   r   )
r   r   rM   r|   r   rk   r   r   r   r   r   rV   rV   rW   r     sN   





c                 C   sZ   dddd| iidd|iigiidddd	iigd
}t j||d}dd |d d D }|S )Nr   r   r   r   candidate_name.keywordi'  lastmodified_atorderasc)r   sizesortr   c                 S   s   g | ]}|d  qS )r   rV   ).0r   rV   rV   rW   
<listcomp>B      z4get_messages_by_user_and_session.<locals>.<listcomp>r   r|   r   )r   Zind_namecustomerr   r~   messagesrV   rV   rW    get_messages_by_user_and_session/  s   


r   c                 C   sP   t  }| D ]}|dd}tj|}tj|d }|| qd|}|S )Nr   r;   r   z, )setrk   r@   rA   rB   splitextaddjoin)	meta_dataZunique_sourcesitemr   rQ   Zname_without_extZunique_sources_strrV   rV   rW   unique_sources_with_pagesE  s   
r   
SECRET_KEYZOPENAI_API_KEYZGOOGLE_API_KEYZ'AIzaSyALMkcDFVJ0GUpKFpqLqdjjvYsmBxEWuYUes_index_namechatname_indexZtextlensindexZ
formsindexZriea_org_indexZriea_token_indexZcompareindexru   rv   rw   c                  C   s*   t dd} tjd| d}|jdkrdS dS )NZelasticZ8oKIqy312EBsAPzWT64NUzjiz9https://scic-elasticsearch.es.us-central1.gcp.cloud.es.io)auth   TF)r#   r_   rk   status_code)basicr~   rV   rV   rW   check_elastic_statusi  s
   

r   )docx_new_pagepdf_new_pageadditional_commercial_formsdocx_pdf_pagebotcoach_indexhtml_unstructured1excel_json_indpdf_json_indiso_forms_pdf_full_pdfiso_forms_pdf_pagewisejson_othersallibot_faqallibot-faqs-full	caa_indexcaais_index&carrier_appointments_master_list_caais#carrier_appointments_master_list_cl)carrier_appointments_master_list_personal,carrier_appointments_master_list_wholesalersr  r  r  r  r  r  Zes_connectionrs   Z	embeddingZstrategyc                 C   sT   ddd|iid}t j| |d}d}|d d D ]}|d }d	|v r'|d	}q|S )
N  r   session_id.keyword)r   r   r   r;   r   r   chat_id_name)r|   r   rk   )rs   rp   r   r~   r  r   r   rV   rV   rW   get_chat_id_names_by_session_id  s   

r  c	                 C   s   t |tr	| stdd|  d S | ||||tt d tt d tjtt	|ddd|||d}	t
j||	d}
|
d	}|S )
Nr   z5Invalid user_prompt: must be a non-empty string, got   r   Tensure_asciicandidate_name	modelnamer   rp   ra   r   r   historyzhuman tokenis_safeflagged_categoriesr   r{   )
isinstancestrr   rX   roundr   r]   r   r   r!   r|   ry   rk   )r   r   rp   ra   ry   
user_tokenr  r   r!  r}   resZdocument_idrV   rV   rW   add_user_message_to_es  s*   
r'  c           
      C   s`   | ||||t t d t t d tjtt|dddd||ddddd}tj||d	}	|	S )
Nr  r   )r   r   Tr  F)ZviolenceZhateZ	self_harmr  r   )r$  r   r]   r   r   r-   r|   ry   )
r   r   rp   ra   ry   r%  r  r   r}   r&  rV   rV   rW   add_chat_message_to_es  s(   r(  c                 C   sT   | ||||t t d t t d tjtt|ddd|dd
}tjt|d}|S )Nr  r  Tr  null)
r  r  r   rp   answerr   r   r  zassistant tokenfeedbackr   )	r$  r   r]   r   r   r"   r|   ry   r   )r   r   rp   r*  assistant_tokenr  r}   r&  rV   rV   rW   add_assistant_message_to_es  s    r-  c                 C   sP   | |||t t d t t d tjtt|ddddd}tjt|d}|S )Nr  r  Tr  r)  )r  r   rp   r*  r   r   r  r+  r   )	r$  r   r]   r   r   r,   r|   ry   r   )r   r   rp   r*  r}   r&  rV   rV   rW   add_system_message_to_es  s   r.  c                 C   s8   ddd|iidddiigd}t j| |d}|d	 d	 S )
Nr  matchrp   r   r   r   )r   r   r   r   r   r   )rs   rp   r   r~   rV   rV   rW   search_by_session_id  s   r0  c                 C   sz   t | drd| jv rtj| jd }|| jd< t| tr*| dd| dddS | jt | dr9t	
| jdS ddS )Nmetadatar   page_contentr;   r2  r1  )hasattrr1  r@   rA   rB   r"  dictrk   r2  r]   r   )r}   rr   rV   rV   rW   document_to_dict  s   



r6  c           	   	   C   sR   dd |D }| ||t t d t t d tj|dd|d}tj||d}|S )Nc                 S      g | ]}t |qS rV   )r6  r   r}   rV   rV   rW   r   %  r   z)add_document_ai_to_es.<locals>.<listcomp>r  Tr  )r   rp   ra   r   r   r  rQ   r   )r$  r   r]   r   r|   ry   )	r   rp   docs1r   ZorderidfileZdocuments_dictsr}   r&  rV   rV   rW   add_document_ai_to_es#  s   r;  c                 C   s4   t t  d }dd|ii}tj| ||d d S )Nr  r}   r   r   )intr
   rG   rT   r|   r   )rs   doc_idZcurrent_timestampZupdate_bodyrV   rV   rW   update_lastmodified_at7  s   r>  c                 C   s*   t | |}|D ]}|d }t| | qd S )Nr{   )r0  r>  )rs   rp   	documentsr}   r=  rV   rV   rW   update_create_at_with_sessionidE  s
   
r@     rB  c                 C   sX   t | }||t j t }|j|dd | W  d    S 1 s%w   Y  d S )NBMPformat)r)   rJ   	thumbnailLANCZOSr*   savegetvalue)r   r   imgbufferrV   rV   rW   resize_imageN  s   $rL  c                 C   s   t | dS )Nr   )r   r   r   )Z
image_datarV   rV   rW   encode_imageU  s   rM  c                   C   s   t   S N)r
   utcnow	isoformatrV   rV   rV   rW   get_current_iuser_idso_dateX  s   rQ  c                 C   s   | j ||d d S )Nrx   r   )r|   rs   
chat_entryrV   rV   rW   store_chat_history[  s   rS  c                 C   s2   t j| |d}|d r|d }|d |d fS dS )Nry   r   foundr   r?  chat_history)NN)r|   rk   )	indexnamerp   r~   Z	chat_datarV   rV   rW   retrieve_chat_data^  s
   rX  c                 C   sB   ||t t d t t d t|t|g| d}t||| d S )Nr  )r   rp   r   r   r   rJ  r$  r   img_message_to_dictrS  )Zbase64_imager   rO   r~   rp   Zindex_name_imgr|   rR  rV   rV   rW   chat_hist_imgj  s   r[  c                 C   s   t | tr| j}t |trddd |D }dd|idS t | tr<| j}t |tr5ddd |D }dd|idS td	t|  )
Nr   c                 S       g | ]}|d  dkr|d qS typer[   rV   r   partrV   rV   rW   r          z'img_message_to_dict.<locals>.<listcomp>humanr   r^  r   c                 S   r\  r]  rV   r_  rV   rV   rW   r     ra  aizUnexpected message type: )r"  r!   r   listr   r"   
ValueErrorr^  )rO   r   rV   rV   rW   rZ  |  s   



rZ  c                 C   sd   t | trdd | D S | d}|dkrt| d d dS |dkr+t| d d dS td	| )
Nc                 S   r7  rV   )img_message_from_dict)r   mrV   rV   rW   r     r   z)img_message_from_dict.<locals>.<listcomp>r^  rb  r   r   r  rd  zGot unexpected message type: )r"  re  rk   r!   r"   rf  )message_dictZmessage_typerV   rV   rW   rg    s   

rg  c                 C   s(   | j |dd|iiddd}|d d S )Nr/  rp   r  r   r   r   r   )r   )r|   rs   rp   r&  rV   rV   rW   retrieve_chat_history  s   
rk  c                 C   s@   | |t t d t t d t|t|gd}t||| d S )Nr  )r   rp   r   r   r   rY  )r   rp   rO   r~   r|   rs   rR  rV   rV   rW   chat_hist_txt  s   rl  c                 C   B   t | d}t| dW  d    S 1 sw   Y  d S Nr   r   rJ   r   r   r   r   )r   
image_filerV   rV   rW   encode_image1     $rq  c                 C   rm  rn  ro  )r:  rp  rV   rV   rW   avi_to_base64  rr  rs  c                 C   s   t | O}d}||t j |jdkr4|d}| dd}|j|dd t	|  |W  d    S | d|j
  d|j
  }|| |W  d    S 1 sWw   Y  d S )	N)   rt  rC  ZRGB.bmp.jpegZJPEGrD  re   )r)   rJ   rF  rG  rE  convertreplacerH  r@   removeri   )r   rJ  max_sizeZconverted_pathrV   rV   rW   resize_and_convert_image  s   


$
$r{  c              
   C   s  t j| \}}d}d}z| dkrt| }n| dkr%t| }nx| dkr1t| }nq| dv r=t| }nj| dkrIt	| }nc| dkrUt
| }n\| d	krat| }nU| d
krmt| }nN| dkryt| }nG| dkrt| }n@| dkrt| }n9td| W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS W ||fS  ty } zd}W Y d }~||fS d }~ww )N   r;   z.xlsxz.csvz.tsv)z.docz.docxz.pdfz.pptxz.xmlz.jsonz.txtz.mdz.htmlzUnsupported file type: r   )r@   rA   r   ri   document_loaderZexcel_file_loaderZcsv_file_loaderZtsv_file_loaderZdoc_docx_file_loaderZpdf_file_loaderZpptx_file_loaderZxml_file_loaderZjson_file_loaderZtext_file_loaderZmarkdown_file_loaderZhtml_file_loaderrf  r   )Z
file_name1_Zextension_file_name1respr9  r   rV   rV   rW   add_documents  sl   
r  c                 C   rm  rn  ro  )rl   r:  rV   rV   rW   encode_file  rr  r  c                 C   s   zt j|| d}|d d }W n tjy   g }Y nw |dd|ii t j|| | |tt d tt d |||ddd	d
 d S )NrT  r   r   r   r   r  )rp   r   r   r   r   r  alliance_numberTr}   Zdoc_as_upsertr   )r|   rk   elasticsearchr%   r   r   r$  r   )	sessionidr   Znew_contentrW  	candidater  r}   Zexisting_messagesrV   rV   rW   index_chat_message  s2   	
r  c           	      C   sz   |  dd }tdd| ddddd	t|  id
gd}t|}t|||g||| tddd	t|  id
gdS )Nr   rf   r[   r   r   r]  	image_urlr\   zdata:image/jpeg;base64,r^  r  r  )rj   r!   r  message_to_dict_imgr  )	rl   r  r   rW  r  r  rQ   rO   Z
image_convrV   rV   rW   image_upload  s    
r  c                 C   s|   g }| D ]7}|j dkr|d|jd q|j dkr-|j}d| dg}|d|d q|j dkr;|d|jd q|S Nrb  rc  rd  r[   r]  system)r^  r   r   )histZ	hist_listr  Zai_histai_hist_contentrV   rV   rW   message_to_dict_list5  s   


r  c                 C   s`   | j dkr| j}d|dS | j dkr!| j}d| dg}d|dS | j dkr.| j}d|dS d S r  )r^  r   )r  Zcontr  rV   rV   rW   r  B  s   





r  c           	      C   s  g }| D ]}t |trE|ddkrt|d}|| q|d ddkrD|dg }|rDt |trDt|d dd}|| qt |tr|D ]=}|ddkrbt|d}|| qL|ddkrvt|d}|| qL|ddkrt|d}|| qLq|S )	Nr^  rb  r   rd  r   r[   r;   r  )r"  r5  rk   r!   r   re  r"   r,   )	Zhis_listZcon_listiZhuman_contentZai_dataZai_conltZ
ai_contentZsys_contentrV   rV   rW   message_from_dict_listP  s2   



r  c           	   
   C   s   t j|ddddd| iigiiid}g }z+|d d d	 d
kr>|d d D ]}|d }|d D ]}|d d }|| q/q%W n tyV } ztd| W Y d }~nd }~ww |r[|S d S )Nr   r   r   r/  rp   r   r   r   r   r   r   r   r   r   z"error inside retrieve chat content)r|   r   r   r   rX   )	r  rW  r~   rY   r   rz   rO   r   r   rV   rV   rW   retrieve_chat_contentj  s4   
r  c                 C   s@   t dd}d}tj||d}| }| t|dg }||}|S )Ngpt-4or   
similaritysearch_typer  )r   dbr   r"   invoke)Zpast_messagesZ
user_queryllmr  retrieved_dataformatted_dataZcombined_messagesr~   rV   rV   rW   query_and_combine  s   
r  c                 C   sh   ddddd|iidd|iigiii}t j| |d}|d }|d	kr.td
| d| d d S td d S )Nr   r   r   r/  rp   ra   r   deletedr   zSuccessfully deleted z documents with session_id 'z' and user_prompt 'upload1'.zJNo documents found with the provided session ID and user_prompt 'upload1'.)r|   Zdelete_by_queryrM   )rs   rp   Zcondition_checkr   Zdelete_responseZdeleted_countrV   rV   rW   delete_document_content  s   

r  stringr   c                 C   s   t |}t|| }|S )z.Returns the number of tokens in a text string.)tiktokenZget_encodinglenencode)r  r   encoding
num_tokensrV   rV   rW   num_tokens_from_string  s   
r  c                 C   sD   t | trtdd t|  D S t | tr tdd | D S | S )z1Convert dictionary to a tuple that can be hashed.c                 s   s     | ]\}}|t |fV  qd S rN  dict_to_tuple)r   kvrV   rV   rW   	<genexpr>  s    z dict_to_tuple.<locals>.<genexpr>c                 s   s    | ]}t |V  qd S rN  r  )r   r  rV   rV   rW   r        )r"  r5  tuplesorteditemsre  )drV   rV   rW   r    s
   

r  c                 C   s   |  dd}|S )N"z\")rx  )r[   Z
fixed_textrV   rV   rW   fix_syntax_error  s   r  c                 C   s:  d }t jtdddd| iigdd|iidddddiiiigd	iid
d d }|rd}|D ]J}|d }	|d }
|
d|krwd}|
dd}|
dd}|| }|| }|| }d|||||| ddi}t jt|	|d td|  |	} nq-|s|||||| | ddd}t jt|d
}td|  |d }|S |S )Nr   r   r/  orgnamer   Zmust_notexistsfield)r   Zshouldr   r   Fr{   r   Tinput_tokensr   output_tokensr}   r|  )r   r   r  r  overall_tokensr  limits_tokenr   zDocument updated for username:   )r   r   r  r  r  r  r  assigned_tokensz#New document created for username: )r|   r   user_details_indexrk   r   rM   ry   )org_namer   r  r  mailidZ
corp_orgidZuser_tokensZusername_foundr}   r=  
doc_sourceinpoutpinput_overall_tokensoutput_overall_tokensr  Zupdated_docZnew_docresultrV   rV   rW   user_org_tokens  sx   

%
r  c                 C   s   t jtddd|iiidd d }|rZ|D ]
}|d }|d }q|dd	}|d
d	}|dd	}	| | }
|| }|
| }t||	 d }d||
|||di}t jt||d}d S d S )Nr   r/  organization_namer   r   r{   r   r  r   r  r  d   r}   )r  r  r  r  Z
percenatger   )r|   r   org_details_indexrk   r   r   )r  r  r  Zorg_toksr}   r=  r  r  r  r  r  r  r  Z
percentageZ
update_docr&  rV   rV   rW   	org_tokenK  sD   		

r  c              
   C   s   z+t  }|jjd| d}| }|d d d }|d d d }| |r(|fW S i fW S  tyH } ztd|  di fW  Y d	}~S d	}~ww )
zg
    Check user input using the Moderation API.
    Returns a tuple (is_safe, flagged_categories).
    zomni-moderation-latest)r   inputresultsr   flagged
categorieszModeration API failed: TN)r(   ZmoderationscreateZto_dictr   rM   )r[   r   r~   r:   r  r  r   rV   rV   rW   moderate_inputy  s   r  c           R   
      s
  zK| j dkrG| jd}| jd}| jd}td| tddd}tjtj	t
d	  tj s<t  |d
krtd | jd}| jd}| jd}| jd}| jd}	| jd}
| jd}tt| t|d}t|}t|\}}t|	|||t|d||	 |r{zt d r_|r_ fdd|D }|r_g }g }|D ]}| drt|||t|	}tt|d}|t| ||j tj|}t |d| qt!|\}}|"dd }dd |D }t#dd| d dd!| d"d#| d gd}d#|}d$| d%| d&| d'}t$|}|d(d) d*d+d) d}t|}t%|||gt|	| |tt|d || tj|}t |d| qW n t&y| }  zt|  td,|  W Y d } ~ nd } ~ ww t d }!|!D ]}"t'|" q|rt(d- t)|t}#|#r|#}$t*|$}$ng }$d.}%t+d/d0d1d2}&t,|&g d3t-t,. d4}'t#d|d gd}(d5})|'j/||)d6}*g }g }+|*D ]"},|,j0d7d8}-tj1tj|-d) }.|+|. d9d |*D }qd#|+}/t#|d}0t2|}1|1r td:|1 |1d;}/|r|$|(g }2n|$|(g }2ntd< |r0|$|0g |(g }2n|$|(g }2d=}%d>| d?| d@|# dA}3tdB|3 |2|% tddC|  tddD|	  dEdFdGdHdI|iidHdJ|	iigiii}4|&j/t3|4dK}d)}5d)}6tdLdM|  |dN dN D ]}7tddO|7  |7dP dQd)}5|7dP dRd)}6qzt$|3}8|8d(d) d*d+d) d}8tdS|8 W n  t&y }  ztdT|  t4dU|dVW  Y d } ~ W S d } ~ ww t|8}9tddW|5  tdX|5 tddY|9  tdZ|9 |5|6|9  d)k rWtd[ |&j/t3dEdFdGdHdI|iidHdJ|	iigiiidK}|dN d\ d] d)krH|dN dN d) d^ }:|&j5t3|:d_d`dad)idbidc |&j/t3|4dK}t4dd|dVW S t6|(g};|;tt|8d t%|||;t|	| t|8}9dEdFdGdHdI|iidHdJ|	iigiii}4|&j/t3|4dK}d)}5d)}6|dN dN D ]}7|7dP dQd)}5|7dP dRd)}6q|8}<t7|	|||<|9d t8||9|
 t9|
|	||9| z1t:|}=t:|}>t:|<}?t:|/}@de|> df|? df|= df|@ }At;|A tdg|A t4|<|dVW W S  t&y }  ztdT|  t4|<|dVW  Y d } ~ W S d } ~ ww g }Bt< }C|D ]}Dt=|D}E|E|Cvr1|B|D |C>|E qt?d)t@|BD ]}t%|||B| gt|	| q:dhtAt<|}Ft|F}9t|Fd}Gt7|	|||F|9d t8||9|
 t9|
|	||9| t4|F|dVW S t4di||djW S |dkkrJ| jBdl}tdm |st4dndodpdqdrW S |D ].}tj |jC}HtD|Hds}I|E D ]}J|IF|J qW d    n	1 sw   Y  qzAt dt t du  t dv  t dw  }Kg d}Lg }K|LD ]}M|KGttj dx|M  q|KD ]}NtH|N}Oqdy}PW n t&y' }  zd)}PW Y d } ~ nd } ~ ww |Pdykr=tIt@|dz }Qt4d{|QdpW S d|}Qt4d}|QdpW S W d S W d S  t&yg }  zt4dntI| iW  Y d } ~ S d } ~ ww )~Nr   operational_typeemailr   r   r   r   )r   r   z
/image_pthaction1zinto action 1r   rp   rO   zfile_names[]r  r  r   cl100k_basegemini-flash-2.0/*c                    s   g | ]} d  | qS )r   rV   )r   rQ   
upload_dirrV   rW   r         zallibot.<locals>.<listcomp>.jpgrv  .pngru  r  Truer   rf   c                 S   s   g | ]}|j qS rV   r2  r   r   rV   rV   rW   r     s    r[   z
Filename: r]  z2Here is the text content of the uploaded document z:
r9   z
                                                The following is the extracted text content from the uploaded document titled "z4":

                                                au  

                                                Please carefully review the content above and generate a well-structured summary in paragraph form.
                                                The summary should be clear, concise, and limited to approximately 300 words.
                                                If it is relevant, you may mention the filename "zB" in the summary.
                                                r   r   r   rZ   z,info - Exception occured in the image uploadr   a_  Answer the question in your own words as truthfully as possible from the context given to you.
                                You are a helpful assistant. 
                                Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                                Give a line of space between the paragraphs.
                
                                If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.
                
                                If questions are asked where there is no relevant context available, simply respond with:
                                "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"
                                Remove ** or any ### from the response and don't mentions that in response about the formatting
                                Answer the question in a natural way and even though if I ask the question along with the context, only answer the question and don't mention about the context in the response similarly provided on the context.
                                        ru   rv   Trw   )r   r  r  r  r  r  r  r  r  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r   r;   c                 S   &   g | ]}|j  d krd|j dqS z{"text": " "}r[   r]  r2  r   r  rV   rV   rW   r   "  
    
zDocument found:rr   z'No document found with that session_id.a`  Answer the question in your own words as truthfully as possible from the context given to you.
                                You are a helpful assistant. 
                                Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                                Give a line of space between the paragraphs.
                
                                If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.
                
                                If questions are asked where there is no relevant context available, simply respond with:
                                "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"
                                Remove ** or any ### from the response and don't mentions that in response about the formatting
                                Answer the question in a natural way and even though if I ask the question along with the context, only answer the question and don't mention about the context in the response similarly provided on the context.

                                        zAnswer the question in your own words as truthfully as possible from the context given to you.
                                You are a helpful assistant. 

                                user question:z+

                                context: z0

                                chat history: a  


                                Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                                Give a line of space between the paragraphs.
                
                                If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.
                
                                If questions are asked where there is no relevant context available, simply respond with:
                                "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"
                                Remove ** or any ### from the response and don't mentions that in response about the formatting
                                Answer the question in a natural way and even though if I ask the question along with the context, only answer the question and don't mention about the context in the response similarly provided on the context.

                                        Clean_templatezalliance number z
user name r   r   r   r   r   r   r   Zinfo1zuser detail: r   zuser details r   r  r  img_response1zerror at allibot responsezCThere was an error processing your request. Please try again later.r~   rp   zassigned_tokens zassigned tokenszoutput tokens Zoupzif satisfiedr   r   r{   scripta  
                                                if (ctx._source.containsKey('limits_token')) {
                                                    // If limits_token exists, update its value
                                                    ctx._source.limits_token = params.limits_token;
                                                } else {
                                                    // If limits_token doesn't exist, create it and set the value
                                                    ctx._source['limits_token'] = params.limits_token;
                                                }
                                            r  )r   paramsr   zbYou have reached your usage limit. Please contact your administrator to request additional tokens.zF/var/www/html/CAA_Allibot30/AlliBot30_app/Quesgen_answer_relevancy.py r   zerror at quest1

jThe Question you asked violates AlliBot usage policies.Don't repeat this from blocking of AlliBot account.r~   rp   r!  action2fileszinto action 2r   No files uploadedstatusrO     r  wb+/*.jpg/*.jpeg/*.png/*.bmp*r|   uploads completesuccesszUnable to process image.failed)Jr   r   rk   r   rX   r   r@   rA   r   dirname__file__r  makedirsrM   getlistr@  r   r  r   r  r'  r   ri   endswithr  imtext_indexr"   r   r   r  r   rB   r   r  rj   r!   rd   r  r   ry  r   r  r  r$   r   
embeddingsExactRetrievalStrategyr   r1  r   r   r  r   r   r  r-  r  r  r   popenr   r  r   ranger  re  FILESr   rJ   chunksrL   extendr{  r#  )Rrc   action_typer  r   r  r   rp   ra   r  r  r  r  input_token_cost_gpt4oZinput_token_cost_gemini_flash_2r   r!  Za_with_pathZai_type_convZimg_only_respr  Z	encodeimgr~   Zfilename_onlyind_response1r9  r:  r  rO   Zjoined_dataZtemplate_filerY   Zimg_doc_contr   	files_dirrU   ret_chatconverted_chat
clean_tempr|   r  
user_inputr  r  Zsource_data_histr   source_pathZsource_nameZsource_string
ai_messager  all_chatr  r   r  r  r   r  Z output_token_cost_gemini_flash_2r   r  Z
clean_respjson_strZ	user_quesZanswer_promptZunique_entries_strcmdZunique_dataseenr  Z
dict_tupleZonly_files_responseoutput_token_cost_gpt4orl   destinationchunkimage_filessupported_formatsrm   rp  processed_image_pathZind_responseres_msgrV   r  rW   allibot  s>  





	$













$

















0"
      &r  c                 C   s4   t  }|jjjdd| dgdd}|jd jj}|S )Nr  r   )r   r   F)r   r   streamr   )r(   ZchatZcompletionsr  choicesrO   r   )formatted_templater   r~   assistant_responserV   rV   rW   get_resptableA  s   
r$  c           K      C   s&  zn| j dkrj| jd}|dkr
tdd | jd}| jd}| jd}td	| | jd
}| jd}| jd}tj|dd}z5tt| t	|\}	}
d}zt
||||t|d|	|
	 W n ty{ } ztd| W Y d }~nd }~ww W n ty } ztd| W Y d }~nd }~ww |	rd}tdddd}td|dgd}d}tj||d}dd |D }td| t|t}td| |rt|ttfr|}t|}ng }t|d}tjtdd|iid d!}|rt|d" d" dkrd#d |d" d" D }ng }g }|r[|D ]K}|d d$r"t|d d$nd%}|d d&}t|tsPzt|tr=t|ni }W n tjtfyO   i }Y nw |t||d' q|ri||g |g | }n||g }d(}|| td| d)d*d+d,d|iigiii}tjt|d-}d}d} |d" d" D ]}!|!d. d/d}|!d. d0d} qtd/| td0|  d1d2d3 |D  }}d4| d5| d6| d7}td8| t |}"|"d9d d:d;d d}"td|" d}#t!||||"|#d t"|"|d<W S t"d=||
d>W S |d?kr| j#$d@}$| jd}%| jdA}|$s.t"dBdCdDdEdFW S t%j&t%j&'t(dG }&t%j&)|&sFt%*|& |$D ].}'t%j&|&|'j+}(t,|(dH})|'- D ]}*|).|* q\W d    n	1 spw   Y  qHt/|&dI }+t0t|%dJ t/|&dK t/|&dL  t/|&dM  t/|&dN  },|,rg dO}-g },|-D ]}.|,1t/t%j&|&dP|.  qt2dQdR}/|,D ]W}0tdS|0 t3|0}1tdT|1 t4|1}2|15dUs|15dVrdW}3n|15dXrdY}3ndW}3|/6t7dZdtddZdd[d\d]|3 d^|2 id_gdg}4d`}5|4j8}6tda|4j8 qnt9|+d \}5}6|+d :dbdc }'t|6ts3|6g}6|6rst;|6D ].\}}7t|7trWdd|' de|7 df|+d id'|6|< q:t<|7d$rgdd|' dg|7j=i|7_=q:t>||%|6tdJ|'  |+D ]}8t%?|8 qu|5d`krtt|$dh }9t"di|9dDW S dj}9t"dk|9dDW S |dlkr | j#$d@}$| jd}%| jdA}|$st"dBdCdDdEdFW S t%j&t%j&'t(dG }&t%j&)|&st%*|& |$D ].}'t%j&|&|'j+}(t,|(dH})|'- D ]}*|).|* qW d    n	1 sw   Y  qt/|&dI }+t0t|%dm t/|&dK t/|&dL  t/|&dM  t/|&dN  },|,rg dO}-g },|-D ]}.|,1t/t%j&|&dP|.  q9t2dQdR}/|,D ]G}0t3|0}1t4|1}2|15dUsi|15dVrldW}3n|15dXrudY}3ndW}3|/6t7dndtddZdd[d\d]|3 d^|2 id_gdg}4d`}:|4j8};qSnt9|+d \}:};|+d :dbdc }'t|;ts|;g};|;rt;|;D ].\}}7t|7trdd|' de|7 df|+d id'|;|< qt<|7d$rdd|' dg|7j=i|7_=qt>||%|;tdm|'  |+D ]}8t%?|8 q|:d`krtt|$dh }9t"di|9dDW S do}9t"dk|9dDW S |dpkrm| jd}| jd}| jd
}| jd}| jd}dq}tt| t	|\}	}
d}	d}
dr}t@||||t|d|	 |	rads}tdddd}tjtdd|iid d!}|rt|d" d" dkrdtd |d" d" D }ng }d}<i }=|D ]N}|<d`7 }<g }>|D ]5}?|?d$i }@|?d&i }At|A}B|Bdf}At|@trtAdud3 |@B D d%}Cnt|@}C|>|C qdv|>|Adw|=dx|< < qdy}DtC|D}E|=dzd:}F|=d{d:}Gtd| |=dzd}}H|=d{d}}Itd~ d|H d|F d|I d|G d|E d}Jt |J}"|"d9d d:d;d d}"|"Ddd%}"|"Ddd%}"|"Ddd%}"tE||||" t"|"|d<W S t"d=||
d>W S W d S W d S  ty } ztdBd|  t"dBt|iW  Y d }~S d }~ww )Nr   r  r  r   zinto action1r   rp   rO   zprompt at allidocsr  r  r  r  r  r   r  z
error at 0z
error at 1a+  Answer the question in your own words as truthfully as possible from the context given to you.
                    You are a helpful assistant. Include the filename within your response where relevant.
                    Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                    Give a line of space between the paragraphs.

                    If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.

                    If questions are asked where there is no relevant context available, simply respond with:
                    "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"

                    Context: {context}  

                    {chat_history}  
                    Human: {question}  
                    Assistant:rV  Tr*  Z
memory_keyZreturn_messagesZ
output_keyr[   r]  r  c                 S   r  r  r  r  rV   rV   rW   r     r  z formscompare.<locals>.<listcomp>zformatted datazreieved datar/  lastmodified_at:ascry   r   r   r   c                 S      g | ]}t |d  d qS r   r  r]   loadsr   rz   rV   rV   rW   r         r2  r;   r1  r3  al  Answer the question in your own words as truthfully as possible from the context given to you.
                                You are a helpful assistant. 
                                Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                                Give a line of space between the paragraphs.
                
                                If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.
                
                                If questions are asked where there is no relevant context available, simply respond with:
                                "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"
                                Remove ** or any ### from the response and don't mentions that in response about the formatting
                                        r   r   r   r   r   r   r  r  r   c                 s   s    | ]}|j V  qd S rN  r  r8  rV   rV   rW   r    s    zformscompare.<locals>.<genexpr>a#  Answer the question in your own words as truthfully as possible from the context given to you.
                                    You are a helpful assistant. Include the filename within your response where relevant.
                                    Response must be in and around 200 words. It must be in paragraph manner and it must not exceed 4 paragraphs.
                                    Give a line of space between the paragraphs.

                                    If there is extensive context, use as much as possible to form a detailed response around 200 words. If the context is minimal, provide a response based on the available information, without the need to meet the word limit.

                                    If questions are asked where there is no relevant context available, simply respond with:
                                    "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"

                                    Context: z(  

                                    z.  
                                    Human: z1  
                                    Assistant:templater   r   rZ   r  r  r  r  r  r   r   r  r  r  r  z/formsupload_filesr  r  Zupload1r  r  r  r  r  r  r   r  rp  r  rv  r  rg   r  rh   z?You are a useful bot that is especially good at OCR from imagesr  r\   zdata:z;base64,r  r|  zimg outr   rf   zcontext of the first uploaded z image: r   z	 documentr  r  z$unable to process upload1 documents.r  Zaction3Zupload2zzYou are a useful bot that is especially good at OCR from images and don't respond with these ** and ### special charactersz$unable to process upload2 documents.Zaction4a}  Extract the following information from each uploaded document and return the output strictly in a table format only, with column names as the document names and row fields as: Policy Number, Policy Effective Date, Policy Issue Date, Schedule of Forms and Endorsements, Common Policy Declarations and Conditions, Declarations, Coverage, and Endorsements.

                Ensure the following:  
                1. The Policy Number format must strictly follow the pattern of two alphabetic characters, a space, followed by two digits, a space, repeated thrice (e.g., AG 00 02 30 06). The Policy Number is typically found in the header or footer of the document. If the policy number field appears empty, perform a thorough search across the document, including all sections (header, footer, body text, metadata, etc.), to ensure the policy number is identified and extracted wherever it exists.
                2. If any field is missing or not found in a document, include only the data from the fields that are present. Use as much detail as possible, up to 100 words, in each field that is available. Leave missing fields blank in the table.  

                For Specific Fields:  
                - Schedule of Forms and Endorsements: Get data based on the list of policy forms, schedules, and endorsements by line of business.  
                - Declarations: Include line of coverage-specific declarations (general liability, business auto, business income, contractors equipment, etc.).  
                - Coverages: Provide a comparison of coverages offered by the underlying policy (general liability, business auto, business income, contractors equipment, etc.).  
                - Endorsements: List the name of each endorsement included in the document with a summary of how each endorsement affects the underlying policy.  

                Response Format:  
                - Provide the extracted data in a table format as described above.  
                - Include an "Overall Summary" section directly under the table. This summary must summarise the table data's important observations across the documents, titled as **summary**.
                Z987a?  Answer the question in your own words as truthfully as possible from the context given to you.  
                    You are a helpful assistant. Include the filename within your response where relevant.  
                    Extract the following information from each uploaded document and return the output strictly in a table format only.  

                    Table Format Requirements:  
                    - The table must have a header row with "Field Name" in the first column, followed by the uploaded document names as subsequent column headers (e.g., "Uploaded Document 1 Name" and "Uploaded Document 2 Name").  
                    - Row fields should include the actual values for the following fields: Policy Number, Policy Effective Date, Policy Issue Date, Schedule of Forms and Endorsements, Common Policy Declarations and Conditions, Declarations, Coverage, and Endorsements.  

                    Ensure the following:  
                    1.The Policy Number must follow one of the patterns below:
                    - Two alphabetic characters, a space, followed by two digits, a space, repeated thrice (e.g., AG 00 02 30 06).
                    - Three alphabetic characters, a space, followed by two digits, a space, repeated twice (e.g., CNI 90 22 11 22).
                    - Two alphabetic characters, a space, another two alphabetic characters, a space, followed by two digits, a space, and repeated once (e.g., CG DS 01 10 01).
                    - Two alphabetic characters, a space, one alphabetic character, a space, followed by three digits, a space, then two digits, a space, and two digits (e.g., IL U 002 05 10).
                      The Policy Number is usually found in the header or footer of the document. Ensure to search these sections carefully when extracting the number.
                    2. If any field is missing or not found in a document, include only the data from the fields that are present. Use as much detail as possible, up to 100 words, in each field that is available. Leave missing fields blank in the table.  
                    3. Perform a comparison for each field across all uploaded documents to ensure consistency.

                    For Specific Fields:  
                    - Schedule of Forms and Endorsements: Get data based on the list of policy forms, schedules, and endorsements by line of business.  
                    - Declarations: Include line of coverage-specific declarations (general liability, business auto, business income, contractors equipment).  
                    - Coverages: Provide a comparison of coverages offered by the underlying policy (general liability, business auto, business income, contractors equipment).  
                    - Endorsements: List the name of each endorsement included in the document with a summary of how each endorsement affects the underlying policy.  

                    Response Format:  
                    - Provide the extracted data in a table format as described above.  
                    - Include an "Overall Summary" section directly under the table. This summary must summarise the table data's important observations across the documents, titled as summary.  

                    Additional Note:  
                    If questions are asked where there is no relevant context available, simply respond with:  
                    "That's a great question! I'm not sure of the answer right now. Can you ask your question a different way? I am excited to assist you further!"

                    Context: {context}  

                    {chat_history}  
                    Assistant:c                 S   r(  r)  r*  r,  rV   rV   rW   r   1	  r-  c                 s   s    | ]
}t |tr|V  qd S rN  )r"  r#  )r   r  rV   rV   rW   r  G	  s    r  )r   r   Zdoc_z-/var/www/html/AlliBot_30/table structure.xlsxZdoc_1Zdoc_2z
pdf passedr   zpdf name passedzI
                    The extracted structured data for the file name 1 - z:
                    zJ

                    The extracted structured data for the file name 2 - zz

                    Compare both documents and highlight differences and give me a table like response as in this excel uu  .
                    - start with 'Here’s a comparison of the two policies:' and don't mention anything than that. 
                    - Don't mention 'based on the structured data provided.' or anything based on the structure in the response.
                    - give me the name of the file names along with the extesion of the file too.
                    - Don't provide me the file name exclusively in a different field instead give as a topic of a column
                    - Give me the summary of the 2 policies in the last row of the table - Important Don't miss this
                    - give me the response in html and inside the cell give me proper line spacing.
                    - Give the form numbers in seperate lines.
                    - In the ‘Schedule of Forms and Endorsements’ row, list each form number on a separate line.
                    u*   Here’s a comparison of the two policies:z```htmlz```zerror at formscompare: )Fr   r   rk   rX   r   r  r   r@  r   r  r'  r   r   r!   r  r"  re  r5  r  r|   forms_indexr  r#  r]   r+  JSONDecodeError	TypeErrorr   BaseDocumentr  rM   r   rd   r-  r   r  r   r@   rA   r  r  r  r  r   rJ   r  rL   r   r  r	  r   r{  rq  r  r  r"   r   r  rj   	enumerater4  r2  r;  ry  r(  nextvaluesr   rx  r.  )Krc   r
  r   rp   ra   r  r  r  r  r   r!  r%  r   r.  memoryr  r  r  r  r  r  r  	doc_itemsuploaded_document_docsr  r2  r1  r  r  r   r~   r  r  r   r#  r,  r  Z	conver_idr  r:  rl   r  r  r  r  r  rm   chainrp  r  imager   msgr  r9  r}   rU   r  Zind_response2Zdocs2countZdocsZdoc_pagejr   r   Zparsed_datar   Z
excel_pathZtable_structureZ	pdf1_textZ	pdf2_textZ	pdf1_nameZ	pdf2_nameZcomparison_promptrV   rV   rW   formscompareL  s  










&







$



0"

%



0"
#



'



$
        +r>  c              
   C   sb   | j dkr)ztddiW S  ty( } ztdt|iddW  Y d }~S d }~ww tddiddS )	Nr   Zactivate_convTr   r  r  zMethod not allowed  )r   r   r   r#  )rc   r   rV   rV   rW   activate_query_conversation	  s   
 r@  c           
   	   C   s(  z~| j dkr|td | jd}| jd}| jd}| jd}td tj|dd	|iid
dd}|d d d }|dkrr|d d D ]#}|d }d|d v ri|d d }	|	|kritj||d|iddd qFtdddW S tdddddW S W d S  t	y   td| d| d  Y d S w )!Nr   zthumbs flagrp   rW  r*  thumbs_flagr   r   r  r  rj  r   r   r   r   r   r{   r   r+  Tr  r   r  zDocument updatedr  ZfailurezNo documents foundi  r  z)Error: No documents found for session_id z and answer 'z'.)
r   rM   r   rk   r   r|   r   r   r   r%   )
rc   rp   rs   r*  rA  r~   Z
total_docsr   r=  r  rV   rV   rW   update_thumbs_flag	  sH   


.rB  c              
   C   s  | j dkr| jd}| jd}| jd}| jd}d}tt| t|d}d}td	d
dd}tj	t
dd|iidd}	|	rVt|	d d dkrVdd |	d d D }
ng }
g }|
r|
D ]F}|d drpt|d dnd}|d d}t|tszt|trt|ni }W n tjtfy   i }Y nw |t||d q^|}|ddi}|d	 }|j||d}t|}td| t|d}ddddd|iigiii}tj	t|d }d}d}|d d D ]}|d! d"d}|d! d#d}qt||d$S td%d&id'd(S ))Nr   rp   r  r  r  aI  Analyze the information provided from both ISO forms and generate a letter emphasizing the key similarities, differences, and insights. Ensure the letter clearly reflects the comparison, maintaining adherence to the formats specified in the ISO forms. Include the filename(s) of the ISO forms within your response where relevant.r  a  Analyze the information provided from both ISO forms and generate a letter emphasizing the key similarities, differences, and insights. Ensure the letter clearly reflects the comparison, maintaining adherence to the formats specified in the ISO forms. Include the filename(s) of the ISO forms within your response where relevant.  

        Context: {context}  

        {chat_history}  
        Assistant:rV  Tr*  r%  r/  r&  r'  r   r   c                 S   r(  r)  r*  r,  rV   rV   rW   r   	  r-  z'compforms_lettergen.<locals>.<listcomp>r2  r;   r1  r3  r  )contextrV  r#  r   r   r   r   r   r   r   r  r  r  r   Invalid request methodr?  r  )r   r   rk   r   r@  r   r  r   r|   r   r/  r  r#  r"  r5  r]   r+  r0  r1  r   r2  Zload_memory_variablesrE  r$  rM   r  r   )rc   rp   r  r  r  ra   r  r.  r6  r  r7  r8  r  r2  r1  Zcombined_docsZmemory_variablesrV  r"  r#  r  r   r~   r  r  r   rV   rV   rW   compforms_lettergen	  st   


	

$



;rE  c              
   C   s   | j dkrN| jd}|stddiddS ztjjjdd|d	}|j}t	
|d
}td|iW S  tyM } ztdt|iddW  Y d }~S d }~ww tddiddS )Nr   rO   r   zNo input text providedr  r  ztts-1Zalloy)r   Zvoicer  r   audior  rD  r?  )r   r   rk   r   openairF  Zspeechr  r   r   r   r   r   r#  )rc   
input_textr~   Zaudio_contentZbase64_encoded_audior   rV   rV   rW   text_to_speechp
  s$   
 rI  c              
   C   s   | j dkrOd| jv rG| jd }z| }t }|jjjd|j|fdd}td|iW S  t	yF } ztdt
|idd	W  Y d }~S d }~ww tdd
idd	S tddidd	S )Nr   r:  z	whisper-1r[   )r   r:  Zresponse_formattranscriptionr   r  r  zNo file providedr  rD  r?  )r   r  r   r(   rF  Ztranscriptionsr  r   r   r   r#  )rc   Z
audio_fileZaudio_bytesr   rJ  r   rV   rV   rW   speech_to_text
  s$   


 rK  c              
   C   s  z>g }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|D ]}tj
|d d tjd}||  krJ|krQn q2|| q2|r5|dkr4ttd	d
 |D }	i }
g }|	d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}|d d D ]}}|d dkrd|d d d i}ne|d dkrdd|d d d d  i}nP|d dkrd|d d d v r|d d d dd  }d |i}n+t|d d!krd|d d v rdd|d d d d  i}n
d"|d d d i}|| qqqd#d$ |D }tt|}|||dd%}
||
 qqn|d&krttd'd
 |D }	i }
g }|	d d d D ]}tjtdd|iidd}|rwt|d d dkrwd(d$ |d d D }g }i }|D ]}d |i}|| q}tjtdd|iidd}|rt|d d dkrtd)d
 |d d D }|D ]!}t|tkrd"|ji}nt|tkrd|ji}|| qtt|}|||d&d%}
||
 qPnF|d*krttd+d
 |D }	i }
g }|	d d d D ]}td,| tjtdd|iidd}|r2t|d d dkr2d-d$ |d d D }g }i }|D ]}d |i}|| q8tjtdd|iidd}|rt|d d dkrtd.d
 |d d D }|dd  D ].}t|tkrd"|ji}nt|tkrd|ji}nt|tkrd|ji}|| qrtt|}|||d*d%}
||
 qn{ttd/d
 |D }	i }
g }|	d d d D ]c}tjtdd|iidd}|r.t|d d dkr.td0d
 |d d D }g }i }|D ]!}t|tkr
d"|ji}nt|tkrd|ji}|| qtt|}|||d1d%}
||
 qd|i}ndg i}td|iW S  t y\ } ztd2t!|id3d4W  Y d }~S d }~ww )5N
X-UsernameX-Candidate-Name
image_logsr   hourminutesecondmicrosecondr   r  tzc                 s       | ]}|d  V  qdS rp   NrV   r   r  rV   rV   rW   r  
  r  z!today_messages.<locals>.<genexpr>rf   r/  rp   r&  r'  r   r   r   r   r   r^  rd  	assistantr[   r  data:image/jpeg;base64r|  r  r\   rb  	Filename:r:  r   r   c                 S      g | ]}|r|qS rV   rV   r  rV   rV   rW   r   
  r   z"today_messages.<locals>.<listcomp>rp   rV  r   Z
app_switchdoc_logsc                 s   rV  rW  rV   rX  rV   rV   rW   r  
  r  c                 S      g | ]}|d  d qS r   rQ   rV   r,  rV   rV   rW   r   
  r  c                 s   "    | ]}t |d  d V  qdS r   r  Nr*  r,  rV   rV   rW   r        	forms_logc                 s   rV  rW  rV   rX  rV   rV   rW   r    r  id_userc                 S   r_  r`  rV   r,  rV   rV   rW   r   '  r  c                 s   ra  rb  r*  r,  rV   rV   rW   r  6  rc  c                 s   rV  rW  rV   rX  rV   rV   rW   r  T  r  c                 s   ra  rb  r*  r,  rV   rV   rW   r  ^  rc  alli_logr   r  r  )"r^   rk   r   r  r   r
   rG   r	   utcrx  fromtimestampr   re  r&   fromkeysr|   r   rj   r   r  r  r   	doc_indexr   r^  r!   r   r"   rM   r/  r,   r   r   r#  )rc   tr   r   testcurrent_datetimestart_of_todayr   lastmodified_at_datetimesession_ids	yest_dictr  re  r  tot_histimtexrz   r=  r  rQ   chat_id_namesresult1session_hist1chat_hsession_histformatted_yest_dictr   rV   rV   rW   today_messages
  sp  



$

,





0





5



 rz  c              
   C   s  zLg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|t
dd }|t
dd }|D ]}	tj|	d	 d
 tjd}
||
  krX|kr_n q@||	 q@|rC|dkrBttdd |D }i }g }|d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}|d d D ]}}|d dkrd|d d d i}ne|d dkrdd|d d d d  i}nP|d d krd!|d d d v r|d d d d!d  }d"|i}n+t|d d#krd|d d v rdd|d d d d  i}n
d$|d d d i}|| qqqd%d& |D }tt|}|||dd'}|| qn|d(krttd)d |D }i }g }|d d d D ]}tjtdd|iidd}|rt|d d dkrd*d& |d d D }g }i }|D ]}d"|i}|| qtjtdd|iidd}|rt|d d dkrtd+d |d d D }|D ]!}t|tkrd$|ji}nt|tkrd|ji}|| qtt|}|||d(d'}|| q^nF|d,krttd-d |D }i }g }|d d d D ]}td.| tjtdd|iidd}|r@t|d d dkr@d/d& |d d D }g }i }|D ]}d"|i}|| qFtjtdd|iidd}|rt|d d dkrtd0d |d d D }|dd  D ].}t|tkrd$|ji}nt|tkrd|ji}nt|tkrd|ji}|| qtt|}|||d,d'}|| qn{ttd1d |D }i }g }|d d d D ]c}tjtdd|iidd}|r<t|d d dkr<td2d |d d D }g }i }|D ]!}t|tkrd$|ji}nt|tkr$d|ji}|| q	tt|}|||d3d'}|| qd|i}ndg i}t d|iW S  t!yj } zt d4t"|id5d6W  Y d }~S d }~ww )7NrL  rM  rN  r   rO  r|  days)microsecondsr   r  rT  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  z%yesterday_messages.<locals>.<genexpr>rf   r/  rp   r&  r'  r   r   r   r   r   r^  rd  rY  r[   r  rZ  r  r\   rb  r[  r:  r   r   c                 S   r\  rV   rV   r  rV   rV   rW   r     r   z&yesterday_messages.<locals>.<listcomp>r]  r^  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  c                 S   r_  r`  rV   r,  rV   rV   rW   r     r  c                 s   ra  rb  r*  r,  rV   rV   rW   r    rc  rd  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  re  c                 S   r_  r`  rV   r,  rV   rV   rW   r     r  c                 s   ra  rb  r*  r,  rV   rV   rW   r    rc  c                 s   rV  rW  rV   rX  rV   rV   rW   r  3  r  c                 s   ra  rb  r*  r,  rV   rV   rW   r  >  rc  rf  r   r  r  #r^   rk   r   r  r   r
   rG   r	   rg  rx  r   rh  r   re  r&   ri  r|   r   rj   r   r  r  r   rj  r   r^  r!   r   r"   rM   r/  r,   r   r   r#  )rc   yr   r   rl  rm  rn  start_of_yesterdayZend_of_yesterdayr   ro  rp  rq  r  re  r  rr  rs  rz   r=  r  rQ   rt  ru  rv  rw  rx  ry  r   rV   rV   rW   yesterday_messages  st  



$

(





/





5



 r  c              
   C   s  zLg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|t
dd }|t
dd }|D ]}	tj|	d	 d
 tjd}
||
  krX|k r_n q@||	 q@|rC|dkrBttdd |D }i }g }|d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}|d d D ]}}|d dkrd|d d d i}ne|d dkrdd|d d d d  i}nP|d d krd!|d d d v r|d d d d!d  }d"|i}n+t|d d#krd|d d v rdd|d d d d  i}n
d$|d d d i}|| qqqd%d& |D }tt|}|||dd'}|| qn|d(krttd)d |D }i }g }|d d d D ]}tjtdd|iidd}|rt|d d dkrd*d& |d d D }g }i }|D ]}d"|i}|| qtjtdd|iidd}|rt|d d dkrtd+d |d d D }|D ]!}t|tkrd$|ji}nt|tkrd|ji}|| qtt|}|||d(d'}|| q^nF|d,krttd-d |D }i }g }|d d d D ]}td.| tjtdd|iidd}|r@t|d d dkr@d/d& |d d D }g }i }|D ]}d"|i}|| qFtjtdd|iidd}|rt|d d dkrtd0d |d d D }|dd  D ].}t|tkrd$|ji}nt|tkrd|ji}nt|tkrd|ji}|| qtt|}|||d,d'}|| qn{ttd1d |D }i }g }|d d d D ]c}tjtdd|iidd}|r<t|d d dkr<td2d |d d D }g }i }|D ]!}t|tkrd$|ji}nt|tkr$d|ji}|| q	tt|}|||d3d'}|| qd|i}ndg i}t d|iW S  t!yj } zt d4t"|id5d6W  Y d }~S d }~ww )7NrL  rM  rN  r   rO  r|  r{     r   r  rT  c                 s   rV  rW  rV   rX  rV   rV   rW   r  |  r  z"previous_7_days.<locals>.<genexpr>rf   r/  rp   r&  r'  r   r   r   r   r   r^  rd  rY  r[   r  rZ  r  r\   rb  r[  r:  r   r   c                 S   r\  rV   rV   r  rV   rV   rW   r     r   z#previous_7_days.<locals>.<listcomp>r]  r^  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  c                 S   r_  r`  rV   r,  rV   rV   rW   r     r  c                 s   ra  rb  r*  r,  rV   rV   rW   r    rc  rd  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  re  c                 S   r_  r`  rV   r,  rV   rV   rW   r     r  c                 s   ra  rb  r*  r,  rV   rV   rW   r     rc  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  c                 s   ra  rb  r*  r,  rV   rV   rW   r  *  rc  rf  r   r  r  r~  )rc   sr   r   rl  rm  rn  r  start_of_seven_days_agor   ro  rp  rq  r  re  r  rr  rs  rz   r=  r  rQ   rt  ru  rv  rw  rx  ry  r   rV   rV   rW   previous_7_dayse  st  



$

)





9





6



 r  c              
   C   s  zLg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|t
dd }|t
dd }|D ]}	tj|	d	 d
 tjd}
||
  krX|k r_n q@||	 q@|rC|dkrBttdd |D }i }g }|d d d D ]}tjtdd|iidd}g }i }|d d D ]}|d d D ]}|d d D ]}}|d dkrd|d d d i}ne|d dkrdd|d d d d   i}nP|d d!krd"|d d d v r|d d d d"d  }d#|i}n+t|d d$krd|d d v rdd|d d d d   i}n
d%|d d d i}|| qqqd&d' |D }tt|}|||dd(}|| qn|d)krttd*d |D }i }g }|d d d D ]}tjtdd|iidd}|rt|d d dkrd+d' |d d D }g }i }|D ]}d#|i}|| qtjtdd|iidd}|rt|d d dkrtd,d |d d D }|D ]!}t|tkrd%|ji}nt|tkrd|ji}|| qtt|}|||d)d(}|| q^nF|d-krttd.d |D }i }g }|d d d D ]}td/| tjtdd|iidd}|r@t|d d dkr@d0d' |d d D }g }i }|D ]}d#|i}|| qFtjtdd|iidd}|rt|d d dkrtd1d |d d D }|dd  D ].}t|tkrd%|ji}nt|tkrd|ji}nt|tkrd|ji}|| qtt|}|||d-d(}|| qn{ttd2d |D }i }g }|d d d D ]c}tjtdd|iidd}|r<t|d d dkr<td3d |d d D }g }i }|D ]!}t|tkrd%|ji}nt|tkr$d|ji}|| q	tt|}|||d4d(}|| qd|i}ndg i}t d|iW S  t!yj } zt d5t"|id6d7W  Y d }~S d }~ww )8NrL  rM  rN  r   rO  r  r{     r   r  rT  c                 s   rV  rW  rV   rX  rV   rV   rW   r  m  r  z#previous_30_days.<locals>.<genexpr>rf   r/  rp   r&  r'  r   r   r   r   r   r^  rd  rY  r[   r  rZ  r|  r  r\   rb  r[  r:  r   r   c                 S   r\  rV   rV   r  rV   rV   rW   r     r   z$previous_30_days.<locals>.<listcomp>r]  r^  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  c                 S   r_  r`  rV   r,  rV   rV   rW   r     r  c                 s   ra  rb  r*  r,  rV   rV   rW   r    rc  rd  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  re  c                 S   r_  r`  rV   r,  rV   rV   rW   r     r  c                 s   ra  rb  r*  r,  rV   rV   rW   r    rc  c                 s   rV  rW  rV   rX  rV   rV   rW   r    r  c                 s   ra  rb  r*  r,  rV   rV   rW   r    rc  rf  r   r  r  r~  )rc   r  r   r   rl  rm  rn  r  start_of_thirty_days_agor   ro  rp  Zpre_30_days_dictr  re  r  rr  rs  rz   r=  r  rQ   rt  ru  rv  rw  rx  Zformatted_previous30days_dictr   rV   rV   rW   previous_30_daysR  sr  



$

)





9





6




 r  c              
   C   sd  zg }| j d}| j d}|dkrt|t|}nt|t|}ttj}|j	ddddd}|t
dd }t|dd	 d
D ]}tj|d d tjd}	|	|k rV|| q?t|dd	 dd}t| jdd}
t| jdd}||
t|
| t| }|d d d }t|t|
| t|k}|rttdd |D }g }|d d d D ]P}tjtdd|iidd}|rt|d d dkrtdd |d d D }g }|D ]}t|trd|jind|ji}|| qtt|}||||d q||d }ng d!d }td"|iW S  ty1 } ztd#|  td$t |id%d&W  Y d }~S d }~ww )'NrL  rM  rN  r   rO  r  r{  c                 S      | d S Nr   rV   )xrV   rV   rW   <lambda>R      z$older_than_30_days.<locals>.<lambda>)keyr   r  rT  c                 S   r  r  rV   )r   rV   rV   rW   r  Z  r  T)r  r3   offsetlimit
   rf   c                 s   rV  rW  rV   rX  rV   rV   rW   r  h  r  z%older_than_30_days.<locals>.<genexpr>r/  rp   r&  r'  r   c                 s   ra  rb  r*  r,  rV   rV   rW   r  q  rc  r   rY  )rp   rV  r   )r   has_moreFr   z"Error in older_than_30_days view: r   r  r  )!r^   rk   r   r  r   r
   rG   r	   rg  rx  r   r  rh  r   r<  GETminr  re  r&   ri  r|   r   r   r"  r!   r   r  r   r   r   rM   r#  )rc   r  r   r   rl  rm  rn  r  r   ro  r  r  Zpaginated_older_messagesr  rp  r  re  r  rx  rr  r  rw  rt  Zformatted_older_dictr   rV   rV   rW   older_than_30_days=  sj   




 r  )ro   )rA  )Zdjango.shortcutsr   Zdjango.contrib.auth.decoratorsr   django.contrib.auth.modelsr   django.httpr   r   django.views.decorators.csrfr   r]   r
   r   r	   r   r   Zlangchain.chat_modelsr   Zlangchain_google_genair   googler   Zgoogle.genai.typesr   r@   sysiotypingr   Zlangchain.load.dumpr   asyncioZaiohttpr  Zlangchain_core.messagesr   r   r   r   mathr   Zlangchain.promptsr   r   Z"langchain.chains.combine_documentsr   Zlangchain_openair   rG  r   Z langchain_community.vectorstoresr   Zlangchain.chainsr   Zlangchain.memoryr   r    Zlangchain_core.messages.humanr!   Zlangchain_core.messages.air"   r_   Zrequests.authr#   r$   r%   collectionsr&   Zdecoupler'   r(   PILr)   r*   django.core.mailr+   Zlangchain_core.messages.systemr,   Zlangchain_core.messages.chatr-   rA   r   r   r  r  Zlangchain_core.documentsr.   Zpandasr   ZLoader_functions_utilsr/   r}  Zlangchain.docstore.documentr2  r  django.contrib.authr0   r1   r2   django.urlsr3   concurrent.futuresr4   r5   
concurrentr   r=   rM   rG   rH   abspathBASE_DIRZLOG_DIRr  rK   rX   r   r   rd   rn   r#  r   r5  r   r   r   r   r   r   r   r   r   r   r   r   r   environgetenvr  r   r   r  r/  r  r  rj  r|   r   r  r  r   exr  r'  r(  r-  r.  r0  r6  r;  r>  r@  rL  rM  rQ  rS  rX  r[  rZ  rg  rk  rl  rq  rs  r{  r  r  r  r  r  r  r  r  r  r  r<  r  r  r  r  r  r  r  r$  r>  r@  rB  rE  rI  rK  rz  r  r  r  r  rV   rV   rV   rW   <module>   s    
F
4

;
-
	
5



	
#":	d.

   0
    F

5
 

 e a n
 k