o ifT@s<ddlZddlmZddlmZmZmZGdddeZ dS)N)Text)PDFHTML CleanRulesc@seZdZd ddZddZddZdd Zd d Zd d ZddZ ddZ ddZ ddZ ddZ ddZddZddZddZdS)!CleanerNcCs||_||_||_dSN)textlangdoc_type)selfrr r r F/var/www/html/corbot_env/lib/python3.10/site-packages/pysbd/cleaner.py__init__ s zCleaner.__init__cCs|js|jS||||t|jjtj|_| t|jt j |_| | |||jSr)rremove_all_newlinesreplace_double_newlinesreplace_newlinesreplace_escaped_newlinesrapplyrAllreplace_punctuation_in_bracketscrInlineFormattingRuleclean_quotationsclean_table_of_contents'check_for_no_space_in_between_sentencesclean_consecutive_charactersr r r r cleansz Cleaner.cleancCs||dSr)$remove_newline_in_middle_of_sentence remove_newline_in_middle_of_wordrr r r rs zCleaner.remove_all_newlinescCdd}td||j|_dS)NcSs|}ttjd|}|S)N)groupresubr#NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEXmatchr$r r r replace_w_blank#szECleaner.remove_newline_in_middle_of_sentence..replace_w_blankz (?:[^\.])*r#r$r)r r(r r r r"sz,Cleaner.remove_newline_in_middle_of_sentencecCst|jtj|_dSr)rrrrNewLineInMiddleOfWordRulerr r r r)sz(Cleaner.remove_newline_in_middle_of_wordcCt|jtjtj|_dSr)rrrrDoubleNewLineWithSpaceRuleDoubleNewLineRulerr r r r,s zCleaner.replace_double_newlinescCs t|jtjtjtj|_dSr)rrrrNewLineFollowedByBulletRulerNewLineInMiddleOfSentenceRule%NewLineInMiddleOfSentenceNoSpacesRulerr r r remove_pdf_line_breaks0szCleaner.remove_pdf_line_breakscCs2|jdkr |dSt|jtjtj|_dS)Npdf)r r1rrrrNewLineFollowedByPeriodRule$ReplaceNewlineWithCarriageReturnRulerr r r r6s  zCleaner.replace_newlinescCs$t|jtjtjtjtj|_dSr)rrrrEscapedNewLineRuleEscapedCarriageReturnRuleTypoEscapedNewLineRuleTypoEscapedCarriageReturnRulerr r r r>sz Cleaner.replace_escaped_newlinescCr )NcSs,|}d|vrttdd|}|S|S)N?u&ᓷ&)r"r#r$escaper&r r r replace_punctFs z>Cleaner.replace_punctuation_in_brackets..replace_punctz\[(?:[^\]])*\]r))r r;r r r rEsz'Cleaner.replace_punctuation_in_bracketscCs.tdd|j|_t|jtjtj|_dS)N`')r#r$rrrrQuotationsFirstRuleQuotationsSecondRulerr r r rNs   zCleaner.clean_quotationscCs t|jtjtjtj|_dSr)rrrrTableOfContentsRuleConsecutivePeriodsRuleConsecutiveForwardSlashRulerr r r rWs  zCleaner.clean_table_of_contentscsRt|s|StfddtjDr|St|}tt||}|S)Nc3s|]}|vVqdSrr ).0kwordr r `sz9Cleaner.search_for_connected_sentences..) r#searchanyrURL_EMAIL_KEYWORDSrrr$r:)r rFtxtregexrulenew_wordr rEr search_for_connected_sentences]s z&Cleaner.search_for_connected_sentencescCsJ|jd}|D]}|||jtjtj|_|||jtjtj|_qdS)N )rsplitrOr NO_SPACE_BETWEEN_SENTENCES_REGEXNoSpaceBetweenSentencesRule&NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX NoSpaceBetweenSentencesDigitRule)r wordsrFr r r rfs z/Cleaner.check_for_no_space_in_between_sentencescCr+r)rrrrrArBrr r r rls  z$Cleaner.clean_consecutive_charactersr)__name__ __module__ __qualname__rrrrrrr1rrrrrrOrrr r r r rs     r) r# pysbd.utilsrpysbd.clean.rulesrrrrobjectrr r r r s