o ZhT@s<ddlZddlmZddlmZmZmZGdddeZ dS)N)Text)PDFHTML CleanRulesc@seZdZd ddZddZddZdd Zd d Zd d ZddZ ddZ ddZ ddZ ddZ ddZddZddZddZdS)!CleanerNcCs||_||_||_dSN)textlangdoc_type)selfrr r r D/var/www/html/lang_env/lib/python3.10/site-packages/pysbd/cleaner.py__init__ s zCleaner.__init__cCs|js|jS||||t|jjtj|_| t|jt j |_| | |||jSr)rremove_all_newlinesreplace_double_newlinesreplace_newlinesreplace_escaped_newlinesrapplyrZAllreplace_punctuation_in_bracketscrZInlineFormattingRuleclean_quotationsclean_table_of_contents'check_for_no_space_in_between_sentencesclean_consecutive_charactersr r r r cleansz Cleaner.cleancCs||dSr)$remove_newline_in_middle_of_sentence remove_newline_in_middle_of_wordrr r r rs zCleaner.remove_all_newlinescCdd}td||j|_dS)NcSs|}ttjd|}|S)N)groupresubrZ#NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEXmatchr"r r r replace_w_blank#szECleaner.remove_newline_in_middle_of_sentence..replace_w_blankz (?:[^\.])*r!r"r)r r%r r r r"sz,Cleaner.remove_newline_in_middle_of_sentencecCst|jtj|_dSr)rrrrZNewLineInMiddleOfWordRulerr r r r)sz(Cleaner.remove_newline_in_middle_of_wordcCt|jtjtj|_dSr)rrrrZDoubleNewLineWithSpaceRuleZDoubleNewLineRulerr r r r,s zCleaner.replace_double_newlinescCs t|jtjtjtj|_dSr)rrrrZNewLineFollowedByBulletRulerZNewLineInMiddleOfSentenceRuleZ%NewLineInMiddleOfSentenceNoSpacesRulerr r r remove_pdf_line_breaks0szCleaner.remove_pdf_line_breakscCs2|jdkr |dSt|jtjtj|_dS)NZpdf)r r(rrrrZNewLineFollowedByPeriodRuleZ$ReplaceNewlineWithCarriageReturnRulerr r r r6s  zCleaner.replace_newlinescCs$t|jtjtjtjtj|_dSr)rrrrZEscapedNewLineRuleZEscapedCarriageReturnRuleZTypoEscapedNewLineRuleZTypoEscapedCarriageReturnRulerr r r r>sz Cleaner.replace_escaped_newlinescCr)NcSs,|}d|vrttdd|}|S|S)N?u&ᓷ&)r r!r"escaper#r r r replace_punctFs z>Cleaner.replace_punctuation_in_brackets..replace_punctz\[(?:[^\]])*\]r&)r r+r r r rEsz'Cleaner.replace_punctuation_in_bracketscCs.tdd|j|_t|jtjtj|_dS)N`')r!r"rrrrZQuotationsFirstRuleZQuotationsSecondRulerr r r rNs   zCleaner.clean_quotationscCs t|jtjtjtj|_dSr)rrrrZTableOfContentsRuleConsecutivePeriodsRuleConsecutiveForwardSlashRulerr r r rWs  zCleaner.clean_table_of_contentscsRt|s|StfddtjDr|St|}tt||}|S)Nc3s|]}|vVqdSrr ).0kwordr r `sz9Cleaner.search_for_connected_sentences..) r!searchanyrZURL_EMAIL_KEYWORDSrrr"r*)r r3txtregexruleZnew_wordr r2r search_for_connected_sentences]s z&Cleaner.search_for_connected_sentencescCsJ|jd}|D]}|||jtjtj|_|||jtjtj|_qdS)N )rsplitr:rZ NO_SPACE_BETWEEN_SENTENCES_REGEXZNoSpaceBetweenSentencesRuleZ&NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEXZ NoSpaceBetweenSentencesDigitRule)r wordsr3r r r rfs z/Cleaner.check_for_no_space_in_between_sentencescCr'r)rrrrr.r/rr r r rls  z$Cleaner.clean_consecutive_charactersr)__name__ __module__ __qualname__rrrrrrr(rrrrrr:rrr r r r rs     r) r!Z pysbd.utilsrZpysbd.clean.rulesrrrrobjectrr r r r s