chenx221/Textractor_test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improve repetition remover perf

Browse Source
This commit is contained in:
Akash Mozumdar 2018-11-15 00:06:49 -05:00
parent 1d17902c64
commit d9a3d5cefd
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
extensions/removerepeat/removerepeat.cpp
View File

@ -22,7 +22,17 @@ void RemoveRepeatedChars(std::wstring& sentence)
void RemoveCyclicRepeats(std::wstring& sentence) void RemoveCyclicRepeats(std::wstring& sentence)
{ {
for (std::wsmatch results; std::regex_search(sentence, results, std::wregex(L"^([^\\x00]{5,})[^\\x00]*?\\1")); sentence.erase(0, results[1].length())); remove:
std::wstring junk = sentence;
while (junk.size() > 4)
{
junk.pop_back();
if (sentence.rfind(junk) > 0)
{
sentence.erase(0, junk.size());
goto remove;
}
}
} }
bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)