Public-Mirror/Textractor
1
0
Fork 1
mirror of https://github.com/Artikash/Textractor.git synced 2024-12-24 01:14:12 +08:00
Code Issues Projects Releases Wiki Activity

clean up repetition remover. dont need to remove repeated sentences since that detection was added in textthread

Browse Source
This commit is contained in:
Akash Mozumdar 2018-11-07 10:43:54 -05:00
parent 219804cecc
commit ddc8313d37
1 changed files with 10 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
extensions/removerepeat/removerepeat.cpp
View File

@ -1,61 +1,34 @@
#include "../extension.h" #include "../extension.h"
bool RemoveRepeatedChars(std::wstring& sentence) void RemoveRepeatedChars(std::wstring& sentence)
{ {
int repeatNumber = 0; int repeatNumber = 0;
wchar_t prevChar = sentence[0]; wchar_t prevChar = sentence[0];
for (auto c : sentence) for (auto c : sentence)
if (c == prevChar) repeatNumber++; if (c == prevChar) repeatNumber++;
else break; else break;
if (repeatNumber == 1) return false; if (repeatNumber == 1) return;
for (int i = 0; i < sentence.size(); i += repeatNumber) for (int i = 0; i < sentence.size(); i += repeatNumber)
for (int j = i; j < sentence.size(); ++j) for (int j = i; j < sentence.size(); ++j)
if (sentence[j] != sentence[i]) if (sentence[j] != sentence[i])
if ((j - i) % repeatNumber != 0) return false; if ((j - i) % repeatNumber != 0) return;
else break; else break;
std::wstring newSentence = L""; std::wstring newSentence = L"";
for (int i = 0; i < sentence.size(); ++i) if (i % repeatNumber == 0) newSentence.push_back(sentence[i]); for (int i = 0; i < sentence.size(); i += repeatNumber) newSentence.push_back(sentence[i]);
sentence = newSentence; sentence = newSentence;
return true;
} }
bool RemoveCyclicRepeats(std::wstring& sentence) void RemoveCyclicRepeats(std::wstring& sentence)
{ {
if (sentence == L"") Skip(); for (std::wsmatch results; std::regex_search(sentence, results, std::wregex(L"^([^\\x00]{5,})[^\\x00]*?\\1")); sentence.erase(0, results[1].length()));
int junkLength = 0;
wchar_t junk[2000] = {};
while (wcsstr(sentence.c_str() + junkLength, junk))
{
junk[junkLength] = sentence[junkLength];
if (++junkLength >= 2000) return false;
}
if (--junkLength >= 5) // If the first 5 characters appear later on, there's probably a repetition issue.
{
sentence = std::wstring(sentence.c_str() + junkLength);
RemoveCyclicRepeats(sentence);
return true;
}
return false;
}
bool RemoveRepeatedSentences(std::wstring& sentence, int64_t handle)
{
static std::unordered_map<int64_t, std::unordered_set<std::wstring>> seenSentences;
static std::mutex m;
std::lock_guard<std::mutex> l(m);
if (seenSentences[handle].count(sentence)) Skip();
seenSentences[handle].insert(sentence);
return false;
} }
bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
{ {
if (sentenceInfo["hook address"] == -1) return false; if (sentenceInfo["hook address"] == -1) return false;
bool ret = false; RemoveRepeatedChars(sentence);
ret |= RemoveRepeatedChars(sentence); RemoveCyclicRepeats(sentence);
ret |= RemoveCyclicRepeats(sentence); return true;
ret |= RemoveRepeatedSentences(sentence, sentenceInfo["text handle"]);
return ret;
} }