Textractor_test/extensions/removerepeat.cpp
Akash Mozumdar 8e45b35ebe refactor
2018-10-30 21:21:21 -04:00

63 lines
1.8 KiB
C++

#include "extension.h"
#include <set>
#include <mutex>
bool RemoveRepeatedChars(std::wstring& sentence)
{
int repeatNumber = 0;
wchar_t prevChar = sentence[0];
for (auto c : sentence)
if (c == prevChar) repeatNumber++;
else break;
if (repeatNumber == 1) return false;
for (int i = 0; i < sentence.size(); i += repeatNumber)
for (int j = i; j < sentence.size(); ++j)
if (sentence[j] != sentence[i])
if ((j - i) % repeatNumber != 0) return false;
else break;
std::wstring newSentence = L"";
for (int i = 0; i < sentence.size(); ++i) if (i % repeatNumber == 0) newSentence.push_back(sentence[i]);
sentence = newSentence;
return true;
}
bool RemoveCyclicRepeats(std::wstring& sentence)
{
if (sentence == L"") Skip();
int junkLength = 0;
wchar_t junk[2000] = {};
while (wcsstr(sentence.c_str() + junkLength, junk))
{
junk[junkLength] = sentence[junkLength];
if (++junkLength >= 2000) return false;
}
if (--junkLength >= 5) // If the first 5 characters appear later on, there's probably a repetition issue.
{
sentence = std::wstring(sentence.c_str() + junkLength);
RemoveCyclicRepeats(sentence);
return true;
}
return false;
}
bool RemoveRepeatedSentences(std::wstring& sentence, int64_t handle)
{
static std::set<std::pair<int64_t, std::wstring>> seenSentences;
static std::mutex m;
std::lock_guard<std::mutex> l(m);
if (seenSentences.count({ handle, sentence }) != 0) Skip();
seenSentences.insert({ handle, sentence });
return false;
}
bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
{
if (sentenceInfo["hook address"] == -1) return false;
bool ret = false;
ret |= RemoveRepeatedChars(sentence);
ret |= RemoveCyclicRepeats(sentence);
ret |= RemoveRepeatedSentences(sentence, sentenceInfo["text handle"]);
return ret;
}