chenx221/Textractor
1
0
Fork 0
forked from Public-Mirror/Textractor
Code Pull Requests Activity

upgrade character deduplication algorithm

Browse Source
This commit is contained in:
Akash Mozumdar 2021-06-30 20:58:40 -06:00
parent 50f5b183f1
commit b28b68d218
2 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
extensions/removerepeatchar.cpp
View File

@ -5,30 +5,25 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
if (sentenceInfo["text number"] == 0) return false; if (sentenceInfo["text number"] == 0) return false;
std::vector<int> repeatNumbers(sentence.size() + 1, 0); std::vector<int> repeatNumbers(sentence.size() + 1, 0);
int repeatNumber = 1; for (int i = 0; i < sentence.size(); ++i)
wchar_t prevChar = 0;
for (auto nextChar : sentence)
{ {
if (nextChar == prevChar) if (sentence[i] != sentence[i + 1])
{ {
repeatNumber += 1; int j = i;
} while (sentence[j] == sentence[i] && --j >= 0);
else repeatNumbers[i - j] += 1;
{
prevChar = nextChar;
repeatNumbers.at(repeatNumber) += 1;
repeatNumber = 1;
} }
} }
if ((repeatNumber = std::distance(repeatNumbers.begin(), std::max_element(repeatNumbers.begin(), repeatNumbers.end()))) == 1) return false; int repeatNumber = std::distance(repeatNumbers.begin(), std::max_element(repeatNumbers.rbegin(), repeatNumbers.rend()).base() - 1);
if (repeatNumber < 2) return false;
std::wstring newSentence; std::wstring newSentence;
for (int i = 0; i < sentence.size();) for (int i = 0; i < sentence.size();)
{ {
newSentence.push_back(sentence.at(i)); newSentence.push_back(sentence[i]);
for (int j = i; j <= sentence.size(); ++j) for (int j = i; j <= sentence.size(); ++j)
{ {
if (j == sentence.size() || sentence.at(i) != sentence.at(j)) if (j == sentence.size() || sentence[i] != sentence[j])
{ {
i += (j - i) % repeatNumber == 0 ? repeatNumber : 1; i += (j - i) % repeatNumber == 0 ? repeatNumber : 1;
break; break;
@ -44,8 +39,11 @@ TEST(
InfoForExtension nonConsole[] = { { "text number", 1 }, {} }; InfoForExtension nonConsole[] = { { "text number", 1 }, {} };
std::wstring repeatedChars = L"aaaaaaaaaaaabbbbbbcccdddaabbbcccddd"; std::wstring repeatedChars = L"aaaaaaaaaaaabbbbbbcccdddaabbbcccddd";
std::wstring someRepeatedChars = L"abcdefaabbccddeeff";
ProcessSentence(repeatedChars, { nonConsole }); ProcessSentence(repeatedChars, { nonConsole });
ProcessSentence(someRepeatedChars, { nonConsole });
assert(repeatedChars.find(L"aaaabbcd") == 0); assert(repeatedChars.find(L"aaaabbcd") == 0);
assert(someRepeatedChars == L"abcdefabcdef");
std::wstring empty = L"", one = L" ", normal = L"This is a normal sentence. はい"; std::wstring empty = L"", one = L" ", normal = L"This is a normal sentence. はい";
ProcessSentence(empty, { nonConsole }); ProcessSentence(empty, { nonConsole });

2
extensions/removerepeatsentence.cpp
View File

@ -30,7 +30,7 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
static std::mutex m; static std::mutex m;
m.lock(); m.lock();
if (textNumber + 1 > cache.size()) cache.resize(textNumber + 1); if (textNumber + 1 > cache.size()) cache.resize(textNumber + 1);
auto prevSentences = cache.at(textNumber).Acquire(); auto prevSentences = cache[textNumber].Acquire();
m.unlock(); m.unlock();
auto& inserted = prevSentences->emplace_back(sentence); auto& inserted = prevSentences->emplace_back(sentence);
auto firstLocation = std::find(prevSentences->begin(), prevSentences->end(), sentence); auto firstLocation = std::find(prevSentences->begin(), prevSentences->end(), sentence);