chenx221/Textractor_test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add repeated sentences filter (laxer than before)

Browse Source
This commit is contained in:
Akash Mozumdar 2019-06-04 15:30:48 -04:00
parent e529046d7d
commit d16db4d319
2 changed files with 19 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
GUI/host/textthread.h
View File

@ -2,7 +2,6 @@
#include "common.h" #include "common.h"
#include "types.h" #include "types.h"
#include <deque>
class TextThread class TextThread
{ {

19
extensions/removerepeat.cpp
View File

@ -1,5 +1,23 @@
#include "extension.h" #include "extension.h"
void RemoveRepeatedSentences(std::wstring& sentence, uint64_t textNumber)
{
static std::deque<ThreadSafe<std::vector<std::wstring>>> cache;
static std::mutex m;
m.lock();
if (textNumber + 1 > cache.size()) cache.resize(textNumber + 1);
auto[lock, prevSentences] = cache.at(textNumber).operator->();
m.unlock();
auto& inserted = prevSentences->emplace_back(sentence);
auto firstLocation = std::find(prevSentences->begin(), prevSentences->end(), sentence);
if (&*firstLocation != &inserted)
{
prevSentences->erase(firstLocation);
sentence.clear();
}
if (prevSentences->size() > 50) prevSentences->erase(prevSentences->begin());
}
void RemoveRepeatedChars(std::wstring& sentence) void RemoveRepeatedChars(std::wstring& sentence)
{ {
std::vector<int> repeatNumbers(sentence.size() + 1, 0); std::vector<int> repeatNumbers(sentence.size() + 1, 0);
@ -63,6 +81,7 @@ void RemoveCyclicRepeats(std::wstring& sentence)
bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
{ {
if (sentenceInfo["text number"] == 0) return false; if (sentenceInfo["text number"] == 0) return false;
RemoveRepeatedSentences(sentence, sentenceInfo["text number"]);
RemoveRepeatedChars(sentence); RemoveRepeatedChars(sentence);
RemoveCyclicRepeats(sentence); RemoveCyclicRepeats(sentence);
return true; return true;