From d16db4d3193cc0d4246510754b387ad6d512197b Mon Sep 17 00:00:00 2001
From: Akash Mozumdar <akashmozumdar@gmail.com>
Date: Tue, 4 Jun 2019 15:30:48 -0400
Subject: [PATCH] add repeated sentences filter (laxer than before)

---
 GUI/host/textthread.h       |  1 -
 extensions/removerepeat.cpp | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/GUI/host/textthread.h b/GUI/host/textthread.h
index 945eae2..95bb5e0 100644
--- a/GUI/host/textthread.h
+++ b/GUI/host/textthread.h
@@ -2,7 +2,6 @@
 
 #include "common.h"
 #include "types.h"
-#include <deque>
 
 class TextThread
 {
diff --git a/extensions/removerepeat.cpp b/extensions/removerepeat.cpp
index 5e15471..ea30680 100644
--- a/extensions/removerepeat.cpp
+++ b/extensions/removerepeat.cpp
@@ -1,5 +1,23 @@
 #include "extension.h"
 
+void RemoveRepeatedSentences(std::wstring& sentence, uint64_t textNumber)
+{
+	static std::deque<ThreadSafe<std::vector<std::wstring>>> cache;
+	static std::mutex m;
+	m.lock();
+	if (textNumber + 1 > cache.size()) cache.resize(textNumber + 1);
+	auto[lock, prevSentences] = cache.at(textNumber).operator->();
+	m.unlock();
+	auto& inserted = prevSentences->emplace_back(sentence);
+	auto firstLocation = std::find(prevSentences->begin(), prevSentences->end(), sentence);
+	if (&*firstLocation != &inserted)
+	{
+		prevSentences->erase(firstLocation);
+		sentence.clear();
+	}
+	if (prevSentences->size() > 50) prevSentences->erase(prevSentences->begin());
+}
+
 void RemoveRepeatedChars(std::wstring& sentence)
 {
 	std::vector<int> repeatNumbers(sentence.size() + 1, 0);
@@ -63,6 +81,7 @@ void RemoveCyclicRepeats(std::wstring& sentence)
 bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
 {
 	if (sentenceInfo["text number"] == 0) return false;
+	RemoveRepeatedSentences(sentence, sentenceInfo["text number"]);
 	RemoveRepeatedChars(sentence);
 	RemoveCyclicRepeats(sentence);
 	return true;