performance improvements and autoselect native language for translations

2020-02-25 04:39:27 -07:00 · 2020-02-25 04:39:27 -07:00 · dc48f2a3c8
commit dc48f2a3c8
parent 2e23d4016d
6 changed files with 89 additions and 62 deletions
--- a/GUI/host/host.cpp
+++ b/GUI/host/host.cpp
@ -68,10 +68,8 @@ namespace
 	void RemoveThreads(std::function<bool(ThreadParam)> removeIf)
 	{
 		std::vector<TextThread*> threadsToRemove;
-		{
-			auto textThreadsByParams = ::textThreadsByParams.Acquire();
-			std::for_each(textThreadsByParams->begin(), textThreadsByParams->end(), [&](auto& it) { if (removeIf(it.first)) threadsToRemove.push_back(&it.second); });
-		}
+		for (auto& [tp, thread] : textThreadsByParams.Acquire().contents)
+			if (removeIf(tp)) threadsToRemove.push_back(&thread);
 		for (auto thread : threadsToRemove)
 		{
 			OnDestroy(*thread);
@ -131,14 +129,14 @@ namespace
 				{
 					auto tp = *(ThreadParam*)buffer;
 					auto textThreadsByParams = ::textThreadsByParams.Acquire();
-					auto textThread = textThreadsByParams->find(tp);
-					if (textThread == textThreadsByParams->end())
+					auto thread = textThreadsByParams->find(tp);
+					if (thread == textThreadsByParams->end())
 					{
-						try { textThread = textThreadsByParams->try_emplace(tp, tp, processRecordsByIds->at(tp.processId).GetHook(tp.addr).hp).first; }
+						try { thread = textThreadsByParams->try_emplace(tp, tp, processRecordsByIds->at(tp.processId).GetHook(tp.addr).hp).first; }
 						catch (std::out_of_range) { continue; } // probably garbage data in pipe, try again
-						OnCreate(textThread->second);
+						OnCreate(thread->second);
 					}
-					textThread->second.Push(buffer + sizeof(tp), bytesRead - sizeof(tp));
+					thread->second.Push(buffer + sizeof(tp), bytesRead - sizeof(tp));
 				}
 				break;
 				}
@ -247,9 +245,9 @@ namespace Host

 	TextThread* GetThread(int64_t handle)
 	{
-		auto textThreadsByParams = ::textThreadsByParams.Acquire();
-		auto thread = std::find_if(textThreadsByParams->begin(), textThreadsByParams->end(), [&](const auto& thread) { return thread.second.handle == handle; });
-		return thread != textThreadsByParams->end() ? &thread->second : nullptr;	
+		for (auto& [tp, thread] : textThreadsByParams.Acquire().contents)
+			if (thread.handle == handle) return &thread;
+		return nullptr;	
 	}

 	void AddConsoleOutput(std::wstring text)
--- a/extensions/bingtranslate.cpp
+++ b/extensions/bingtranslate.cpp
@ -9,7 +9,6 @@ extern Synchronized<std::wstring> translateTo;
 const char* TRANSLATION_PROVIDER = "Bing";
 QStringList languages
 {
-	"English: en",
 	"Arabic: ar",
 	"Bosnian: bs-Latn",
 	"Bulgarian: bg",
@ -20,6 +19,7 @@ QStringList languages
 	"Czech: cs",
 	"Danish: da",
 	"Dutch: nl",
+	"English: en",
 	"Estonian: et",
 	"Finnish: fi",
 	"French: fr",
--- a/extensions/blockmarkup.h
+++ b/extensions/blockmarkup.h
@ -3,53 +3,57 @@
 #include "common.h"
 #include <istream>

-template <typename C, int DelimiterCount, int bufferStartSize = 200>
+template <typename C, int delimiterCount, int blockSize = 0x1000 / sizeof(C)> // windows file block size
 class BlockMarkupIterator
 {
 public:
-	BlockMarkupIterator(std::istreambuf_iterator<char> it, const std::basic_string_view<C> (&delimiters)[DelimiterCount]) :
-		it(it)
+	BlockMarkupIterator(const std::istream& it, const std::basic_string_view<C>(&delimiters)[delimiterCount]) :
+		streambuf(*it.rdbuf())
 	{
-		std::copy_n(delimiters, DelimiterCount, this->delimiters.begin());
+		std::copy_n(delimiters, delimiterCount, this->delimiters.begin());
 	}
-	std::optional<std::array<std::basic_string<C>, DelimiterCount>> Next()
+
+	std::optional<std::array<std::basic_string<C>, delimiterCount>> Next()
 	{
-		std::array<std::basic_string<C>, DelimiterCount> results;
-		std::basic_string<C> buffer;
-		buffer.reserve(bufferStartSize);
-		Find(buffer, delimiters[0]);
-		buffer.clear();
-		for (int i = 0; i < DelimiterCount; ++i)
+		std::array<std::basic_string<C>, delimiterCount> results;
+		Find(delimiters[0], true);
+		for (int i = 0; i < delimiterCount; ++i)
 		{
-			const auto delimiter = i + 1 < DelimiterCount ? delimiters[i + 1] : end;
-			if (!Find(buffer, delimiter)) return {};
-			buffer.erase(buffer.size() - delimiter.size());
-			results[i] = std::move(buffer);
-			(buffer = {}).reserve(bufferStartSize);
+			const auto delimiter = i + 1 < delimiterCount ? delimiters[i + 1] : end;
+			if (auto found = Find(delimiter, false)) results[i] = std::move(found.value());
+			else return {};
 		}
 		return results;
 	}

 private:
-	bool Find(std::basic_string<C>& result, std::basic_string_view<C> delimiter)
+	std::optional<std::basic_string<C>> Find(std::basic_string_view<C> delimiter, bool discard)
 	{
-		while (Read((result += C{}).back())) if (result.back() == '|' && result.find(delimiter, result.size() - delimiter.size()) != std::string::npos) return true;
-		return false;
-	}
-
-	bool Read(C& out)
-	{
-		BYTE buffer[sizeof(C)];
-		for (int i = 0; i < sizeof(C); ++i, ++it)
-			if (it.equal({})) return false;
-			else buffer[i] = *it;
-		out = reinterpret_cast<C&>(buffer);
-		return true;
+		for (int i = 0; ;)
+		{
+			int pos = buffer.find(delimiter, i);
+			if (pos != std::string::npos)
+			{
+				auto result = !discard ? std::optional(std::basic_string(buffer.begin(), buffer.begin() + pos)) : std::nullopt;
+				buffer.erase(buffer.begin(), buffer.begin() + pos + delimiter.size());
+				return result;
+			}
+			int oldSize = buffer.size();
+			buffer.resize(oldSize + blockSize);
+			if (!streambuf.sgetn((char*)buffer.data() + oldSize, blockSize * sizeof(C))) return {};
+			i = max(0, oldSize - (int)delimiter.size());
+			if (discard)
+			{
+				buffer.erase(0, i);
+				i = 0;
+			}
+		}
 	}

 	static constexpr C endImpl[5] = { '|', 'E', 'N', 'D', '|' };
 	static constexpr std::basic_string_view end{ endImpl, 5 };

-	std::istreambuf_iterator<char> it;
-	std::array<std::basic_string_view<C>, DelimiterCount> delimiters;
+	std::basic_streambuf<char>& streambuf;
+	std::basic_string<C> buffer;
+	std::array<std::basic_string_view<C>, delimiterCount> delimiters;
 };
--- a/extensions/googletranslate.cpp
+++ b/extensions/googletranslate.cpp
@ -11,7 +11,6 @@ extern Synchronized<std::wstring> translateTo;
 const char* TRANSLATION_PROVIDER = "Google";
 QStringList languages
 {
-	"English: en",
 	"Afrikaans: af",
 	"Arabic: ar",
 	"Albanian: sq",
@ -26,6 +25,7 @@ QStringList languages
 	"Czech: cs",
 	"Danish: da",
 	"Dutch: nl",
+	"English: en",
 	"Esperanto: eo",
 	"Estonian: et",
 	"Filipino: tl",
--- a/extensions/translatewrapper.cpp
+++ b/extensions/translatewrapper.cpp
@ -1,10 +1,13 @@
 #include "qtcommon.h"
 #include "extension.h"
 #include "defs.h"
+#include "util.h"
+#include "blockmarkup.h"
 #include "network.h"
-#include <map>
+#include <fstream>
 #include <QTimer>

+extern const char* NATIVE_LANGUAGE;
 extern const char* SELECT_LANGUAGE;
 extern const char* SELECT_LANGUAGE_MESSAGE;
 extern const char* LANGUAGE_SAVED;
@ -15,19 +18,25 @@ extern QStringList languages;
 std::pair<bool, std::wstring> Translate(const std::wstring& text);

 const char* LANGUAGE = u8"Language";
-const QString CACHE_FILE = QString("%1 Cache.txt").arg(TRANSLATION_PROVIDER);
+const std::string TRANSLATION_CACHE_FILE = FormatString("%sCache.txt", TRANSLATION_PROVIDER);

 Synchronized<std::wstring> translateTo = L"en";
 QSettings settings(CONFIG_FILE, QSettings::IniFormat);
+
+struct Translation
+{
+	std::wstring original, translation;
+	bool operator<(const Translation& other) const { return original < other.original; }
+};
+Synchronized<std::vector<Translation>> translationCache;
 int savedSize;
-Synchronized<std::map<std::wstring, std::wstring>> translationCache;

 void SaveCache()
 {
-	QTextFile file(CACHE_FILE, QIODevice::WriteOnly | QIODevice::Truncate);
-	auto translationCache = ::translationCache.Acquire();
-	for (const auto& [original, translation] : translationCache.contents)
-		file.write(S(FormatString(L"%s|T|\n%s|T|\n", original, translation)).toUtf8());
+	std::wstring allTranslations(L"\xfeff");
+	for (const auto& [original, translation] : translationCache.Acquire().contents)
+		allTranslations.append(L"|SENTENCE|").append(original).append(L"|TRANSLATION|").append(translation).append(L"|END|\r\n");
+	std::ofstream(TRANSLATION_CACHE_FILE, std::ios::binary | std::ios::trunc).write((const char*)allTranslations.c_str(), allTranslations.size() * sizeof(wchar_t));
 	savedSize = translationCache->size();
 }

@ -46,7 +55,7 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved
 				SELECT_LANGUAGE,
 				QString(SELECT_LANGUAGE_MESSAGE).arg(TRANSLATION_PROVIDER),
 				languages,
-				0,
+				std::find_if(languages.begin(), languages.end(), [](QString language) { return language.startsWith(NATIVE_LANGUAGE); }) - languages.begin(),
 				false,
 				nullptr,
 				Qt::WindowCloseButtonHint
@ -56,10 +65,16 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved
 			QMessageBox::information(nullptr, SELECT_LANGUAGE, QString(LANGUAGE_SAVED).arg(CONFIG_FILE));
 		});

-		QStringList savedCache = QString(QTextFile(CACHE_FILE, QIODevice::ReadOnly).readAll()).split("|T|\n", QString::SkipEmptyParts);
-		for (int i = 0; i < savedCache.size() - 1; i += 2)
-			translationCache->insert({ S(savedCache[i]), S(savedCache[i + 1]) });
+		std::ifstream stream(TRANSLATION_CACHE_FILE, std::ios::binary);
+		BlockMarkupIterator savedTranslations(stream, Array<std::wstring_view>{ L"|SENTENCE|", L"|TRANSLATION|" });
+		auto translationCache = ::translationCache.Acquire();
+		while (auto read = savedTranslations.Next())
+		{
+			const auto& [original, translation] = read.value();
+			translationCache->push_back({ original, translation });
+		}
 		savedSize = translationCache->size();
+		std::sort(translationCache->begin(), translationCache->end());
 	}
 	break;
 	case DLL_PROCESS_DETACH:
@ -94,14 +109,15 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)

 	bool cache = false;
 	std::wstring translation;
-	if (translationCache->count(sentence) != 0) translation = translationCache->at(sentence);
-	else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS;
-	else std::tie(cache, translation) = Translate(sentence);
-	if (cache && sentenceInfo["current select"])
 	{
-		translationCache->insert({ sentence, translation });
-		if (translationCache->size() > savedSize + 50) SaveCache();
+		auto translationCache = ::translationCache.Acquire();
+		auto translationLocation = std::lower_bound(translationCache->begin(), translationCache->end(), Translation{ sentence });
+		if (translationLocation != translationCache->end() && translationLocation->original == sentence) translation = translationLocation->translation;
+		else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS;
+		else std::tie(cache, translation) = Translate(sentence);
+		if (cache && sentenceInfo["current select"]) translationCache->insert(translationLocation, { sentence, translation });
 	}
+	if (cache && translationCache->size() > savedSize + 50) SaveCache();

 	Unescape(translation);
 	sentence += L"\n" + translation;
--- a/text.cpp
+++ b/text.cpp
@ -4,6 +4,7 @@
 #define ARCH "x86"
 #endif

+const char* NATIVE_LANGUAGE = "English";
 const char* ATTACH = u8"Attach to game";
 const char* LAUNCH = u8"Launch game";
 const char* DETACH = u8"Detach from game";
@ -185,6 +186,7 @@ const char* HEXADECIMAL = u8"Hexadecimal";
 static auto _ = []
 {
 #ifdef TURKISH
+	NATIVE_LANGUAGE = "Turkish";
 	ATTACH = u8"Oyuna bağla";
 	DETACH = u8"Oyundan kopar";
 	ADD_HOOK = u8"Kanca ekle";
@ -218,6 +220,7 @@ Kaynak kodu GKLv3 koruması altında proje ana sayfasında mevcut)";
 #endif // TURKISH

 #ifdef SPANISH
+	NATIVE_LANGUAGE = "Spanish";
 	ATTACH = u8"Adjuntar juego";
 	LAUNCH = u8"Iniciar juego";
 	DETACH = u8"Desconectar juego";
@ -280,6 +283,7 @@ Clic y arrastra los bordes de la ventana para moverla, o en la esquina inferior
 #endif // SPANISH

 #ifdef SIMPLIFIED_CHINESE
+	NATIVE_LANGUAGE = "Chinese(Simplified)"
 	ATTACH = u8"附加到游戏";
 	LAUNCH = u8"启动游戏";
 	DETACH = u8"从游戏分离";
@ -344,6 +348,7 @@ Clic y arrastra los bordes de la ventana para moverla, o en la esquina inferior
 #endif // SIMPLIFIED_CHINESE

 #ifdef RUSSIAN
+	NATIVE_LANGUAGE = "Russian";
 	ATTACH = u8"Присоединить к игре";
 	LAUNCH = u8"Запустить игру";
 	DETACH = u8"Отсоединить от игры";
@ -475,6 +480,7 @@ end)";
 #endif // RUSSIAN

 #ifdef INDONESIAN
+	NATIVE_LANGUAGE = "Indonesian";
 	ATTACH = u8"Tempelkan kedalam game";
 	LAUNCH = u8"Mulai game";
 	DETACH = u8"Lepaskan dari game";
@ -539,6 +545,7 @@ Klik dan tarik pinggiran jendela untuk memindahkan, atau sudut kanan bawah untuk
 #endif // INDONESIAN

 #ifdef PORTUGUESE
+	NATIVE_LANGUAGE = "Portuguese";
 	ATTACH = u8"Anexar ao Jogo";
 	LAUNCH = u8"Iniciar Jogo";
 	DETACH = u8"Desconectar do Jogo";
@ -626,6 +633,7 @@ Esse arquívo deve ser codifícado em (UTF-16 little endian).)";
 #endif // PORTUGUESE

 #ifdef THAI
+	NATIVE_LANGUAGE = "Thai";
 	ATTACH = u8"เชื่อมเกม";
 	LAUNCH = u8"เริ่มเกม";
 	DETACH = u8"ยกเลิกการเชื่อม";
@ -687,6 +695,7 @@ Source code สามารถหาได้จากส่วนของ GPLv
 #endif // THAI

 #ifdef KOREAN
+	NATIVE_LANGUAGE = "Korean";
 	ATTACH = u8"게임에 부착";
 	LAUNCH = u8"게임 실행";
 	DETACH = u8"게임에서 탈착";