From dc48f2a3c8a7833729c4a990e49bb682ca8d19c9 Mon Sep 17 00:00:00 2001 From: Akash Mozumdar Date: Tue, 25 Feb 2020 04:39:27 -0700 Subject: [PATCH] performance improvements and autoselect native language for translations --- GUI/host/host.cpp | 22 +++++------ extensions/bingtranslate.cpp | 2 +- extensions/blockmarkup.h | 66 +++++++++++++++++---------------- extensions/googletranslate.cpp | 2 +- extensions/translatewrapper.cpp | 50 ++++++++++++++++--------- text.cpp | 9 +++++ 6 files changed, 89 insertions(+), 62 deletions(-) diff --git a/GUI/host/host.cpp b/GUI/host/host.cpp index 5609445..b055e14 100644 --- a/GUI/host/host.cpp +++ b/GUI/host/host.cpp @@ -68,10 +68,8 @@ namespace void RemoveThreads(std::function removeIf) { std::vector threadsToRemove; - { - auto textThreadsByParams = ::textThreadsByParams.Acquire(); - std::for_each(textThreadsByParams->begin(), textThreadsByParams->end(), [&](auto& it) { if (removeIf(it.first)) threadsToRemove.push_back(&it.second); }); - } + for (auto& [tp, thread] : textThreadsByParams.Acquire().contents) + if (removeIf(tp)) threadsToRemove.push_back(&thread); for (auto thread : threadsToRemove) { OnDestroy(*thread); @@ -131,14 +129,14 @@ namespace { auto tp = *(ThreadParam*)buffer; auto textThreadsByParams = ::textThreadsByParams.Acquire(); - auto textThread = textThreadsByParams->find(tp); - if (textThread == textThreadsByParams->end()) + auto thread = textThreadsByParams->find(tp); + if (thread == textThreadsByParams->end()) { - try { textThread = textThreadsByParams->try_emplace(tp, tp, processRecordsByIds->at(tp.processId).GetHook(tp.addr).hp).first; } + try { thread = textThreadsByParams->try_emplace(tp, tp, processRecordsByIds->at(tp.processId).GetHook(tp.addr).hp).first; } catch (std::out_of_range) { continue; } // probably garbage data in pipe, try again - OnCreate(textThread->second); + OnCreate(thread->second); } - textThread->second.Push(buffer + sizeof(tp), bytesRead - sizeof(tp)); + thread->second.Push(buffer + sizeof(tp), bytesRead - sizeof(tp)); } break; } @@ -247,9 +245,9 @@ namespace Host TextThread* GetThread(int64_t handle) { - auto textThreadsByParams = ::textThreadsByParams.Acquire(); - auto thread = std::find_if(textThreadsByParams->begin(), textThreadsByParams->end(), [&](const auto& thread) { return thread.second.handle == handle; }); - return thread != textThreadsByParams->end() ? &thread->second : nullptr; + for (auto& [tp, thread] : textThreadsByParams.Acquire().contents) + if (thread.handle == handle) return &thread; + return nullptr; } void AddConsoleOutput(std::wstring text) diff --git a/extensions/bingtranslate.cpp b/extensions/bingtranslate.cpp index 8f57f02..7a1b784 100644 --- a/extensions/bingtranslate.cpp +++ b/extensions/bingtranslate.cpp @@ -9,7 +9,6 @@ extern Synchronized translateTo; const char* TRANSLATION_PROVIDER = "Bing"; QStringList languages { - "English: en", "Arabic: ar", "Bosnian: bs-Latn", "Bulgarian: bg", @@ -20,6 +19,7 @@ QStringList languages "Czech: cs", "Danish: da", "Dutch: nl", + "English: en", "Estonian: et", "Finnish: fi", "French: fr", diff --git a/extensions/blockmarkup.h b/extensions/blockmarkup.h index f3975d8..905f2ff 100644 --- a/extensions/blockmarkup.h +++ b/extensions/blockmarkup.h @@ -3,53 +3,57 @@ #include "common.h" #include -template +template // windows file block size class BlockMarkupIterator { public: - BlockMarkupIterator(std::istreambuf_iterator it, const std::basic_string_view (&delimiters)[DelimiterCount]) : - it(it) + BlockMarkupIterator(const std::istream& it, const std::basic_string_view(&delimiters)[delimiterCount]) : + streambuf(*it.rdbuf()) { - std::copy_n(delimiters, DelimiterCount, this->delimiters.begin()); + std::copy_n(delimiters, delimiterCount, this->delimiters.begin()); } - std::optional, DelimiterCount>> Next() + + std::optional, delimiterCount>> Next() { - std::array, DelimiterCount> results; - std::basic_string buffer; - buffer.reserve(bufferStartSize); - Find(buffer, delimiters[0]); - buffer.clear(); - for (int i = 0; i < DelimiterCount; ++i) + std::array, delimiterCount> results; + Find(delimiters[0], true); + for (int i = 0; i < delimiterCount; ++i) { - const auto delimiter = i + 1 < DelimiterCount ? delimiters[i + 1] : end; - if (!Find(buffer, delimiter)) return {}; - buffer.erase(buffer.size() - delimiter.size()); - results[i] = std::move(buffer); - (buffer = {}).reserve(bufferStartSize); + const auto delimiter = i + 1 < delimiterCount ? delimiters[i + 1] : end; + if (auto found = Find(delimiter, false)) results[i] = std::move(found.value()); + else return {}; } return results; } private: - bool Find(std::basic_string& result, std::basic_string_view delimiter) + std::optional> Find(std::basic_string_view delimiter, bool discard) { - while (Read((result += C{}).back())) if (result.back() == '|' && result.find(delimiter, result.size() - delimiter.size()) != std::string::npos) return true; - return false; - } - - bool Read(C& out) - { - BYTE buffer[sizeof(C)]; - for (int i = 0; i < sizeof(C); ++i, ++it) - if (it.equal({})) return false; - else buffer[i] = *it; - out = reinterpret_cast(buffer); - return true; + for (int i = 0; ;) + { + int pos = buffer.find(delimiter, i); + if (pos != std::string::npos) + { + auto result = !discard ? std::optional(std::basic_string(buffer.begin(), buffer.begin() + pos)) : std::nullopt; + buffer.erase(buffer.begin(), buffer.begin() + pos + delimiter.size()); + return result; + } + int oldSize = buffer.size(); + buffer.resize(oldSize + blockSize); + if (!streambuf.sgetn((char*)buffer.data() + oldSize, blockSize * sizeof(C))) return {}; + i = max(0, oldSize - (int)delimiter.size()); + if (discard) + { + buffer.erase(0, i); + i = 0; + } + } } static constexpr C endImpl[5] = { '|', 'E', 'N', 'D', '|' }; static constexpr std::basic_string_view end{ endImpl, 5 }; - std::istreambuf_iterator it; - std::array, DelimiterCount> delimiters; + std::basic_streambuf& streambuf; + std::basic_string buffer; + std::array, delimiterCount> delimiters; }; diff --git a/extensions/googletranslate.cpp b/extensions/googletranslate.cpp index eacd7c1..e6ce5c8 100644 --- a/extensions/googletranslate.cpp +++ b/extensions/googletranslate.cpp @@ -11,7 +11,6 @@ extern Synchronized translateTo; const char* TRANSLATION_PROVIDER = "Google"; QStringList languages { - "English: en", "Afrikaans: af", "Arabic: ar", "Albanian: sq", @@ -26,6 +25,7 @@ QStringList languages "Czech: cs", "Danish: da", "Dutch: nl", + "English: en", "Esperanto: eo", "Estonian: et", "Filipino: tl", diff --git a/extensions/translatewrapper.cpp b/extensions/translatewrapper.cpp index 5a2e174..dd7694f 100644 --- a/extensions/translatewrapper.cpp +++ b/extensions/translatewrapper.cpp @@ -1,10 +1,13 @@ #include "qtcommon.h" #include "extension.h" #include "defs.h" +#include "util.h" +#include "blockmarkup.h" #include "network.h" -#include +#include #include +extern const char* NATIVE_LANGUAGE; extern const char* SELECT_LANGUAGE; extern const char* SELECT_LANGUAGE_MESSAGE; extern const char* LANGUAGE_SAVED; @@ -15,19 +18,25 @@ extern QStringList languages; std::pair Translate(const std::wstring& text); const char* LANGUAGE = u8"Language"; -const QString CACHE_FILE = QString("%1 Cache.txt").arg(TRANSLATION_PROVIDER); +const std::string TRANSLATION_CACHE_FILE = FormatString("%sCache.txt", TRANSLATION_PROVIDER); Synchronized translateTo = L"en"; QSettings settings(CONFIG_FILE, QSettings::IniFormat); + +struct Translation +{ + std::wstring original, translation; + bool operator<(const Translation& other) const { return original < other.original; } +}; +Synchronized> translationCache; int savedSize; -Synchronized> translationCache; void SaveCache() { - QTextFile file(CACHE_FILE, QIODevice::WriteOnly | QIODevice::Truncate); - auto translationCache = ::translationCache.Acquire(); - for (const auto& [original, translation] : translationCache.contents) - file.write(S(FormatString(L"%s|T|\n%s|T|\n", original, translation)).toUtf8()); + std::wstring allTranslations(L"\xfeff"); + for (const auto& [original, translation] : translationCache.Acquire().contents) + allTranslations.append(L"|SENTENCE|").append(original).append(L"|TRANSLATION|").append(translation).append(L"|END|\r\n"); + std::ofstream(TRANSLATION_CACHE_FILE, std::ios::binary | std::ios::trunc).write((const char*)allTranslations.c_str(), allTranslations.size() * sizeof(wchar_t)); savedSize = translationCache->size(); } @@ -46,7 +55,7 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved SELECT_LANGUAGE, QString(SELECT_LANGUAGE_MESSAGE).arg(TRANSLATION_PROVIDER), languages, - 0, + std::find_if(languages.begin(), languages.end(), [](QString language) { return language.startsWith(NATIVE_LANGUAGE); }) - languages.begin(), false, nullptr, Qt::WindowCloseButtonHint @@ -56,10 +65,16 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved QMessageBox::information(nullptr, SELECT_LANGUAGE, QString(LANGUAGE_SAVED).arg(CONFIG_FILE)); }); - QStringList savedCache = QString(QTextFile(CACHE_FILE, QIODevice::ReadOnly).readAll()).split("|T|\n", QString::SkipEmptyParts); - for (int i = 0; i < savedCache.size() - 1; i += 2) - translationCache->insert({ S(savedCache[i]), S(savedCache[i + 1]) }); + std::ifstream stream(TRANSLATION_CACHE_FILE, std::ios::binary); + BlockMarkupIterator savedTranslations(stream, Array{ L"|SENTENCE|", L"|TRANSLATION|" }); + auto translationCache = ::translationCache.Acquire(); + while (auto read = savedTranslations.Next()) + { + const auto& [original, translation] = read.value(); + translationCache->push_back({ original, translation }); + } savedSize = translationCache->size(); + std::sort(translationCache->begin(), translationCache->end()); } break; case DLL_PROCESS_DETACH: @@ -94,14 +109,15 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) bool cache = false; std::wstring translation; - if (translationCache->count(sentence) != 0) translation = translationCache->at(sentence); - else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS; - else std::tie(cache, translation) = Translate(sentence); - if (cache && sentenceInfo["current select"]) { - translationCache->insert({ sentence, translation }); - if (translationCache->size() > savedSize + 50) SaveCache(); + auto translationCache = ::translationCache.Acquire(); + auto translationLocation = std::lower_bound(translationCache->begin(), translationCache->end(), Translation{ sentence }); + if (translationLocation != translationCache->end() && translationLocation->original == sentence) translation = translationLocation->translation; + else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS; + else std::tie(cache, translation) = Translate(sentence); + if (cache && sentenceInfo["current select"]) translationCache->insert(translationLocation, { sentence, translation }); } + if (cache && translationCache->size() > savedSize + 50) SaveCache(); Unescape(translation); sentence += L"\n" + translation; diff --git a/text.cpp b/text.cpp index ced37c5..3c16331 100644 --- a/text.cpp +++ b/text.cpp @@ -4,6 +4,7 @@ #define ARCH "x86" #endif +const char* NATIVE_LANGUAGE = "English"; const char* ATTACH = u8"Attach to game"; const char* LAUNCH = u8"Launch game"; const char* DETACH = u8"Detach from game"; @@ -185,6 +186,7 @@ const char* HEXADECIMAL = u8"Hexadecimal"; static auto _ = [] { #ifdef TURKISH + NATIVE_LANGUAGE = "Turkish"; ATTACH = u8"Oyuna bağla"; DETACH = u8"Oyundan kopar"; ADD_HOOK = u8"Kanca ekle"; @@ -218,6 +220,7 @@ Kaynak kodu GKLv3 koruması altında proje ana sayfasında mevcut)"; #endif // TURKISH #ifdef SPANISH + NATIVE_LANGUAGE = "Spanish"; ATTACH = u8"Adjuntar juego"; LAUNCH = u8"Iniciar juego"; DETACH = u8"Desconectar juego"; @@ -280,6 +283,7 @@ Clic y arrastra los bordes de la ventana para moverla, o en la esquina inferior #endif // SPANISH #ifdef SIMPLIFIED_CHINESE + NATIVE_LANGUAGE = "Chinese(Simplified)" ATTACH = u8"附加到游戏"; LAUNCH = u8"启动游戏"; DETACH = u8"从游戏分离"; @@ -344,6 +348,7 @@ Clic y arrastra los bordes de la ventana para moverla, o en la esquina inferior #endif // SIMPLIFIED_CHINESE #ifdef RUSSIAN + NATIVE_LANGUAGE = "Russian"; ATTACH = u8"Присоединить к игре"; LAUNCH = u8"Запустить игру"; DETACH = u8"Отсоединить от игры"; @@ -475,6 +480,7 @@ end)"; #endif // RUSSIAN #ifdef INDONESIAN + NATIVE_LANGUAGE = "Indonesian"; ATTACH = u8"Tempelkan kedalam game"; LAUNCH = u8"Mulai game"; DETACH = u8"Lepaskan dari game"; @@ -539,6 +545,7 @@ Klik dan tarik pinggiran jendela untuk memindahkan, atau sudut kanan bawah untuk #endif // INDONESIAN #ifdef PORTUGUESE + NATIVE_LANGUAGE = "Portuguese"; ATTACH = u8"Anexar ao Jogo"; LAUNCH = u8"Iniciar Jogo"; DETACH = u8"Desconectar do Jogo"; @@ -626,6 +633,7 @@ Esse arquívo deve ser codifícado em (UTF-16 little endian).)"; #endif // PORTUGUESE #ifdef THAI + NATIVE_LANGUAGE = "Thai"; ATTACH = u8"เชื่อมเกม"; LAUNCH = u8"เริ่มเกม"; DETACH = u8"ยกเลิกการเชื่อม"; @@ -687,6 +695,7 @@ Source code สามารถหาได้จากส่วนของ GPLv #endif // THAI #ifdef KOREAN + NATIVE_LANGUAGE = "Korean"; ATTACH = u8"게임에 부착"; LAUNCH = u8"게임 실행"; DETACH = u8"게임에서 탈착";