From 87c056a5b30ff656fc524f59971f09b573ef5ce3 Mon Sep 17 00:00:00 2001 From: Akash Mozumdar Date: Thu, 26 Mar 2020 05:41:21 -0600 Subject: [PATCH] add settings to translation dialog and implement deepl --- GUI/mainwindow.cpp | 6 +- deploy.ps1 | 1 + extensions/CMakeLists.txt | 2 + extensions/bingtranslate.cpp | 4 +- extensions/deepltranslate.cpp | 113 ++++++++++++++++++++++++++++ extensions/extrawindow.cpp | 9 ++- extensions/googlecloudtranslate.cpp | 2 +- extensions/googletranslate.cpp | 2 +- extensions/network.cpp | 4 + extensions/network.h | 1 + extensions/translatewrapper.cpp | 49 +++++++++--- text.cpp | 8 +- 12 files changed, 181 insertions(+), 20 deletions(-) create mode 100644 extensions/deepltranslate.cpp diff --git a/GUI/mainwindow.cpp b/GUI/mainwindow.cpp index 19f2715..cce191e 100644 --- a/GUI/mainwindow.cpp +++ b/GUI/mainwindow.cpp @@ -297,8 +297,8 @@ namespace QDialog dialog(This, Qt::WindowCloseButtonHint); QFormLayout layout(&dialog); - QCheckBox cjkCheckbox(&dialog); - layout.addRow(SEARCH_CJK, &cjkCheckbox); + QCheckBox cjkCheckBox(&dialog); + layout.addRow(SEARCH_CJK, &cjkCheckBox); QDialogButtonBox confirm(QDialogButtonBox::Ok | QDialogButtonBox::Help | QDialogButtonBox::Retry, &dialog); layout.addRow(&confirm); confirm.button(QDialogButtonBox::Ok)->setText(START_HOOK_SEARCH); @@ -388,7 +388,7 @@ namespace else { sp.length = 0; // use default - filter.setPattern(cjkCheckbox.isChecked() ? "[\\x{3000}-\\x{a000}]{4,}" : "[\\x{0020}-\\x{1000}]{4,}"); + filter.setPattern(cjkCheckBox.isChecked() ? "[\\x{3000}-\\x{a000}]{4,}" : "[\\x{0020}-\\x{1000}]{4,}"); } filter.optimize(); diff --git a/deploy.ps1 b/deploy.ps1 index 77d6f92..13c30bd 100644 --- a/deploy.ps1 +++ b/deploy.ps1 @@ -37,6 +37,7 @@ foreach ($language in @{ "texthook.dll", "Bing Translate.dll", "Copy to Clipboard.dll", + "DeepL Translate.dll", "Extra Newlines.dll", "Extra Window.dll", "Google Translate.dll", diff --git a/extensions/CMakeLists.txt b/extensions/CMakeLists.txt index 6c1acd3..172a246 100644 --- a/extensions/CMakeLists.txt +++ b/extensions/CMakeLists.txt @@ -6,6 +6,7 @@ cmake_policy(SET CMP0037 OLD) add_library(Bing\ Translate MODULE bingtranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) add_library(Copy\ to\ Clipboard MODULE copyclipboard.cpp extensionimpl.cpp) +add_library(DeepL\ Translate MODULE deepltranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) add_library(Extra\ Newlines MODULE extranewlines.cpp extensionimpl.cpp) add_library(Extra\ Window MODULE extrawindow.cpp extensionimpl.cpp) add_library(Google\ Translate MODULE googletranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) @@ -20,6 +21,7 @@ add_library(Replacer MODULE replacer.cpp extensionimpl.cpp) add_library(Thread\ Linker MODULE threadlinker.cpp extensionimpl.cpp) target_link_libraries(Bing\ Translate winhttp Qt5::Widgets) +target_link_libraries(DeepL\ Translate winhttp Qt5::Widgets) target_link_libraries(Extra\ Window Qt5::Widgets) target_link_libraries(Google\ Translate winhttp Qt5::Widgets) target_link_libraries(Google\ Cloud\ Translate winhttp Qt5::Widgets) diff --git a/extensions/bingtranslate.cpp b/extensions/bingtranslate.cpp index a85fac9..5cd5d36 100644 --- a/extensions/bingtranslate.cpp +++ b/extensions/bingtranslate.cpp @@ -79,7 +79,7 @@ QStringList languages "Yucatec Maya: yua" }; -std::pair Translate(const std::wstring& text) +std::pair Translate(const std::wstring& text, SentenceInfo) { if (HttpRequest httpRequest{ L"Mozilla/5.0 Textractor", @@ -88,7 +88,7 @@ std::pair Translate(const std::wstring& text) FormatString(L"/ttranslatev3?fromLang=auto-detect&to=%s&text=%s", translateTo->c_str(), Escape(text)).c_str() }) // Response formatted as JSON: translation starts with text":" and ends with ","to - if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"text\":\"(.+)\",\"t"))) return { true, results[1] }; + if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"text\":\"(.+?)\",\""))) return { true, results[1] }; else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) }; else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) }; } diff --git a/extensions/deepltranslate.cpp b/extensions/deepltranslate.cpp new file mode 100644 index 0000000..9ad9369 --- /dev/null +++ b/extensions/deepltranslate.cpp @@ -0,0 +1,113 @@ +#include "qtcommon.h" +#include "extension.h" +#include "network.h" +#include + +extern const wchar_t* TRANSLATION_ERROR; +extern const char* USE_PREV_SENTENCE_CONTEXT; + +extern QSettings settings; +extern QFormLayout* display; +extern Synchronized translateTo; + +const char* TRANSLATION_PROVIDER = "DeepL Translate"; +QStringList languages +{ + "Chinese: ZH", + "Dutch: NL", + "English: EN", + "French: FR", + "German: DE", + "Italian: IT", + "Japanese: JA", + "Polish: PL", + "Portuguese: PT", + "Russian: RU", + "Spanish: ES", +}; + +const wchar_t* accept[] = { L"*/*", nullptr }; + +Synchronized LMTBID; + +bool useContext = true; +Synchronized> context; + +BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) +{ + switch (ul_reason_for_call) + { + case DLL_PROCESS_ATTACH: + { + auto checkbox = new QCheckBox; + checkbox->setChecked(useContext); + display->addRow(USE_PREV_SENTENCE_CONTEXT, checkbox); + QObject::connect(checkbox, &QCheckBox::clicked, [](bool checked) { settings.setValue(USE_PREV_SENTENCE_CONTEXT, useContext = checked); }); + } + break; + case DLL_PROCESS_DETACH: + { + } + break; + } + return TRUE; +} + +std::pair Translate(const std::wstring& text, SentenceInfo sentenceInfo) +{ + // the following code was reverse engineered from the DeepL website; it's as close as I could make it but I'm not sure what parts of this could be removed and still have it work + int64_t r = _time64(nullptr), n = std::count(text.begin(), text.end(), L'i') + 1LL; + static std::atomic id = 10000 * std::uniform_int_distribution(0, 9999)(std::mt19937(std::random_device()())); + std::string jsonText; + for (auto ch : WideStringToString(text)) + if (ch == '"') jsonText += "\\\""; + else jsonText += ch; + // user_preferred_langs? what should preferred_num_beans and priority be? does timestamp do anything? other translation quality options? + auto body = FormatString(R"( +{ + "id": %d, + "jsonrpc": "2.0", + "method": "LMT_handle_jobs", + "params": { + "priority": -1, + "timestamp": %lld, + "lang": { + "source_lang_user_selected": "auto", + "target_lang": "%s" + }, + "jobs": [{ + "kind": "default", + "preferred_num_beams": 4, + "quality": "fast", + "raw_en_context_after": [], + "raw_en_sentence": "%s", + "raw_en_context_before": [%s] + }] + } +} + )", ++id, r + (n - r % n), WideStringToString(translateTo->c_str()), jsonText, useContext ? WideStringToString(context->operator[](sentenceInfo["text number"])) : ""); + context->insert_or_assign(sentenceInfo["text number"], L'"' + text + L'"'); + std::wstring headers = L"Host: www2.deepl.com\r\nAccept-Language: en-US,en;q=0.5\r\nContent-type: text/plain\r\nOrigin: https://www.deepl.com\r\nTE: Trailers" + + LMTBID.Acquire().contents; + if (HttpRequest httpRequest{ + L"Mozilla/5.0 Textractor", + L"www2.deepl.com", + L"POST", + L"/jsonrpc", + WINHTTP_FLAG_SECURE, + NULL, + L"https://www.deepl.com/translator", + accept, + headers.c_str(), + body.data(), + body.size() + }) + { + auto LMTBID = httpRequest.headers.find(L"LMTBID="), end = httpRequest.headers.find(L';', LMTBID); // not sure if this cookie does anything + if (LMTBID != std::wstring::npos && end != std::wstring::npos) ::LMTBID->assign(L"\r\nCookie: " + httpRequest.headers.substr(LMTBID, end - LMTBID)); + // Response formatted as JSON: translation starts with preprocessed_sentence":" and ends with "," + if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"postprocessed_sentence\":\"(.+?)\",\""))) return { true, results[1] }; + else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) }; + } + else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) }; +} diff --git a/extensions/extrawindow.cpp b/extensions/extrawindow.cpp index b1c75a4..affb1ef 100644 --- a/extensions/extrawindow.cpp +++ b/extensions/extrawindow.cpp @@ -270,8 +270,9 @@ private: dictionaryWindow.ui.display->setFixedWidth(ui.display->width() * 3 / 4); dictionaryWindow.setTerm(sentence.mid(i)); int left = i == 0 ? 0 : textPositionMap[i - 1].x(), right = textPositionMap[i].x(), - x = textPositionMap[i].x() > ui.display->width() / 2 ? -dictionaryWindow.width() + (right * 3 + left) / 4 : (left * 3 + right) / 4; - dictionaryWindow.move(ui.display->mapToGlobal(QPoint(x, textPositionMap[i].y()))); + x = textPositionMap[i].x() > ui.display->width() / 2 ? -dictionaryWindow.width() + (right * 3 + left) / 4 : (left * 3 + right) / 4, y = 0; + for (auto point : textPositionMap) if (point.y() > y && point.y() < textPositionMap[i].y()) y = point.y(); + dictionaryWindow.move(ui.display->mapToGlobal(QPoint(x, y - dictionaryWindow.height() + 1))); } bool eventFilter(QObject*, QEvent* event) override @@ -397,7 +398,7 @@ private: for (term = term.left(100); !term.isEmpty(); term.chop(1)) for (const auto& [rootTerm, definition, inflections] : LookupDefinitions(term, foundDefinitions)) definitions.push_back( - QStringLiteral("

%1 (%5/%6)

%2%3

%4

").arg( + QStringLiteral("

%1 (%5/%6)

%2%3%4").arg( term.split("<<")[0].toHtmlEscaped(), rootTerm.split("<<")[0].toHtmlEscaped(), inflections.join(""), @@ -455,7 +456,9 @@ private: int scroll = event->angleDelta().y(); if (scroll > 0 && definitionIndex > 0) definitionIndex -= 1; if (scroll < 0 && definitionIndex + 1 < definitions.size()) definitionIndex += 1; + int oldHeight = height(); ShowDefinition(); + move(x(), y() + oldHeight - height()); } struct Inflection diff --git a/extensions/googlecloudtranslate.cpp b/extensions/googlecloudtranslate.cpp index 29a2e05..f67f1d5 100644 --- a/extensions/googlecloudtranslate.cpp +++ b/extensions/googlecloudtranslate.cpp @@ -101,7 +101,7 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved return TRUE; } -std::pair Translate(const std::wstring& text) +std::pair Translate(const std::wstring& text, SentenceInfo) { if (HttpRequest httpRequest{ diff --git a/extensions/googletranslate.cpp b/extensions/googletranslate.cpp index d718fe7..564498a 100644 --- a/extensions/googletranslate.cpp +++ b/extensions/googletranslate.cpp @@ -152,7 +152,7 @@ bool IsHash(const std::wstring& result) return result.size() == 32 && std::all_of(result.begin(), result.end(), [](char ch) { return (ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'z'); }); } -std::pair Translate(const std::wstring& text) +std::pair Translate(const std::wstring& text, SentenceInfo) { if (!TKK) if (HttpRequest httpRequest{ L"Mozilla/5.0 Textractor", L"translate.google.com", L"GET", L"/" }) diff --git a/extensions/network.cpp b/extensions/network.cpp index e45fdac..a075ded 100644 --- a/extensions/network.cpp +++ b/extensions/network.cpp @@ -22,6 +22,10 @@ HttpRequest::HttpRequest( if (WinHttpSendRequest(request, headers, -1UL, body, bodyLength, bodyLength, NULL)) { WinHttpReceiveResponse(request, NULL); + DWORD size = 0; + WinHttpQueryHeaders(request, WINHTTP_QUERY_RAW_HEADERS_CRLF, WINHTTP_HEADER_NAME_BY_INDEX, NULL, &size, WINHTTP_NO_HEADER_INDEX); + this->headers.resize(size); + WinHttpQueryHeaders(request, WINHTTP_QUERY_RAW_HEADERS_CRLF, WINHTTP_HEADER_NAME_BY_INDEX, this->headers.data(), &size, WINHTTP_NO_HEADER_INDEX); std::string data; DWORD availableSize, downloadedSize; do diff --git a/extensions/network.h b/extensions/network.h index 90a2653..9546cfa 100644 --- a/extensions/network.h +++ b/extensions/network.h @@ -23,6 +23,7 @@ struct HttpRequest operator bool() { return errorCode == ERROR_SUCCESS; } std::wstring response; + std::wstring headers; InternetHandle connection = NULL; InternetHandle request = NULL; DWORD errorCode = ERROR_SUCCESS; diff --git a/extensions/translatewrapper.cpp b/extensions/translatewrapper.cpp index 08f1de9..c046cff 100644 --- a/extensions/translatewrapper.cpp +++ b/extensions/translatewrapper.cpp @@ -8,11 +8,16 @@ extern const char* NATIVE_LANGUAGE; extern const char* TRANSLATE_TO; +extern const char* RATE_LIMIT_ALL_THREADS; +extern const char* RATE_LIMIT_SELECTED_THREAD; +extern const char* USE_TRANS_CACHE; +extern const char* RATE_LIMIT_TOKEN_COUNT; +extern const char* RATE_LIMIT_TOKEN_RESTORE_DELAY; extern const wchar_t* TOO_MANY_TRANS_REQUESTS; extern const char* TRANSLATION_PROVIDER; extern QStringList languages; -std::pair Translate(const std::wstring& text); +std::pair Translate(const std::wstring& text, SentenceInfo sentenceInfo); const char* LANGUAGE = u8"Language"; const std::string TRANSLATION_CACHE_FILE = FormatString("%s Cache.txt", TRANSLATION_PROVIDER); @@ -21,6 +26,8 @@ QFormLayout* display; QSettings settings = openSettings(); Synchronized translateTo = L"en"; +bool rateLimitAll = true, rateLimitSelected = false, useCache = true; +int tokenCount = 30, tokenRestoreDelay = 60000; Synchronized> translationCache; int savedSize; @@ -51,8 +58,32 @@ public: if (language < 0) language = languageBox->findText("English", Qt::MatchStartsWith); languageBox->setCurrentIndex(language); saveLanguage(languageBox->currentText()); - connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage); display->addRow(TRANSLATE_TO, languageBox); + connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage); + for (auto [value, label] : Array{ + { rateLimitAll, RATE_LIMIT_ALL_THREADS }, + { rateLimitSelected, RATE_LIMIT_SELECTED_THREAD }, + { useCache, USE_TRANS_CACHE }, + }) + { + value = settings.value(label, value).toBool(); + auto checkBox = new QCheckBox(this); + checkBox->setChecked(value); + display->addRow(label, checkBox); + connect(checkBox, &QCheckBox::clicked, [label, &value](bool checked) { settings.setValue(label, value = checked); }); + } + for (auto [value, label] : Array{ + { tokenCount, RATE_LIMIT_TOKEN_COUNT }, + { tokenRestoreDelay, RATE_LIMIT_TOKEN_RESTORE_DELAY }, + }) + { + value = settings.value(label, value).toInt(); + auto spinBox = new QSpinBox(this); + spinBox->setRange(0, INT_MAX); + spinBox->setValue(value); + display->addRow(label, spinBox); + connect(spinBox, qOverload(&QSpinBox::valueChanged), [label, &value](int newValue) { settings.setValue(label, value = newValue); }); + } setWindowTitle(TRANSLATION_PROVIDER); QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection); @@ -92,25 +123,25 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) auto tokens = this->tokens.Acquire(); tokens->push_back(GetTickCount()); if (tokens->size() > tokenCount * 5) tokens->erase(tokens->begin(), tokens->begin() + tokenCount * 3); - tokens->erase(std::remove_if(tokens->begin(), tokens->end(), [this](DWORD token) { return GetTickCount() - token > delay; }), tokens->end()); + tokens->erase(std::remove_if(tokens->begin(), tokens->end(), [](DWORD token) { return GetTickCount() - token > tokenRestoreDelay; }), tokens->end()); return tokens->size() < tokenCount; } private: - const int tokenCount = 30, delay = 60 * 1000; Synchronized> tokens; } rateLimiter; bool cache = false; std::wstring translation; + if (useCache) { auto translationCache = ::translationCache.Acquire(); - auto translationLocation = translationCache->find(sentence); - if (translationLocation != translationCache->end()) translation = translationLocation->second; - else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS; - else std::tie(cache, translation) = Translate(sentence); - if (cache && sentenceInfo["current select"]) translationCache->try_emplace(translationLocation, sentence, translation); + if (auto it = translationCache->find(sentence); it != translationCache->end()) translation = it->second + L"\x200b"; } + if (translation.empty()) + if (rateLimiter.Request() || !rateLimitAll || (!rateLimitSelected && sentenceInfo["current select"])) std::tie(cache, translation) = Translate(sentence, sentenceInfo); + else translation = TOO_MANY_TRANS_REQUESTS; + if (cache) translationCache->try_emplace(sentence, translation); if (cache && translationCache->size() > savedSize + 50) SaveCache(); Unescape(translation); diff --git a/text.cpp b/text.cpp index 3a23432..06a5cc7 100644 --- a/text.cpp +++ b/text.cpp @@ -130,8 +130,14 @@ const char* READ_ERROR = u8"Textractor: Reader ERROR (likely an incorrect R-code const char* HIJACK_ERROR = u8"Textractor: Hijack ERROR"; const char* COULD_NOT_FIND = u8"Textractor: could not find text"; const char* TRANSLATE_TO = u8"Translate to"; -const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Too many translation requests: refuse to make more"; +const char* RATE_LIMIT_ALL_THREADS = u8"Rate limit all text threads"; +const char* RATE_LIMIT_SELECTED_THREAD = u8"Rate limit currently selected text thread"; +const char* USE_TRANS_CACHE = u8"Use translation cache"; +const char* RATE_LIMIT_TOKEN_COUNT = u8"Rate limiter token count"; +const char* RATE_LIMIT_TOKEN_RESTORE_DELAY = u8"Rate limiter token restore delay (ms)"; +const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Rate limit exceeded: refuse to make more translation requests"; const wchar_t* TRANSLATION_ERROR = L"Error while translating"; +const char* USE_PREV_SENTENCE_CONTEXT = u8"Use previous sentence as context"; const char* API_KEY = u8"API key"; const char* EXTRA_WINDOW_INFO = u8R"(Right click to change settings Click and drag on window edges to move, or the bottom right corner to resize)";