add settings to translation dialog and implement deepl

GUI/mainwindow.cpp

@@ -297,8 +297,8 @@ namespace
 
 		QDialog dialog(This, Qt::WindowCloseButtonHint);
 		QFormLayout layout(&dialog);
-		QCheckBox cjkCheckbox(&dialog);
+		QCheckBox cjkCheckBox(&dialog);
-		layout.addRow(SEARCH_CJK, &cjkCheckbox);
+		layout.addRow(SEARCH_CJK, &cjkCheckBox);
 		QDialogButtonBox confirm(QDialogButtonBox::Ok | QDialogButtonBox::Help | QDialogButtonBox::Retry, &dialog);
 		layout.addRow(&confirm);
 		confirm.button(QDialogButtonBox::Ok)->setText(START_HOOK_SEARCH);
@@ -388,7 +388,7 @@ namespace
 		else
 		{
 			sp.length = 0; // use default
-			filter.setPattern(cjkCheckbox.isChecked() ? "[\\x{3000}-\\x{a000}]{4,}" : "[\\x{0020}-\\x{1000}]{4,}");
+			filter.setPattern(cjkCheckBox.isChecked() ? "[\\x{3000}-\\x{a000}]{4,}" : "[\\x{0020}-\\x{1000}]{4,}");
 		}
 		filter.optimize();
 

deploy.ps1

@@ -37,6 +37,7 @@ foreach ($language in @{
 			"texthook.dll",
 			"Bing Translate.dll",
 			"Copy to Clipboard.dll",
+			"DeepL Translate.dll",
 			"Extra Newlines.dll",
 			"Extra Window.dll",
 			"Google Translate.dll",

extensions/CMakeLists.txt

@@ -6,6 +6,7 @@ cmake_policy(SET CMP0037 OLD)
 
 add_library(Bing\ Translate MODULE bingtranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp)
 add_library(Copy\ to\ Clipboard MODULE copyclipboard.cpp extensionimpl.cpp)
+add_library(DeepL\ Translate MODULE deepltranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp)
 add_library(Extra\ Newlines MODULE extranewlines.cpp extensionimpl.cpp)
 add_library(Extra\ Window MODULE extrawindow.cpp extensionimpl.cpp)
 add_library(Google\ Translate MODULE googletranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp)
@@ -20,6 +21,7 @@ add_library(Replacer MODULE replacer.cpp extensionimpl.cpp)
 add_library(Thread\ Linker MODULE threadlinker.cpp extensionimpl.cpp)
 
 target_link_libraries(Bing\ Translate winhttp Qt5::Widgets)
+target_link_libraries(DeepL\ Translate winhttp Qt5::Widgets)
 target_link_libraries(Extra\ Window Qt5::Widgets)
 target_link_libraries(Google\ Translate winhttp Qt5::Widgets)
 target_link_libraries(Google\ Cloud\ Translate winhttp Qt5::Widgets)

extensions/bingtranslate.cpp

@@ -79,7 +79,7 @@ QStringList languages
 	"Yucatec Maya: yua"
 };
 
-std::pair<bool, std::wstring> Translate(const std::wstring& text)
+std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo)
 {
 	if (HttpRequest httpRequest{
 		L"Mozilla/5.0 Textractor",
@@ -88,7 +88,7 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text)
 		FormatString(L"/ttranslatev3?fromLang=auto-detect&to=%s&text=%s", translateTo->c_str(), Escape(text)).c_str()
 	})
 		// Response formatted as JSON: translation starts with text":" and ends with ","to
-		if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"text\":\"(.+)\",\"t"))) return { true, results[1] };
+		if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"text\":\"(.+?)\",\""))) return { true, results[1] };
 		else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) };
 	else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) };
 }

extensions/deepltranslate.cpp

@@ -0,0 +1,113 @@
+#include "qtcommon.h"
+#include "extension.h"
+#include "network.h"
+#include <random>
+
+extern const wchar_t* TRANSLATION_ERROR;
+extern const char* USE_PREV_SENTENCE_CONTEXT;
+
+extern QSettings settings;
+extern QFormLayout* display;
+extern Synchronized<std::wstring> translateTo;
+
+const char* TRANSLATION_PROVIDER = "DeepL Translate";
+QStringList languages
+{
+	"Chinese: ZH",
+	"Dutch: NL",
+	"English: EN",
+	"French: FR",
+	"German: DE",
+	"Italian: IT",
+	"Japanese: JA",
+	"Polish: PL",
+	"Portuguese: PT",
+	"Russian: RU",
+	"Spanish: ES",
+};
+
+const wchar_t* accept[] = { L"*/*", nullptr };
+
+Synchronized<std::wstring> LMTBID;
+
+bool useContext = true;
+Synchronized<std::unordered_map<int64_t, std::wstring>> context;
+
+BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
+{
+	switch (ul_reason_for_call)
+	{
+	case DLL_PROCESS_ATTACH:
+	{
+		auto checkbox = new QCheckBox;
+		checkbox->setChecked(useContext);
+		display->addRow(USE_PREV_SENTENCE_CONTEXT, checkbox);
+		QObject::connect(checkbox, &QCheckBox::clicked, [](bool checked) { settings.setValue(USE_PREV_SENTENCE_CONTEXT, useContext = checked); });
+	}
+	break;
+	case DLL_PROCESS_DETACH:
+	{
+	}
+	break;
+	}
+	return TRUE;
+}
+
+std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo sentenceInfo)
+{
+	// the following code was reverse engineered from the DeepL website; it's as close as I could make it but I'm not sure what parts of this could be removed and still have it work
+	int64_t r = _time64(nullptr), n = std::count(text.begin(), text.end(), L'i') + 1LL;
+	static std::atomic<int> id = 10000 * std::uniform_int_distribution(0, 9999)(std::mt19937(std::random_device()()));
+	std::string jsonText;
+	for (auto ch : WideStringToString(text))
+		if (ch == '"') jsonText += "\\\"";
+		else jsonText += ch;
+	// user_preferred_langs? what should preferred_num_beans and priority be? does timestamp do anything? other translation quality options?
+	auto body = FormatString(R"(
+{
+	"id": %d,
+	"jsonrpc": "2.0",
+	"method": "LMT_handle_jobs",
+	"params": {
+		"priority": -1,
+		"timestamp": %lld,
+		"lang": {
+			"source_lang_user_selected": "auto",
+			"target_lang": "%s"
+		},
+		"jobs": [{
+			"kind": "default",
+			"preferred_num_beams": 4,
+			"quality": "fast",
+			"raw_en_context_after": [],
+			"raw_en_sentence": "%s",
+			"raw_en_context_before": [%s]
+		}]
+	}
+}
+	)", ++id, r + (n - r % n), WideStringToString(translateTo->c_str()), jsonText, useContext ? WideStringToString(context->operator[](sentenceInfo["text number"])) : "");
+	context->insert_or_assign(sentenceInfo["text number"], L'"' + text + L'"');
+	std::wstring headers = L"Host: www2.deepl.com\r\nAccept-Language: en-US,en;q=0.5\r\nContent-type: text/plain\r\nOrigin: https://www.deepl.com\r\nTE: Trailers"
+		+ LMTBID.Acquire().contents;
+	if (HttpRequest httpRequest{
+		L"Mozilla/5.0 Textractor",
+		L"www2.deepl.com",
+		L"POST",
+		L"/jsonrpc",
+		WINHTTP_FLAG_SECURE,
+		NULL,
+		L"https://www.deepl.com/translator",
+		accept,
+		headers.c_str(),
+		body.data(),
+		body.size()
+	})
+	{
+		auto LMTBID = httpRequest.headers.find(L"LMTBID="), end = httpRequest.headers.find(L';', LMTBID); // not sure if this cookie does anything
+		if (LMTBID != std::wstring::npos && end != std::wstring::npos) ::LMTBID->assign(L"\r\nCookie: " + httpRequest.headers.substr(LMTBID, end - LMTBID));
+		// Response formatted as JSON: translation starts with preprocessed_sentence":" and ends with ","
+		if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"postprocessed_sentence\":\"(.+?)\",\""))) return { true, results[1] };
+		else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) };
+	}
+	else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) };
+}

extensions/extrawindow.cpp

@@ -270,8 +270,9 @@ private:
 		dictionaryWindow.ui.display->setFixedWidth(ui.display->width() * 3 / 4);
 		dictionaryWindow.setTerm(sentence.mid(i));
 		int left = i == 0 ? 0 : textPositionMap[i - 1].x(), right = textPositionMap[i].x(),
-			x = textPositionMap[i].x() > ui.display->width() / 2 ? -dictionaryWindow.width() + (right * 3 + left) / 4 : (left * 3 + right) / 4;
+			x = textPositionMap[i].x() > ui.display->width() / 2 ? -dictionaryWindow.width() + (right * 3 + left) / 4 : (left * 3 + right) / 4, y = 0;
-		dictionaryWindow.move(ui.display->mapToGlobal(QPoint(x, textPositionMap[i].y())));
+		for (auto point : textPositionMap) if (point.y() > y && point.y() < textPositionMap[i].y()) y = point.y();
+		dictionaryWindow.move(ui.display->mapToGlobal(QPoint(x, y - dictionaryWindow.height() + 1)));
 	}
 
 	bool eventFilter(QObject*, QEvent* event) override
@@ -397,7 +398,7 @@ private:
 			for (term = term.left(100); !term.isEmpty(); term.chop(1))
 				for (const auto& [rootTerm, definition, inflections] : LookupDefinitions(term, foundDefinitions))
 					definitions.push_back(
-						QStringLiteral("<h3>%1 (%5/%6)</h3><small>%2%3</small><p>%4</p>").arg(
+						QStringLiteral("<h3>%1 (%5/%6)</h3><small>%2%3</small>%4").arg(
 							term.split("<<")[0].toHtmlEscaped(),
 							rootTerm.split("<<")[0].toHtmlEscaped(),
 							inflections.join(""),
@@ -455,7 +456,9 @@ private:
 			int scroll = event->angleDelta().y();
 			if (scroll > 0 && definitionIndex > 0) definitionIndex -= 1;
 			if (scroll < 0 && definitionIndex + 1 < definitions.size()) definitionIndex += 1;
+			int oldHeight = height();
 			ShowDefinition();
+			move(x(), y() + oldHeight - height());
 		}
 
 		struct Inflection

extensions/googlecloudtranslate.cpp

@@ -101,7 +101,7 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved
 	return TRUE;
 }
 
-std::pair<bool, std::wstring> Translate(const std::wstring& text)
+std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo)
 {
 
 	if (HttpRequest httpRequest{

extensions/googletranslate.cpp

@@ -152,7 +152,7 @@ bool IsHash(const std::wstring& result)
 	return result.size() == 32 && std::all_of(result.begin(), result.end(), [](char ch) { return (ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'z'); });
 }
 
-std::pair<bool, std::wstring> Translate(const std::wstring& text)
+std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo)
 {
 	if (!TKK)
 		if (HttpRequest httpRequest{ L"Mozilla/5.0 Textractor", L"translate.google.com", L"GET", L"/" })

extensions/network.cpp

@@ -22,6 +22,10 @@ HttpRequest::HttpRequest(
 				if (WinHttpSendRequest(request, headers, -1UL, body, bodyLength, bodyLength, NULL))
 				{
 					WinHttpReceiveResponse(request, NULL);
+					DWORD size = 0;
+					WinHttpQueryHeaders(request, WINHTTP_QUERY_RAW_HEADERS_CRLF, WINHTTP_HEADER_NAME_BY_INDEX, NULL, &size, WINHTTP_NO_HEADER_INDEX);
+					this->headers.resize(size);
+					WinHttpQueryHeaders(request, WINHTTP_QUERY_RAW_HEADERS_CRLF, WINHTTP_HEADER_NAME_BY_INDEX, this->headers.data(), &size, WINHTTP_NO_HEADER_INDEX);
 					std::string data;
 					DWORD availableSize, downloadedSize;
 					do

extensions/network.h

@@ -23,6 +23,7 @@ struct HttpRequest
 	operator bool() { return errorCode == ERROR_SUCCESS; }
 
 	std::wstring response;
+	std::wstring headers;
 	InternetHandle connection = NULL;
 	InternetHandle request = NULL;
 	DWORD errorCode = ERROR_SUCCESS;

extensions/translatewrapper.cpp

@@ -8,11 +8,16 @@
 
 extern const char* NATIVE_LANGUAGE;
 extern const char* TRANSLATE_TO;
+extern const char* RATE_LIMIT_ALL_THREADS;
+extern const char* RATE_LIMIT_SELECTED_THREAD;
+extern const char* USE_TRANS_CACHE;
+extern const char* RATE_LIMIT_TOKEN_COUNT;
+extern const char* RATE_LIMIT_TOKEN_RESTORE_DELAY;
 extern const wchar_t* TOO_MANY_TRANS_REQUESTS;
 
 extern const char* TRANSLATION_PROVIDER;
 extern QStringList languages;
-std::pair<bool, std::wstring> Translate(const std::wstring& text);
+std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo sentenceInfo);
 
 const char* LANGUAGE = u8"Language";
 const std::string TRANSLATION_CACHE_FILE = FormatString("%s Cache.txt", TRANSLATION_PROVIDER);
@@ -21,6 +26,8 @@ QFormLayout* display;
 QSettings settings = openSettings();
 Synchronized<std::wstring> translateTo = L"en";
 
+bool rateLimitAll = true, rateLimitSelected = false, useCache = true;
+int tokenCount = 30, tokenRestoreDelay = 60000;
 Synchronized<std::map<std::wstring, std::wstring>> translationCache;
 int savedSize;
 
@@ -51,8 +58,32 @@ public:
 		if (language < 0) language = languageBox->findText("English", Qt::MatchStartsWith);
 		languageBox->setCurrentIndex(language);
 		saveLanguage(languageBox->currentText());
-		connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage);
 		display->addRow(TRANSLATE_TO, languageBox);
+		connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage);
+		for (auto [value, label] : Array<bool&, const char*>{
+			{ rateLimitAll, RATE_LIMIT_ALL_THREADS },
+			{ rateLimitSelected, RATE_LIMIT_SELECTED_THREAD },
+			{ useCache, USE_TRANS_CACHE },
+		})
+		{
+			value = settings.value(label, value).toBool();
+			auto checkBox = new QCheckBox(this);
+			checkBox->setChecked(value);
+			display->addRow(label, checkBox);
+			connect(checkBox, &QCheckBox::clicked, [label, &value](bool checked) { settings.setValue(label, value = checked); });
+		}
+		for (auto [value, label] : Array<int&, const char*>{
+			{ tokenCount, RATE_LIMIT_TOKEN_COUNT },
+			{ tokenRestoreDelay, RATE_LIMIT_TOKEN_RESTORE_DELAY },
+		})
+		{
+			value = settings.value(label, value).toInt();
+			auto spinBox = new QSpinBox(this);
+			spinBox->setRange(0, INT_MAX);
+			spinBox->setValue(value);
+			display->addRow(label, spinBox);
+			connect(spinBox, qOverload<int>(&QSpinBox::valueChanged), [label, &value](int newValue) { settings.setValue(label, value = newValue); });
+		}
 
 		setWindowTitle(TRANSLATION_PROVIDER);
 		QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection);
@@ -92,25 +123,25 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
 			auto tokens = this->tokens.Acquire();
 			tokens->push_back(GetTickCount());
 			if (tokens->size() > tokenCount * 5) tokens->erase(tokens->begin(), tokens->begin() + tokenCount * 3);
-			tokens->erase(std::remove_if(tokens->begin(), tokens->end(), [this](DWORD token) { return GetTickCount() - token > delay; }), tokens->end());
+			tokens->erase(std::remove_if(tokens->begin(), tokens->end(), [](DWORD token) { return GetTickCount() - token > tokenRestoreDelay; }), tokens->end());
 			return tokens->size() < tokenCount;
 		}
 
 	private:
-		const int tokenCount = 30, delay = 60 * 1000;
 		Synchronized<std::vector<DWORD>> tokens;
 	} rateLimiter;
 
 	bool cache = false;
 	std::wstring translation;
+	if (useCache)
 	{
 		auto translationCache = ::translationCache.Acquire();
-		auto translationLocation = translationCache->find(sentence);
+		if (auto it = translationCache->find(sentence); it != translationCache->end()) translation = it->second + L"\x200b";
-		if (translationLocation != translationCache->end()) translation = translationLocation->second;
-		else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS;
-		else std::tie(cache, translation) = Translate(sentence);
-		if (cache && sentenceInfo["current select"]) translationCache->try_emplace(translationLocation, sentence, translation);
 	}
+	if (translation.empty())
+		if (rateLimiter.Request() || !rateLimitAll || (!rateLimitSelected && sentenceInfo["current select"])) std::tie(cache, translation) = Translate(sentence, sentenceInfo);
+		else translation = TOO_MANY_TRANS_REQUESTS;
+	if (cache) translationCache->try_emplace(sentence, translation);
 	if (cache && translationCache->size() > savedSize + 50) SaveCache();
 
 	Unescape(translation);

text.cpp

@@ -130,8 +130,14 @@ const char* READ_ERROR = u8"Textractor: Reader ERROR (likely an incorrect R-code
 const char* HIJACK_ERROR = u8"Textractor: Hijack ERROR";
 const char* COULD_NOT_FIND = u8"Textractor: could not find text";
 const char* TRANSLATE_TO = u8"Translate to";
-const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Too many translation requests: refuse to make more";
+const char* RATE_LIMIT_ALL_THREADS = u8"Rate limit all text threads";
+const char* RATE_LIMIT_SELECTED_THREAD = u8"Rate limit currently selected text thread";
+const char* USE_TRANS_CACHE = u8"Use translation cache";
+const char* RATE_LIMIT_TOKEN_COUNT = u8"Rate limiter token count";
+const char* RATE_LIMIT_TOKEN_RESTORE_DELAY = u8"Rate limiter token restore delay (ms)";
+const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Rate limit exceeded: refuse to make more translation requests";
 const wchar_t* TRANSLATION_ERROR = L"Error while translating";
+const char* USE_PREV_SENTENCE_CONTEXT = u8"Use previous sentence as context";
 const char* API_KEY = u8"API key";
 const char* EXTRA_WINDOW_INFO = u8R"(Right click to change settings
 Click and drag on window edges to move, or the bottom right corner to resize)";