chenx221/Textractor_test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add settings to translation dialog and implement deepl

Browse Source
This commit is contained in:
Akash Mozumdar 2020-03-26 05:41:21 -06:00
parent 19f35f743c
commit 87c056a5b3
12 changed files with 181 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
GUI/mainwindow.cpp
View File

@ -297,8 +297,8 @@ namespace
QDialog dialog(This, Qt::WindowCloseButtonHint);
QFormLayout layout(&dialog);
QCheckBox cjkCheckbox(&dialog);
layout.addRow(SEARCH_CJK, &cjkCheckbox);
QCheckBox cjkCheckBox(&dialog);
layout.addRow(SEARCH_CJK, &cjkCheckBox);
QDialogButtonBox confirm(QDialogButtonBox::Ok | QDialogButtonBox::Help | QDialogButtonBox::Retry, &dialog);
layout.addRow(&confirm);
confirm.button(QDialogButtonBox::Ok)->setText(START_HOOK_SEARCH);
@ -388,7 +388,7 @@ namespace
else
{
sp.length = 0; // use default
filter.setPattern(cjkCheckbox.isChecked() ? "[\\x{3000}-\\x{a000}]{4,}" : "[\\x{0020}-\\x{1000}]{4,}");
filter.setPattern(cjkCheckBox.isChecked() ? "[\\x{3000}-\\x{a000}]{4,}" : "[\\x{0020}-\\x{1000}]{4,}");
}
filter.optimize();

1
deploy.ps1
View File

@ -37,6 +37,7 @@ foreach ($language in @{
"texthook.dll",
"Bing Translate.dll",
"Copy to Clipboard.dll",
"DeepL Translate.dll",
"Extra Newlines.dll",
"Extra Window.dll",
"Google Translate.dll",

2
extensions/CMakeLists.txt
View File

@ -6,6 +6,7 @@ cmake_policy(SET CMP0037 OLD)
add_library(Bing\ Translate MODULE bingtranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp)
add_library(Copy\ to\ Clipboard MODULE copyclipboard.cpp extensionimpl.cpp)
add_library(DeepL\ Translate MODULE deepltranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp)
add_library(Extra\ Newlines MODULE extranewlines.cpp extensionimpl.cpp)
add_library(Extra\ Window MODULE extrawindow.cpp extensionimpl.cpp)
add_library(Google\ Translate MODULE googletranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp)
@ -20,6 +21,7 @@ add_library(Replacer MODULE replacer.cpp extensionimpl.cpp)
add_library(Thread\ Linker MODULE threadlinker.cpp extensionimpl.cpp)
target_link_libraries(Bing\ Translate winhttp Qt5::Widgets)
target_link_libraries(DeepL\ Translate winhttp Qt5::Widgets)
target_link_libraries(Extra\ Window Qt5::Widgets)
target_link_libraries(Google\ Translate winhttp Qt5::Widgets)
target_link_libraries(Google\ Cloud\ Translate winhttp Qt5::Widgets)

4
extensions/bingtranslate.cpp
View File

@ -79,7 +79,7 @@ QStringList languages
"Yucatec Maya: yua"
};
std::pair<bool, std::wstring> Translate(const std::wstring& text)
std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo)
{
if (HttpRequest httpRequest{
L"Mozilla/5.0 Textractor",
@ -88,7 +88,7 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text)
FormatString(L"/ttranslatev3?fromLang=auto-detect&to=%s&text=%s", translateTo->c_str(), Escape(text)).c_str()
})
// Response formatted as JSON: translation starts with text":" and ends with ","to
if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"text\":\"(.+)\",\"t"))) return { true, results[1] };
if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"text\":\"(.+?)\",\""))) return { true, results[1] };
else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) };
else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) };
}

113
extensions/deepltranslate.cpp Normal file
View File

@ -0,0 +1,113 @@
#include "qtcommon.h"
#include "extension.h"
#include "network.h"
#include <random>
extern const wchar_t* TRANSLATION_ERROR;
extern const char* USE_PREV_SENTENCE_CONTEXT;
extern QSettings settings;
extern QFormLayout* display;
extern Synchronized<std::wstring> translateTo;
const char* TRANSLATION_PROVIDER = "DeepL Translate";
QStringList languages
{
"Chinese: ZH",
"Dutch: NL",
"English: EN",
"French: FR",
"German: DE",
"Italian: IT",
"Japanese: JA",
"Polish: PL",
"Portuguese: PT",
"Russian: RU",
"Spanish: ES",
};
const wchar_t* accept[] = { L"*/*", nullptr };
Synchronized<std::wstring> LMTBID;
bool useContext = true;
Synchronized<std::unordered_map<int64_t, std::wstring>> context;
BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
{
auto checkbox = new QCheckBox;
checkbox->setChecked(useContext);
display->addRow(USE_PREV_SENTENCE_CONTEXT, checkbox);
QObject::connect(checkbox, &QCheckBox::clicked, [](bool checked) { settings.setValue(USE_PREV_SENTENCE_CONTEXT, useContext = checked); });
}
break;
case DLL_PROCESS_DETACH:
{
}
break;
}
return TRUE;
}
std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo sentenceInfo)
{
// the following code was reverse engineered from the DeepL website; it's as close as I could make it but I'm not sure what parts of this could be removed and still have it work
int64_t r = _time64(nullptr), n = std::count(text.begin(), text.end(), L'i') + 1LL;
static std::atomic<int> id = 10000 * std::uniform_int_distribution(0, 9999)(std::mt19937(std::random_device()()));
std::string jsonText;
for (auto ch : WideStringToString(text))
if (ch == '"') jsonText += "\\\"";
else jsonText += ch;
// user_preferred_langs? what should preferred_num_beans and priority be? does timestamp do anything? other translation quality options?
auto body = FormatString(R"(
{
"id": %d,
"jsonrpc": "2.0",
"method": "LMT_handle_jobs",
"params": {
"priority": -1,
"timestamp": %lld,
"lang": {
"source_lang_user_selected": "auto",
"target_lang": "%s"
},
"jobs": [{
"kind": "default",
"preferred_num_beams": 4,
"quality": "fast",
"raw_en_context_after": [],
"raw_en_sentence": "%s",
"raw_en_context_before": [%s]
}]
}
}
)", ++id, r + (n - r % n), WideStringToString(translateTo->c_str()), jsonText, useContext ? WideStringToString(context->operator[](sentenceInfo["text number"])) : "");
context->insert_or_assign(sentenceInfo["text number"], L'"' + text + L'"');
std::wstring headers = L"Host: www2.deepl.com\r\nAccept-Language: en-US,en;q=0.5\r\nContent-type: text/plain\r\nOrigin: https://www.deepl.com\r\nTE: Trailers"
+ LMTBID.Acquire().contents;
if (HttpRequest httpRequest{
L"Mozilla/5.0 Textractor",
L"www2.deepl.com",
L"POST",
L"/jsonrpc",
WINHTTP_FLAG_SECURE,
NULL,
L"https://www.deepl.com/translator",
accept,
headers.c_str(),
body.data(),
body.size()
})
{
auto LMTBID = httpRequest.headers.find(L"LMTBID="), end = httpRequest.headers.find(L';', LMTBID); // not sure if this cookie does anything
if (LMTBID != std::wstring::npos && end != std::wstring::npos) ::LMTBID->assign(L"\r\nCookie: " + httpRequest.headers.substr(LMTBID, end - LMTBID));
// Response formatted as JSON: translation starts with preprocessed_sentence":" and ends with ","
if (std::wsmatch results; std::regex_search(httpRequest.response, results, std::wregex(L"postprocessed_sentence\":\"(.+?)\",\""))) return { true, results[1] };
else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) };
}
else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) };
}

9
extensions/extrawindow.cpp
View File

@ -270,8 +270,9 @@ private:
dictionaryWindow.ui.display->setFixedWidth(ui.display->width() * 3 / 4);
dictionaryWindow.setTerm(sentence.mid(i));
int left = i == 0 ? 0 : textPositionMap[i - 1].x(), right = textPositionMap[i].x(),
x = textPositionMap[i].x() > ui.display->width() / 2 ? -dictionaryWindow.width() + (right * 3 + left) / 4 : (left * 3 + right) / 4;
dictionaryWindow.move(ui.display->mapToGlobal(QPoint(x, textPositionMap[i].y())));
x = textPositionMap[i].x() > ui.display->width() / 2 ? -dictionaryWindow.width() + (right * 3 + left) / 4 : (left * 3 + right) / 4, y = 0;
for (auto point : textPositionMap) if (point.y() > y && point.y() < textPositionMap[i].y()) y = point.y();
dictionaryWindow.move(ui.display->mapToGlobal(QPoint(x, y - dictionaryWindow.height() + 1)));
}
bool eventFilter(QObject*, QEvent* event) override
@ -397,7 +398,7 @@ private:
for (term = term.left(100); !term.isEmpty(); term.chop(1))
for (const auto& [rootTerm, definition, inflections] : LookupDefinitions(term, foundDefinitions))
definitions.push_back(
QStringLiteral("<h3>%1 (%5/%6)</h3><small>%2%3</small><p>%4</p>").arg(
QStringLiteral("<h3>%1 (%5/%6)</h3><small>%2%3</small>%4").arg(
term.split("<<")[0].toHtmlEscaped(),
rootTerm.split("<<")[0].toHtmlEscaped(),
inflections.join(""),
@ -455,7 +456,9 @@ private:
int scroll = event->angleDelta().y();
if (scroll > 0 && definitionIndex > 0) definitionIndex -= 1;
if (scroll < 0 && definitionIndex + 1 < definitions.size()) definitionIndex += 1;
int oldHeight = height();
ShowDefinition();
move(x(), y() + oldHeight - height());
}
struct Inflection

2
extensions/googlecloudtranslate.cpp
View File

@ -101,7 +101,7 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved
return TRUE;
}
std::pair<bool, std::wstring> Translate(const std::wstring& text)
std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo)
{
if (HttpRequest httpRequest{

2
extensions/googletranslate.cpp
View File

@ -152,7 +152,7 @@ bool IsHash(const std::wstring& result)
return result.size() == 32 && std::all_of(result.begin(), result.end(), [](char ch) { return (ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'z'); });
}
std::pair<bool, std::wstring> Translate(const std::wstring& text)
std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo)
{
if (!TKK)
if (HttpRequest httpRequest{ L"Mozilla/5.0 Textractor", L"translate.google.com", L"GET", L"/" })

4
extensions/network.cpp
View File

@ -22,6 +22,10 @@ HttpRequest::HttpRequest(
if (WinHttpSendRequest(request, headers, -1UL, body, bodyLength, bodyLength, NULL))
{
WinHttpReceiveResponse(request, NULL);
DWORD size = 0;
WinHttpQueryHeaders(request, WINHTTP_QUERY_RAW_HEADERS_CRLF, WINHTTP_HEADER_NAME_BY_INDEX, NULL, &size, WINHTTP_NO_HEADER_INDEX);
this->headers.resize(size);
WinHttpQueryHeaders(request, WINHTTP_QUERY_RAW_HEADERS_CRLF, WINHTTP_HEADER_NAME_BY_INDEX, this->headers.data(), &size, WINHTTP_NO_HEADER_INDEX);
std::string data;
DWORD availableSize, downloadedSize;
do

1
extensions/network.h
View File

@ -23,6 +23,7 @@ struct HttpRequest
operator bool() { return errorCode == ERROR_SUCCESS; }
std::wstring response;
std::wstring headers;
InternetHandle connection = NULL;
InternetHandle request = NULL;
DWORD errorCode = ERROR_SUCCESS;

49
extensions/translatewrapper.cpp
View File

@ -8,11 +8,16 @@
extern const char* NATIVE_LANGUAGE;
extern const char* TRANSLATE_TO;
extern const char* RATE_LIMIT_ALL_THREADS;
extern const char* RATE_LIMIT_SELECTED_THREAD;
extern const char* USE_TRANS_CACHE;
extern const char* RATE_LIMIT_TOKEN_COUNT;
extern const char* RATE_LIMIT_TOKEN_RESTORE_DELAY;
extern const wchar_t* TOO_MANY_TRANS_REQUESTS;
extern const char* TRANSLATION_PROVIDER;
extern QStringList languages;
std::pair<bool, std::wstring> Translate(const std::wstring& text);
std::pair<bool, std::wstring> Translate(const std::wstring& text, SentenceInfo sentenceInfo);
const char* LANGUAGE = u8"Language";
const std::string TRANSLATION_CACHE_FILE = FormatString("%s Cache.txt", TRANSLATION_PROVIDER);
@ -21,6 +26,8 @@ QFormLayout* display;
QSettings settings = openSettings();
Synchronized<std::wstring> translateTo = L"en";
bool rateLimitAll = true, rateLimitSelected = false, useCache = true;
int tokenCount = 30, tokenRestoreDelay = 60000;
Synchronized<std::map<std::wstring, std::wstring>> translationCache;
int savedSize;
@ -51,8 +58,32 @@ public:
if (language < 0) language = languageBox->findText("English", Qt::MatchStartsWith);
languageBox->setCurrentIndex(language);
saveLanguage(languageBox->currentText());
connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage);
display->addRow(TRANSLATE_TO, languageBox);
connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage);
for (auto [value, label] : Array<bool&, const char*>{
{ rateLimitAll, RATE_LIMIT_ALL_THREADS },
{ rateLimitSelected, RATE_LIMIT_SELECTED_THREAD },
{ useCache, USE_TRANS_CACHE },
})
{
value = settings.value(label, value).toBool();
auto checkBox = new QCheckBox(this);
checkBox->setChecked(value);
display->addRow(label, checkBox);
connect(checkBox, &QCheckBox::clicked, [label, &value](bool checked) { settings.setValue(label, value = checked); });
}
for (auto [value, label] : Array<int&, const char*>{
{ tokenCount, RATE_LIMIT_TOKEN_COUNT },
{ tokenRestoreDelay, RATE_LIMIT_TOKEN_RESTORE_DELAY },
})
{
value = settings.value(label, value).toInt();
auto spinBox = new QSpinBox(this);
spinBox->setRange(0, INT_MAX);
spinBox->setValue(value);
display->addRow(label, spinBox);
connect(spinBox, qOverload<int>(&QSpinBox::valueChanged), [label, &value](int newValue) { settings.setValue(label, value = newValue); });
}
setWindowTitle(TRANSLATION_PROVIDER);
QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection);
@ -92,25 +123,25 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
auto tokens = this->tokens.Acquire();
tokens->push_back(GetTickCount());
if (tokens->size() > tokenCount * 5) tokens->erase(tokens->begin(), tokens->begin() + tokenCount * 3);
tokens->erase(std::remove_if(tokens->begin(), tokens->end(), [this](DWORD token) { return GetTickCount() - token > delay; }), tokens->end());
tokens->erase(std::remove_if(tokens->begin(), tokens->end(), [](DWORD token) { return GetTickCount() - token > tokenRestoreDelay; }), tokens->end());
return tokens->size() < tokenCount;
}
private:
const int tokenCount = 30, delay = 60 * 1000;
Synchronized<std::vector<DWORD>> tokens;
} rateLimiter;
bool cache = false;
std::wstring translation;
if (useCache)
{
auto translationCache = ::translationCache.Acquire();
auto translationLocation = translationCache->find(sentence);
if (translationLocation != translationCache->end()) translation = translationLocation->second;
else if (!(rateLimiter.Request() || sentenceInfo["current select"])) translation = TOO_MANY_TRANS_REQUESTS;
else std::tie(cache, translation) = Translate(sentence);
if (cache && sentenceInfo["current select"]) translationCache->try_emplace(translationLocation, sentence, translation);
if (auto it = translationCache->find(sentence); it != translationCache->end()) translation = it->second + L"\x200b";
}
if (translation.empty())
if (rateLimiter.Request() || !rateLimitAll || (!rateLimitSelected && sentenceInfo["current select"])) std::tie(cache, translation) = Translate(sentence, sentenceInfo);
else translation = TOO_MANY_TRANS_REQUESTS;
if (cache) translationCache->try_emplace(sentence, translation);
if (cache && translationCache->size() > savedSize + 50) SaveCache();
Unescape(translation);

8
text.cpp
View File

@ -130,8 +130,14 @@ const char* READ_ERROR = u8"Textractor: Reader ERROR (likely an incorrect R-code
const char* HIJACK_ERROR = u8"Textractor: Hijack ERROR";
const char* COULD_NOT_FIND = u8"Textractor: could not find text";
const char* TRANSLATE_TO = u8"Translate to";
const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Too many translation requests: refuse to make more";
const char* RATE_LIMIT_ALL_THREADS = u8"Rate limit all text threads";
const char* RATE_LIMIT_SELECTED_THREAD = u8"Rate limit currently selected text thread";
const char* USE_TRANS_CACHE = u8"Use translation cache";
const char* RATE_LIMIT_TOKEN_COUNT = u8"Rate limiter token count";
const char* RATE_LIMIT_TOKEN_RESTORE_DELAY = u8"Rate limiter token restore delay (ms)";
const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Rate limit exceeded: refuse to make more translation requests";
const wchar_t* TRANSLATION_ERROR = L"Error while translating";
const char* USE_PREV_SENTENCE_CONTEXT = u8"Use previous sentence as context";
const char* API_KEY = u8"API key";
const char* EXTRA_WINDOW_INFO = u8R"(Right click to change settings
Click and drag on window edges to move, or the bottom right corner to resize)";