From 6b54ec073303503cda873aabc4e2c956bcf83b50 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Wed, 14 Oct 2020 03:37:03 +0300 Subject: [PATCH 01/14] add devtools api and new deepl extension Added the DevTools API and the wrapper with the use of the QtWebSockets library. Added a new DeepL extension which uses the DevTools API. --- extensions/CMakeLists.txt | 16 +- extensions/devtools.cpp | 278 ++++++++++++++++++++++++++ extensions/devtools.h | 49 +++++ extensions/devtoolsdeepltranslate.cpp | 195 ++++++++++++++++++ extensions/devtoolswrapper.cpp | 194 ++++++++++++++++++ extensions/network.cpp | 5 +- extensions/network.h | 3 +- 7 files changed, 735 insertions(+), 5 deletions(-) create mode 100644 extensions/devtools.cpp create mode 100644 extensions/devtools.h create mode 100644 extensions/devtoolsdeepltranslate.cpp create mode 100644 extensions/devtoolswrapper.cpp diff --git a/extensions/CMakeLists.txt b/extensions/CMakeLists.txt index 751ec15..e7367d0 100644 --- a/extensions/CMakeLists.txt +++ b/extensions/CMakeLists.txt @@ -1,7 +1,7 @@ include(QtUtils) msvc_registry_search() -find_qt5(Core Widgets) - +find_qt5(Core Widgets WebSockets) +set(CMAKE_AUTOMOC ON) cmake_policy(SET CMP0037 OLD) add_library(Bing\ Translate MODULE bingtranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) @@ -10,6 +10,7 @@ add_library(DeepL\ Translate MODULE deepltranslate.cpp translatewrapper.cpp netw add_library(Extra\ Newlines MODULE extranewlines.cpp extensionimpl.cpp) add_library(Extra\ Window MODULE extrawindow.cpp extensionimpl.cpp) add_library(Google\ Translate MODULE googletranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) +add_library(DevTools\ DeepL\ Translate MODULE devtoolsdeepltranslate.cpp devtoolswrapper.cpp devtools.cpp network.cpp extensionimpl.cpp) add_library(Lua MODULE lua.cpp extensionimpl.cpp) add_library(Regex\ Filter MODULE regexfilter.cpp extensionimpl.cpp) add_library(Remove\ Repeated\ Characters MODULE removerepeatchar.cpp extensionimpl.cpp) @@ -25,6 +26,7 @@ target_precompile_headers(DeepL\ Translate REUSE_FROM pch) target_precompile_headers(Extra\ Newlines REUSE_FROM pch) target_precompile_headers(Extra\ Window REUSE_FROM pch) target_precompile_headers(Google\ Translate REUSE_FROM pch) +target_precompile_headers(DevTools\ DeepL\ Translate REUSE_FROM pch) target_precompile_headers(Lua REUSE_FROM pch) target_precompile_headers(Regex\ Filter REUSE_FROM pch) target_precompile_headers(Remove\ Repeated\ Characters REUSE_FROM pch) @@ -38,6 +40,16 @@ target_link_libraries(Bing\ Translate winhttp Qt5::Widgets) target_link_libraries(DeepL\ Translate winhttp Qt5::Widgets) target_link_libraries(Extra\ Window Qt5::Widgets) target_link_libraries(Google\ Translate winhttp Qt5::Widgets) +target_link_libraries(DevTools\ DeepL\ Translate winhttp Qt5::Widgets Qt5::WebSockets) target_link_libraries(Lua lua53 Qt5::Widgets) target_link_libraries(Regex\ Filter Qt5::Widgets) target_link_libraries(Thread\ Linker Qt5::Widgets) + +if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll) + add_custom_command(TARGET DevTools\ DeepL\ Translate + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" + COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin + COMMAND Qt5::windeployqt --dir $ "$/DevTools\ DeepL\ Translate.dll" + ) +endif() \ No newline at end of file diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp new file mode 100644 index 0000000..f81828e --- /dev/null +++ b/extensions/devtools.cpp @@ -0,0 +1,278 @@ +#include "devtools.h" + +DevTools::DevTools(QObject* parent) : + QObject(parent), + idcounter(0), + pagenavigated(false), + translateready(false), + status("Stopped"), + session(1) +{ +} + +void DevTools::startDevTools(QString path, bool headless, int port) +{ + if (startChrome(path, headless, port)) + { + QString webSocketDebuggerUrl; + if (GetwebSocketDebuggerUrl(webSocketDebuggerUrl, port)) + { + connect(&webSocket, &QWebSocket::stateChanged, this, &DevTools::stateChanged); + connect(&webSocket, &QWebSocket::textMessageReceived, this, &DevTools::onTextMessageReceived); + webSocket.open(webSocketDebuggerUrl); + session += 1; + } + else + { + status = "Failed to find chrome debug port!"; + emit statusChanged(status); + } + + } + else + { + status = "Failed to start chrome!"; + emit statusChanged(status); + } +} + +int DevTools::getSession() +{ + return session; +} + +QString DevTools::getStatus() +{ + return status; +} + +DevTools::~DevTools() +{ + closeDevTools(); +} + +bool DevTools::startChrome(QString path, bool headless, int port) +{ + if (!std::filesystem::exists(path.toStdWString())) + return false; + DWORD exitCode = 0; + if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) && (exitCode == STILL_ACTIVE)) + return false; + QString args = "--proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=" + + QString::fromStdWString(std::filesystem::current_path()) + + "\\devtoolscache --remote-debugging-port=" + + QString::number(port); + if (headless) + args += " --headless"; + STARTUPINFOW dummy = { sizeof(dummy) }; + if (!CreateProcessW(NULL, (wchar_t*)(path + " " + args).utf16(), nullptr, nullptr, + FALSE, 0, nullptr, nullptr, &dummy, &processInfo)) + return false; + else + return true; +} + +bool DevTools::GetwebSocketDebuggerUrl(QString& url, int port) +{ + url.clear(); + if (HttpRequest httpRequest{ + L"Mozilla/5.0 Textractor", + L"127.0.0.1", + L"POST", + FormatString(L"/json/list").c_str(), + "", + NULL, + NULL, + WINHTTP_FLAG_ESCAPE_DISABLE, + NULL, + NULL, + DWORD(port) + }) + { + QString qtString = QString::fromStdWString(httpRequest.response); + QJsonDocument doc = QJsonDocument::fromJson(qtString.toUtf8()); + QJsonArray rootObject = doc.array(); + + for (const auto obj : rootObject) + if (obj.toObject().value("type") == "page") + { + url.append(obj.toObject().value("webSocketDebuggerUrl").toString()); + break; + } + if (!url.isEmpty()) + return true; + else + return false; + } + else + return false; + +} + +void DevTools::stateChanged(QAbstractSocket::SocketState state) +{ + QMetaEnum metaenum = QMetaEnum::fromType(); + status = metaenum.valueToKey(state); + emit statusChanged(status); +} + +void DevTools::setNavigated(bool value) +{ + mutex.lock(); + pagenavigated = value; + mutex.unlock(); +} + +bool DevTools::getNavigated() +{ + return pagenavigated; +} + +void DevTools::setTranslate(bool value) +{ + mutex.lock(); + translateready = value; + mutex.unlock(); +} + +bool DevTools::getTranslate() +{ + return translateready; +} + +bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root) +{ + if (!isConnected()) + return false; + root = QJsonObject(); + QJsonObject json; + task_completion_event response; + long id = idIncrement(); + json.insert("id", id); + json.insert("method", method); + if (!params.isEmpty()) + json.insert("params", params); + QJsonDocument doc(json); + QString message(doc.toJson(QJsonDocument::Compact)); + mutex.lock(); + mapqueue.insert(std::make_pair(id, response)); + mutex.unlock(); + webSocket.sendTextMessage(message); + webSocket.flush(); + try + { + root = create_task(response).get(); + } + catch (const std::exception& ex) + { + response.set_exception(ex); + return false; + } + if (!root.isEmpty()) + { + if (root.contains("error")) + { + return false; + } + else if (root.contains("result")) + return true; + else + return false; + } + else + return false; +} + +long DevTools::idIncrement() +{ + return ++idcounter; +} + +bool DevTools::isConnected() +{ + if (webSocket.state() == QAbstractSocket::ConnectedState) + return true; + else + return false; +} + +void DevTools::onTextMessageReceived(QString message) +{ + QJsonDocument doc = QJsonDocument::fromJson(message.toUtf8()); + if (doc.isObject()) + { + QJsonObject root = doc.object(); + if (root.contains("method")) + { + if (root.value("method").toString() == "Page.navigatedWithinDocument") + { + mutex.lock(); + pagenavigated = true; + mutex.unlock(); + } + if (root.value("method").toString() == "DOM.attributeModified") + { + if (root.value("params").toObject().value("value") == "lmt__mobile_share_container") + { + mutex.lock(); + translateready = true; + mutex.unlock(); + } + } + return; + + } + if (root.contains("id")) + { + long id = root.value("id").toInt(); + MapResponse::iterator request = mapqueue.find(id); + if (request != mapqueue.end()) + { + request->second.set(root); + mutex.lock(); + mapqueue.erase(request); + mutex.unlock(); + } + return; + } + } +} + +void DevTools::closeDevTools() +{ + if (this->mapqueue.size() > 0) + { + MapResponse::iterator iter = this->mapqueue.begin(); + MapResponse::iterator iend = this->mapqueue.end(); + for (; iter != iend; iter++) + { + iter->second.set_exception("exception"); + } + } + webSocket.close(); + mapqueue.clear(); + idcounter = 0; + + DWORD exitCode = 0; + if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) + { + if (exitCode == STILL_ACTIVE) + { + TerminateProcess(processInfo.hProcess, 0); + WaitForSingleObject(processInfo.hProcess, 100); + CloseHandle(processInfo.hProcess); + CloseHandle(processInfo.hThread); + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + try + { + std::filesystem::remove_all(L"devtoolscache"); + } + catch (const std::exception&) + { + + } + } + status = "Stopped"; + emit statusChanged(status); +} diff --git a/extensions/devtools.h b/extensions/devtools.h new file mode 100644 index 0000000..4b036f5 --- /dev/null +++ b/extensions/devtools.h @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include "network.h" + +using namespace Concurrency; + +typedef std::map> MapResponse; + +class DevTools : public QObject { + Q_OBJECT +public: + explicit DevTools(QObject* parent = nullptr); + ~DevTools(); + +Q_SIGNALS: + void statusChanged(const QString &); + +private Q_SLOTS: + void stateChanged(QAbstractSocket::SocketState state); + void onTextMessageReceived(QString message); + +public: + void startDevTools(QString path, bool headless = false, int port = 9222); + void closeDevTools(); + void setNavigated(bool value); + bool getNavigated(); + void setTranslate(bool value); + bool getTranslate(); + int getSession(); + bool SendRequest(QString command, QJsonObject params, QJsonObject& result); + QString getStatus(); + +private: + bool isConnected(); + bool startChrome(QString path, bool headless = false, int port = 9222); + bool GetwebSocketDebuggerUrl(QString& url, int port = 9222); + long idIncrement(); + int session; + QWebSocket webSocket; + std::mutex mutex; + MapResponse mapqueue; + bool pagenavigated; + bool translateready; + long idcounter; + PROCESS_INFORMATION processInfo; + QString status; +}; \ No newline at end of file diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp new file mode 100644 index 0000000..6b692f7 --- /dev/null +++ b/extensions/devtoolsdeepltranslate.cpp @@ -0,0 +1,195 @@ +#include "qtcommon.h" +#include "extension.h" +#include "network.h" +#include "devtools.h" + +extern const wchar_t* TRANSLATION_ERROR; +extern Synchronized translateTo; + +bool useCache = true, autostartchrome = false, headlesschrome = true; +int maxSentenceSize = 500, chromeport = 9222; + +const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate"; +QString URL = "https://www.deepl.com/en/translator"; +QStringList languages +{ + "Chinese (simplified): zh", + "Dutch: nl", + "English: en", + "French: fr", + "German: de", + "Italian: it", + "Japanese: ja", + "Polish: pl", + "Portuguese: pt", + "Russian: ru", + "Spanish: es", +}; + +int docfound = -1; +int targetNodeId = -1; +int session = -1; + +std::pair Translate(const std::wstring& text, DevTools* devtools) +{ + if (devtools->getStatus() == "Stopped") + { + return { false, FormatString(L"Error: chrome not started") }; + } + if ((devtools->getStatus().startsWith("Fail")) || (devtools->getStatus().startsWith("Unconnected"))) + { + return { false, FormatString(L"Error: %s", S(devtools->getStatus())) }; + } + if (session != devtools->getSession()) + { + session = devtools->getSession(); + docfound = -1; + targetNodeId = -1; + } + + QString qtext = S(text); + + // Check text for repeated symbols (e.g. only ellipsis) + if (qtext.length() > 2) + for (int i = 1; i < (qtext.length() - 1); i++) + { + if (qtext[i] != qtext[1]) + break; + if ((i + 2) == qtext.length() && (qtext.front() == qtext.back())) + { + return { false, text }; + } + } + + // Add spaces near ellipsis for better translation and check for quotes + qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230)); + qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230)); + qtext.replace(QRegularExpression("[" + QString(8230) + "]"), " " + QString(8230) + " "); + bool checkquote = false; + if ((qtext.front() == QString(12300)) && (qtext.back() == QString(12301))) + { + checkquote = true; + qtext.remove(0, 1); + qtext.chop(1); + } + QJsonObject root; + QJsonObject result; + + // Enable page feedback + if (!devtools->SendRequest("Page.enable", {}, root)) + { + return { false, FormatString(L"Error: page enable failed! %s", TRANSLATION_ERROR) }; + } + + // Navigate to site + QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; + devtools->setNavigated(false); + devtools->setTranslate(false); + if (devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) + { + // Wait until page is loaded + float timer = 0; + int timer_stop = 10; + while (!devtools->getNavigated() && timer < timer_stop) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + timer += 0.1; + } + if (timer >= timer_stop) + { + return { false, FormatString(L"Error: page load timeout %d s! %s", timer_stop, TRANSLATION_ERROR) }; + } + QString OuterHTML("
"); + + // Get document + if (docfound == -1) + { + if (!devtools->SendRequest("DOM.getDocument", {}, root)) + { + docfound = -1; + return { false, FormatString(L"Error: getDocument failed! %s", TRANSLATION_ERROR) }; + } + docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); + } + + //Get target selector + if (targetNodeId == -1) + { + if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "textarea.lmt__target_textarea"} }, root)) + || (root.value("result").toObject().value("nodeId").toInt() == 0)) + { + docfound = -1; + return { false, FormatString(L"Error: querySelector result failed! %s", TRANSLATION_ERROR) }; + } + targetNodeId = root.value("result").toObject().value("nodeId").toInt(); + } + + // Wait for translation to appear on the web page + timer = 0; + while (!devtools->getTranslate() && timer < timer_stop) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + timer += 0.1; + } + if (timer >= timer_stop) + { + // Catch notification if timeout + int noteNodeId = -1; + if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root)) + || (root.value("result").toObject().value("nodeId").toInt() == 0)) + { + return { false, FormatString(L"Error: result timeout %d s! %s", timer_stop, TRANSLATION_ERROR) }; + } + noteNodeId = root.value("result").toObject().value("nodeId").toInt(); + + if (devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId + 1} }, root)) + { + OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + OuterHTML.remove(QRegExp("<[^>]*>")); + OuterHTML = OuterHTML.trimmed(); + } + else + { + OuterHTML = "Could not get notification"; + } + return { false, FormatString(L"Error: got notification from translator: %s", S(OuterHTML)) }; + + } + + // Catch the translation + devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root); + result = root.value("result").toObject(); + OuterHTML = result.value("outerHTML").toString(); + OuterHTML.remove(QRegExp("<[^>]*>")); + OuterHTML = OuterHTML.trimmed(); + + // Check if the translator output language does not match the selected language + if (devtools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) + { + QJsonObject result = root.value("result").toObject(); + QJsonArray attributes = result.value("attributes").toArray(); + for (size_t i = 0; i < attributes.size(); i++) + { + if (attributes[i].toString() == "lang") + { + QString targetlang = attributes[i + 1].toString().mid(0, 2); + if (targetlang != S(translateTo.Copy())) + { + return { false, FormatString(L"Error: target langs do not match (%s): %s", S(targetlang), S(OuterHTML)) }; + } + } + } + } + + // Get quotes back + if (checkquote) + { + OuterHTML = "\"" + OuterHTML + "\""; + } + return { true, S(OuterHTML) }; + } + else + { + return { false, FormatString(L"Error: navigate failed! %s", TRANSLATION_ERROR) }; + } +} diff --git a/extensions/devtoolswrapper.cpp b/extensions/devtoolswrapper.cpp new file mode 100644 index 0000000..5044e38 --- /dev/null +++ b/extensions/devtoolswrapper.cpp @@ -0,0 +1,194 @@ +#include "qtcommon.h" +#include "extension.h" +#include "blockmarkup.h" +#include "network.h" +#include +#include +#include +#include "devtools.h" + +extern const char* NATIVE_LANGUAGE; +extern const char* TRANSLATE_TO; +extern const char* TRANSLATE_SELECTED_THREAD_ONLY; +extern const char* USE_TRANS_CACHE; +extern const char* MAX_SENTENCE_SIZE; +extern const char* TRANSLATION_PROVIDER; +extern QStringList languages; +const char* PATH_TO_CHROME = u8"Path to chrome"; +const char* AUTO_START_CHROME = u8"Start chrome automatically"; +const char* HEADLESS_CHROME = u8"Start in headless mode"; +const char* CHROME_DEBUG_PORT = u8"Chrome debug port"; +const char* DEV_TOOLS_STATUS = u8"Status: "; +const char* START_DEV_TOOLS_BUTTON = u8"Start"; +const char* START_DEV_TOOLS = u8"Start chrome"; +const char* STOP_DEV_TOOLS_BUTTON = u8"Stop"; +const char* STOP_DEV_TOOLS = u8"Stop chrome"; + +extern bool useCache, autostartchrome, headlesschrome; +extern int maxSentenceSize, chromeport; + +std::pair Translate(const std::wstring& text, DevTools* devtools); + +const char* LANGUAGE = u8"Language"; +const std::string TRANSLATION_CACHE_FILE = FormatString("%s Cache.txt", TRANSLATION_PROVIDER); + +QFormLayout* display; +QSettings settings = openSettings(); +Synchronized translateTo = L"en"; +Synchronized> translationCache; + +int savedSize; +DevTools* devtools = nullptr; +std::wstring pathtochrome = L""; + +void SaveCache() +{ + std::wstring allTranslations(L"\xfeff"); + for (const auto& [sentence, translation] : translationCache.Acquire().contents) + allTranslations.append(L"|SENTENCE|").append(sentence).append(L"|TRANSLATION|").append(translation).append(L"|END|\r\n"); + std::ofstream(TRANSLATION_CACHE_FILE, std::ios::binary | std::ios::trunc).write((const char*)allTranslations.c_str(), allTranslations.size() * sizeof(wchar_t)); + savedSize = translationCache->size(); +} + +class Window : public QDialog +{ +public: + Window() : + QDialog(nullptr, Qt::WindowMinMaxButtonsHint) + { + display = new QFormLayout(this); + settings.beginGroup(TRANSLATION_PROVIDER); + + auto languageBox = new QComboBox(this); + languageBox->addItems(languages); + int language = -1; + if (settings.contains(LANGUAGE)) language = languageBox->findText(settings.value(LANGUAGE).toString(), Qt::MatchEndsWith); + if (language < 0) language = languageBox->findText(NATIVE_LANGUAGE, Qt::MatchStartsWith); + if (language < 0) language = languageBox->findText("English", Qt::MatchStartsWith); + languageBox->setCurrentIndex(language); + saveLanguage(languageBox->currentText()); + display->addRow(TRANSLATE_TO, languageBox); + connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage); + for (auto [value, label] : Array{ + { useCache, USE_TRANS_CACHE }, + { autostartchrome, AUTO_START_CHROME }, + //{ headlesschrome, HEADLESS_CHROME } + }) + { + value = settings.value(label, value).toBool(); + auto checkBox = new QCheckBox(this); + checkBox->setChecked(value); + display->addRow(label, checkBox); + connect(checkBox, &QCheckBox::clicked, [label, &value](bool checked) { settings.setValue(label, value = checked); }); + } + for (auto [value, label] : Array{ + { maxSentenceSize, MAX_SENTENCE_SIZE }, + { chromeport, CHROME_DEBUG_PORT }, + }) + { + value = settings.value(label, value).toInt(); + auto spinBox = new QSpinBox(this); + spinBox->setRange(0, INT_MAX); + spinBox->setValue(value); + display->addRow(label, spinBox); + connect(spinBox, qOverload(&QSpinBox::valueChanged), [label, &value](int newValue) { settings.setValue(label, value = newValue); }); + } + + auto keyInput = new QLineEdit(settings.value(PATH_TO_CHROME).toString()); + pathtochrome = (S(keyInput->text())); + if (pathtochrome.empty()) + { + for (auto defaultpath : Array{ + { L"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe" }, + { L"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" }, + }) + if (std::filesystem::exists(defaultpath)) + { + pathtochrome = defaultpath; + keyInput->setText(S(pathtochrome)); + } + } + connect(keyInput, &QLineEdit::textChanged, [keyInput](QString key) { settings.setValue(PATH_TO_CHROME, S(pathtochrome = (S(key)))); }); + display->addRow(PATH_TO_CHROME, keyInput); + + connect(&startButton, &QPushButton::clicked, this, &Window::start); + connect(&stopButton, &QPushButton::clicked, this, &Window::stop); + display->addRow(START_DEV_TOOLS, &startButton); + display->addRow(STOP_DEV_TOOLS, &stopButton); + + status.setFrameStyle(QFrame::Panel | QFrame::Sunken); + display->addRow(DEV_TOOLS_STATUS, &status); + + setWindowTitle(TRANSLATION_PROVIDER); + QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection); + + std::ifstream stream(TRANSLATION_CACHE_FILE, std::ios::binary); + BlockMarkupIterator savedTranslations(stream, Array{ L"|SENTENCE|", L"|TRANSLATION|" }); + auto translationCache = ::translationCache.Acquire(); + + while (auto read = savedTranslations.Next()) + { + auto& [sentence, translation] = read.value(); + translationCache->try_emplace(std::move(sentence), std::move(translation)); + } + savedSize = translationCache->size(); + + devtools = new DevTools(this); + connect(devtools, &DevTools::statusChanged, [this](QString text) + { + status.setText(text); + }); + + if (autostartchrome) + QMetaObject::invokeMethod(this, &Window::start, Qt::QueuedConnection); + } + + ~Window() + { + stop(); + if (devtools != nullptr) + delete devtools; + SaveCache(); + } + +private: + void start() + { + if (devtools->getStatus() == "Stopped") + devtools->startDevTools(S(pathtochrome), headlesschrome, chromeport); + } + void stop() + { + devtools->closeDevTools(); + } + + + void saveLanguage(QString language) + { + settings.setValue(LANGUAGE, S(translateTo->assign(S(language.split(": ")[1])))); + } + QPushButton startButton{ START_DEV_TOOLS_BUTTON, this }; + QPushButton stopButton{ STOP_DEV_TOOLS_BUTTON, this }; + QLabel status{ "Stopped" }; +} window; + +bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) +{ + if (sentenceInfo["text number"] == 0 || sentence.size() > maxSentenceSize) return false; + + bool cache = false; + std::wstring translation; + if (useCache) + { + auto translationCache = ::translationCache.Acquire(); + if (auto it = translationCache->find(sentence); it != translationCache->end()) translation = it->second + L"\x200b"; + } + if (translation.empty() && (sentenceInfo["current select"])) + std::tie(cache, translation) = Translate(sentence, devtools); + if (cache) translationCache->try_emplace(sentence, translation); + if (cache && translationCache->size() > savedSize + 50) SaveCache(); + + JSON::Unescape(translation); + sentence += L"\n" + translation; + return true; +} \ No newline at end of file diff --git a/extensions/network.cpp b/extensions/network.cpp index abd4637..ca3cd76 100644 --- a/extensions/network.cpp +++ b/extensions/network.cpp @@ -10,13 +10,14 @@ HttpRequest::HttpRequest( const wchar_t* referrer, DWORD requestFlags, const wchar_t* httpVersion, - const wchar_t** acceptTypes + const wchar_t** acceptTypes, + DWORD port ) { static std::atomic internet = NULL; if (!internet) internet = WinHttpOpen(agentName, WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, NULL, NULL, 0); if (internet) - if (InternetHandle connection = WinHttpConnect(internet, serverName, INTERNET_DEFAULT_PORT, 0)) + if (InternetHandle connection = WinHttpConnect(internet, serverName, port, 0)) if (InternetHandle request = WinHttpOpenRequest(connection, action, objectName, httpVersion, referrer, acceptTypes, requestFlags)) if (WinHttpSendRequest(request, headers, -1UL, body.empty() ? NULL : body.data(), body.size(), body.size(), NULL)) { diff --git a/extensions/network.h b/extensions/network.h index d01ab3e..27ae6a4 100644 --- a/extensions/network.h +++ b/extensions/network.h @@ -16,7 +16,8 @@ struct HttpRequest const wchar_t* referrer = NULL, DWORD requestFlags = WINHTTP_FLAG_SECURE | WINHTTP_FLAG_ESCAPE_DISABLE, const wchar_t* httpVersion = NULL, - const wchar_t** acceptTypes = NULL + const wchar_t** acceptTypes = NULL, + DWORD port = INTERNET_DEFAULT_PORT ); operator bool() { return errorCode == ERROR_SUCCESS; } From a3ebaf002307cb4bdb687f03486246efdc204fe3 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Thu, 15 Oct 2020 02:32:58 +0300 Subject: [PATCH 02/14] add event catch method and refactor error messages Added method that catches specified events from the page Refactored error messages --- extensions/devtools.cpp | 90 ++++++++++++++++----------- extensions/devtools.h | 13 ++-- extensions/devtoolsdeepltranslate.cpp | 79 ++++++++++++----------- 3 files changed, 103 insertions(+), 79 deletions(-) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index f81828e..aa5c517 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -3,10 +3,11 @@ DevTools::DevTools(QObject* parent) : QObject(parent), idcounter(0), + idmethod(0), pagenavigated(false), translateready(false), status("Stopped"), - session(1) + session(0) { } @@ -116,30 +117,6 @@ void DevTools::stateChanged(QAbstractSocket::SocketState state) emit statusChanged(status); } -void DevTools::setNavigated(bool value) -{ - mutex.lock(); - pagenavigated = value; - mutex.unlock(); -} - -bool DevTools::getNavigated() -{ - return pagenavigated; -} - -void DevTools::setTranslate(bool value) -{ - mutex.lock(); - translateready = value; - mutex.unlock(); -} - -bool DevTools::getTranslate() -{ - return translateready; -} - bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root) { if (!isConnected()) @@ -183,11 +160,28 @@ bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root return false; } +long DevTools::methodToReceive(QString method, QJsonObject params) +{ + QJsonObject json; + long id = idmIncrement(); + json.insert("method", method); + json.insert("params", params); + mutex.lock(); + mapmethod.insert(std::make_pair(id, json)); + mutex.unlock(); + return id; +} + long DevTools::idIncrement() { return ++idcounter; } +long DevTools::idmIncrement() +{ + return ++idmethod; +} + bool DevTools::isConnected() { if (webSocket.state() == QAbstractSocket::ConnectedState) @@ -196,6 +190,29 @@ bool DevTools::isConnected() return false; } +bool DevTools::compareJson(QJsonObject storedparams, QJsonObject params) +{ + foreach(const QString & key, storedparams.keys()) + { + if (storedparams.value(key).isArray()) + return false; + if (storedparams.value(key) != params.value(key)) + return false; + if (!compareJson(storedparams.value(key).toObject(), params.value(key).toObject())) + return false; + } + return true; +} + +bool DevTools::checkMethod(long id) +{ + MapMethod::iterator iter = mapmethod.find(id); + if (iter == mapmethod.end()) + return true; + else + return false; +} + void DevTools::onTextMessageReceived(QString message) { QJsonDocument doc = QJsonDocument::fromJson(message.toUtf8()); @@ -204,23 +221,19 @@ void DevTools::onTextMessageReceived(QString message) QJsonObject root = doc.object(); if (root.contains("method")) { - if (root.value("method").toString() == "Page.navigatedWithinDocument") + for (auto iter = mapmethod.cbegin(); iter != mapmethod.cend();) { - mutex.lock(); - pagenavigated = true; - mutex.unlock(); - } - if (root.value("method").toString() == "DOM.attributeModified") - { - if (root.value("params").toObject().value("value") == "lmt__mobile_share_container") + if ((iter->second.value("method") == root.value("method")) + && ((iter->second.value("params").toObject().isEmpty()) + || (compareJson(iter->second.value("params").toObject(), root.value("params").toObject())))) { mutex.lock(); - translateready = true; + mapmethod.erase(iter++); mutex.unlock(); } + ++iter; } return; - } if (root.contains("id")) { @@ -242,17 +255,18 @@ void DevTools::closeDevTools() { if (this->mapqueue.size() > 0) { - MapResponse::iterator iter = this->mapqueue.begin(); - MapResponse::iterator iend = this->mapqueue.end(); + MapResponse::iterator iter = mapqueue.begin(); + MapResponse::iterator iend = mapqueue.end(); for (; iter != iend; iter++) { iter->second.set_exception("exception"); } } webSocket.close(); + mapmethod.clear(); mapqueue.clear(); idcounter = 0; - + idmethod = 0; DWORD exitCode = 0; if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) { diff --git a/extensions/devtools.h b/extensions/devtools.h index 4b036f5..fa4cb5f 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -7,6 +7,7 @@ using namespace Concurrency; typedef std::map> MapResponse; +typedef std::map MapMethod; class DevTools : public QObject { Q_OBJECT @@ -24,12 +25,10 @@ private Q_SLOTS: public: void startDevTools(QString path, bool headless = false, int port = 9222); void closeDevTools(); - void setNavigated(bool value); - bool getNavigated(); - void setTranslate(bool value); - bool getTranslate(); + bool checkMethod(long id); int getSession(); - bool SendRequest(QString command, QJsonObject params, QJsonObject& result); + bool SendRequest(QString method, QJsonObject params, QJsonObject& root); + long methodToReceive(QString method, QJsonObject params); QString getStatus(); private: @@ -37,13 +36,17 @@ private: bool startChrome(QString path, bool headless = false, int port = 9222); bool GetwebSocketDebuggerUrl(QString& url, int port = 9222); long idIncrement(); + long idmIncrement(); + bool compareJson(QJsonObject storedparams, QJsonObject params); int session; QWebSocket webSocket; std::mutex mutex; MapResponse mapqueue; + MapMethod mapmethod; bool pagenavigated; bool translateready; long idcounter; + long idmethod; PROCESS_INFORMATION processInfo; QString status; }; \ No newline at end of file diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 6b692f7..06fb6af 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -10,6 +10,12 @@ bool useCache = true, autostartchrome = false, headlesschrome = true; int maxSentenceSize = 500, chromeport = 9222; const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate"; +const wchar_t* ERROR_CHROME = L"Error: chrome not started"; +const wchar_t* ERROR_START_CHROME = L"Error: failed to start chrome or to connect to it"; +const wchar_t* ERROR_GOT_TIMEOUT = L"Error: timeout (s)"; +const wchar_t* ERROR_COMMAND_FAIL = L"Error: command failed"; +const wchar_t* ERROR_LANGUAGE = L"Error: target languages do not match"; + QString URL = "https://www.deepl.com/en/translator"; QStringList languages { @@ -26,27 +32,10 @@ QStringList languages "Spanish: es", }; -int docfound = -1; -int targetNodeId = -1; -int session = -1; +int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1; std::pair Translate(const std::wstring& text, DevTools* devtools) { - if (devtools->getStatus() == "Stopped") - { - return { false, FormatString(L"Error: chrome not started") }; - } - if ((devtools->getStatus().startsWith("Fail")) || (devtools->getStatus().startsWith("Unconnected"))) - { - return { false, FormatString(L"Error: %s", S(devtools->getStatus())) }; - } - if (session != devtools->getSession()) - { - session = devtools->getSession(); - docfound = -1; - targetNodeId = -1; - } - QString qtext = S(text); // Check text for repeated symbols (e.g. only ellipsis) @@ -57,9 +46,25 @@ std::pair Translate(const std::wstring& text, DevTools* devt break; if ((i + 2) == qtext.length() && (qtext.front() == qtext.back())) { - return { false, text }; + return { true, text }; } } + + if (devtools->getStatus() == "Stopped") + { + return { false, FormatString(L"%s", ERROR_CHROME) }; + } + if ((devtools->getStatus().startsWith("Fail")) || (devtools->getStatus().startsWith("Unconnected"))) + { + return { false, FormatString(L"%s", ERROR_START_CHROME) }; + } + if (session != devtools->getSession()) + { + session = devtools->getSession(); + docfound = -1; + targetNodeId = -1; + pageenabled = -1; + } // Add spaces near ellipsis for better translation and check for quotes qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230)); @@ -73,31 +78,34 @@ std::pair Translate(const std::wstring& text, DevTools* devt qtext.chop(1); } QJsonObject root; - QJsonObject result; // Enable page feedback - if (!devtools->SendRequest("Page.enable", {}, root)) + if (pageenabled == -1) { - return { false, FormatString(L"Error: page enable failed! %s", TRANSLATION_ERROR) }; + if (!devtools->SendRequest("Page.enable", {}, root)) + { + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + } + pageenabled = 1; } + long navigate = devtools->methodToReceive("Page.navigatedWithinDocument", {}); + long target = devtools->methodToReceive("DOM.attributeModified", { {"value" , "lmt__mobile_share_container"} }); // Navigate to site QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; - devtools->setNavigated(false); - devtools->setTranslate(false); if (devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) { // Wait until page is loaded float timer = 0; int timer_stop = 10; - while (!devtools->getNavigated() && timer < timer_stop) + while (!devtools->checkMethod(navigate) && timer < timer_stop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; - } + } if (timer >= timer_stop) { - return { false, FormatString(L"Error: page load timeout %d s! %s", timer_stop, TRANSLATION_ERROR) }; + return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; } QString OuterHTML("
"); @@ -107,7 +115,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt if (!devtools->SendRequest("DOM.getDocument", {}, root)) { docfound = -1; - return { false, FormatString(L"Error: getDocument failed! %s", TRANSLATION_ERROR) }; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); } @@ -119,14 +127,14 @@ std::pair Translate(const std::wstring& text, DevTools* devt || (root.value("result").toObject().value("nodeId").toInt() == 0)) { docfound = -1; - return { false, FormatString(L"Error: querySelector result failed! %s", TRANSLATION_ERROR) }; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } targetNodeId = root.value("result").toObject().value("nodeId").toInt(); } // Wait for translation to appear on the web page timer = 0; - while (!devtools->getTranslate() && timer < timer_stop) + while (!devtools->checkMethod(target) && timer < timer_stop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; @@ -138,7 +146,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root)) || (root.value("result").toObject().value("nodeId").toInt() == 0)) { - return { false, FormatString(L"Error: result timeout %d s! %s", timer_stop, TRANSLATION_ERROR) }; + return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; } noteNodeId = root.value("result").toObject().value("nodeId").toInt(); @@ -152,14 +160,13 @@ std::pair Translate(const std::wstring& text, DevTools* devt { OuterHTML = "Could not get notification"; } - return { false, FormatString(L"Error: got notification from translator: %s", S(OuterHTML)) }; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } // Catch the translation devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root); - result = root.value("result").toObject(); - OuterHTML = result.value("outerHTML").toString(); + OuterHTML = root.value("result").toObject().value("outerHTML").toString(); OuterHTML.remove(QRegExp("<[^>]*>")); OuterHTML = OuterHTML.trimmed(); @@ -175,7 +182,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt QString targetlang = attributes[i + 1].toString().mid(0, 2); if (targetlang != S(translateTo.Copy())) { - return { false, FormatString(L"Error: target langs do not match (%s): %s", S(targetlang), S(OuterHTML)) }; + return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) }; } } } @@ -190,6 +197,6 @@ std::pair Translate(const std::wstring& text, DevTools* devt } else { - return { false, FormatString(L"Error: navigate failed! %s", TRANSLATION_ERROR) }; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } } From 73d1f21bc11ccd62f85c2b8a5d1377257fc75136 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Thu, 15 Oct 2020 23:23:41 +0300 Subject: [PATCH 03/14] add notification alert When timeout from the net the method searches for notifications from the translator and put them to output --- extensions/devtools.cpp | 48 +++++++++++++++------------ extensions/devtools.h | 8 ++--- extensions/devtoolsdeepltranslate.cpp | 36 +++++++++----------- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index aa5c517..0330e29 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -4,11 +4,9 @@ DevTools::DevTools(QObject* parent) : QObject(parent), idcounter(0), idmethod(0), - pagenavigated(false), - translateready(false), status("Stopped"), session(0) -{ +{ } void DevTools::startDevTools(QString path, bool headless, int port) @@ -59,14 +57,14 @@ bool DevTools::startChrome(QString path, bool headless, int port) DWORD exitCode = 0; if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) && (exitCode == STILL_ACTIVE)) return false; - QString args = "--proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=" + QString args = "--proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=" + QString::fromStdWString(std::filesystem::current_path()) - + "\\devtoolscache --remote-debugging-port=" + + "\\devtoolscache --remote-debugging-port=" + QString::number(port); if (headless) args += " --headless"; STARTUPINFOW dummy = { sizeof(dummy) }; - if (!CreateProcessW(NULL, (wchar_t*)(path + " " + args).utf16(), nullptr, nullptr, + if (!CreateProcessW(NULL, (wchar_t*)(path + " " + args).utf16(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &dummy, &processInfo)) return false; else @@ -127,8 +125,7 @@ bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root long id = idIncrement(); json.insert("id", id); json.insert("method", method); - if (!params.isEmpty()) - json.insert("params", params); + json.insert("params", params); QJsonDocument doc(json); QString message(doc.toJson(QJsonDocument::Compact)); mutex.lock(); @@ -190,17 +187,27 @@ bool DevTools::isConnected() return false; } -bool DevTools::compareJson(QJsonObject storedparams, QJsonObject params) +bool DevTools::compareJson(QJsonValue storedparams, QJsonValue params) { - foreach(const QString & key, storedparams.keys()) + if (storedparams.isObject()) { - if (storedparams.value(key).isArray()) - return false; - if (storedparams.value(key) != params.value(key)) - return false; - if (!compareJson(storedparams.value(key).toObject(), params.value(key).toObject())) - return false; + foreach(const QString & key, storedparams.toObject().keys()) + { + QJsonValue storedvalue = storedparams.toObject().value(key); + QJsonValue value = params.toObject().value(key); + if (!compareJson(storedvalue, value)) + return false; + } } + else if (storedparams.isArray()) + { + for (int i = 0; i < storedparams.toArray().size(); i++) + if (!compareJson(storedparams.toArray()[i], params.toArray()[i])) + return false; + } + else if (storedparams.toVariant() != params.toVariant()) + return false; + return true; } @@ -223,9 +230,8 @@ void DevTools::onTextMessageReceived(QString message) { for (auto iter = mapmethod.cbegin(); iter != mapmethod.cend();) { - if ((iter->second.value("method") == root.value("method")) - && ((iter->second.value("params").toObject().isEmpty()) - || (compareJson(iter->second.value("params").toObject(), root.value("params").toObject())))) + if ((iter->second.value("method") == root.value("method")) + && (compareJson(iter->second.value("params"), root.value("params")))) { mutex.lock(); mapmethod.erase(iter++); @@ -248,7 +254,7 @@ void DevTools::onTextMessageReceived(QString message) } return; } - } + } } void DevTools::closeDevTools() @@ -289,4 +295,4 @@ void DevTools::closeDevTools() } status = "Stopped"; emit statusChanged(status); -} +} \ No newline at end of file diff --git a/extensions/devtools.h b/extensions/devtools.h index fa4cb5f..546eb9e 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -16,7 +16,7 @@ public: ~DevTools(); Q_SIGNALS: - void statusChanged(const QString &); + void statusChanged(const QString&); private Q_SLOTS: void stateChanged(QAbstractSocket::SocketState state); @@ -28,7 +28,7 @@ public: bool checkMethod(long id); int getSession(); bool SendRequest(QString method, QJsonObject params, QJsonObject& root); - long methodToReceive(QString method, QJsonObject params); + long methodToReceive(QString method, QJsonObject params = {}); QString getStatus(); private: @@ -37,14 +37,12 @@ private: bool GetwebSocketDebuggerUrl(QString& url, int port = 9222); long idIncrement(); long idmIncrement(); - bool compareJson(QJsonObject storedparams, QJsonObject params); + bool compareJson(QJsonValue storedparams, QJsonValue params); int session; QWebSocket webSocket; std::mutex mutex; MapResponse mapqueue; MapMethod mapmethod; - bool pagenavigated; - bool translateready; long idcounter; long idmethod; PROCESS_INFORMATION processInfo; diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 06fb6af..7ad0956 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -15,6 +15,7 @@ const wchar_t* ERROR_START_CHROME = L"Error: failed to start chrome or to connec const wchar_t* ERROR_GOT_TIMEOUT = L"Error: timeout (s)"; const wchar_t* ERROR_COMMAND_FAIL = L"Error: command failed"; const wchar_t* ERROR_LANGUAGE = L"Error: target languages do not match"; +const wchar_t* ERROR_NOTE = L"Error: notification"; QString URL = "https://www.deepl.com/en/translator"; QStringList languages @@ -49,7 +50,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt return { true, text }; } } - + if (devtools->getStatus() == "Stopped") { return { false, FormatString(L"%s", ERROR_CHROME) }; @@ -88,8 +89,8 @@ std::pair Translate(const std::wstring& text, DevTools* devt } pageenabled = 1; } - long navigate = devtools->methodToReceive("Page.navigatedWithinDocument", {}); - long target = devtools->methodToReceive("DOM.attributeModified", { {"value" , "lmt__mobile_share_container"} }); + long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); + long target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container" } }); // Navigate to site QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; @@ -102,12 +103,11 @@ std::pair Translate(const std::wstring& text, DevTools* devt { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; - } + } if (timer >= timer_stop) { return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; } - QString OuterHTML("
"); // Get document if (docfound == -1) @@ -139,9 +139,13 @@ std::pair Translate(const std::wstring& text, DevTools* devt std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } - if (timer >= timer_stop) + + // Catch the translation + devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root); + QString OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + if (OuterHTML == "
") { - // Catch notification if timeout + // Try to catch the notification int noteNodeId = -1; if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root)) || (root.value("result").toObject().value("nodeId").toInt() == 0)) @@ -150,23 +154,15 @@ std::pair Translate(const std::wstring& text, DevTools* devt } noteNodeId = root.value("result").toObject().value("nodeId").toInt(); - if (devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId + 1} }, root)) + if (devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) { OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - OuterHTML.remove(QRegExp("<[^>]*>")); - OuterHTML = OuterHTML.trimmed(); } - else - { - OuterHTML = "Could not get notification"; - } - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + OuterHTML.remove(QRegExp("<[^>]*>")); + OuterHTML = OuterHTML.trimmed(); + return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; } - - // Catch the translation - devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root); - OuterHTML = root.value("result").toObject().value("outerHTML").toString(); OuterHTML.remove(QRegExp("<[^>]*>")); OuterHTML = OuterHTML.trimmed(); @@ -199,4 +195,4 @@ std::pair Translate(const std::wstring& text, DevTools* devt { return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } -} +} \ No newline at end of file From 2ab780a4917e486bc42a017b91809e86c61c4640 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Sun, 18 Oct 2020 16:16:06 +0300 Subject: [PATCH 04/14] add user-agent switch method Add method that changes user-agent when in the headless mode --- extensions/devtools.cpp | 55 +++++++++++++++------------ extensions/devtools.h | 5 ++- extensions/devtoolsdeepltranslate.cpp | 20 +++++++++- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index 0330e29..bf627e9 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -13,20 +13,31 @@ void DevTools::startDevTools(QString path, bool headless, int port) { if (startChrome(path, headless, port)) { + QJsonDocument doc; QString webSocketDebuggerUrl; - if (GetwebSocketDebuggerUrl(webSocketDebuggerUrl, port)) + if (GetJsonfromHTTP(doc, "/json/list", port)) { - connect(&webSocket, &QWebSocket::stateChanged, this, &DevTools::stateChanged); - connect(&webSocket, &QWebSocket::textMessageReceived, this, &DevTools::onTextMessageReceived); - webSocket.open(webSocketDebuggerUrl); - session += 1; + for (const auto obj : doc.array()) + if (obj.toObject().value("type") == "page") + { + webSocketDebuggerUrl.append(obj.toObject().value("webSocketDebuggerUrl").toString()); + break; + } + if (!webSocketDebuggerUrl.isEmpty()) + { + if (GetJsonfromHTTP(doc, "/json/version", port)) + { + useragent = doc.object().value("User-Agent").toString(); + } + connect(&webSocket, &QWebSocket::stateChanged, this, &DevTools::stateChanged); + connect(&webSocket, &QWebSocket::textMessageReceived, this, &DevTools::onTextMessageReceived); + webSocket.open(webSocketDebuggerUrl); + session += 1; + return; + } } - else - { - status = "Failed to find chrome debug port!"; - emit statusChanged(status); - } - + status = "Failed to find chrome debug port!"; + emit statusChanged(status); } else { @@ -45,6 +56,11 @@ QString DevTools::getStatus() return status; } +QString DevTools::getUserAgent() +{ + return useragent; +} + DevTools::~DevTools() { closeDevTools(); @@ -71,14 +87,13 @@ bool DevTools::startChrome(QString path, bool headless, int port) return true; } -bool DevTools::GetwebSocketDebuggerUrl(QString& url, int port) +bool DevTools::GetJsonfromHTTP(QJsonDocument& doc, QString object, int port) { - url.clear(); if (HttpRequest httpRequest{ L"Mozilla/5.0 Textractor", L"127.0.0.1", L"POST", - FormatString(L"/json/list").c_str(), + object.toStdWString().c_str(), "", NULL, NULL, @@ -89,16 +104,8 @@ bool DevTools::GetwebSocketDebuggerUrl(QString& url, int port) }) { QString qtString = QString::fromStdWString(httpRequest.response); - QJsonDocument doc = QJsonDocument::fromJson(qtString.toUtf8()); - QJsonArray rootObject = doc.array(); - - for (const auto obj : rootObject) - if (obj.toObject().value("type") == "page") - { - url.append(obj.toObject().value("webSocketDebuggerUrl").toString()); - break; - } - if (!url.isEmpty()) + doc = QJsonDocument::fromJson(qtString.toUtf8()); + if (!doc.isEmpty()) return true; else return false; diff --git a/extensions/devtools.h b/extensions/devtools.h index 546eb9e..82641d1 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -1,4 +1,3 @@ -#include #include #include #include @@ -30,11 +29,12 @@ public: bool SendRequest(QString method, QJsonObject params, QJsonObject& root); long methodToReceive(QString method, QJsonObject params = {}); QString getStatus(); + QString getUserAgent(); private: bool isConnected(); bool startChrome(QString path, bool headless = false, int port = 9222); - bool GetwebSocketDebuggerUrl(QString& url, int port = 9222); + bool GetJsonfromHTTP(QJsonDocument& doc, QString object, int port = 9222); long idIncrement(); long idmIncrement(); bool compareJson(QJsonValue storedparams, QJsonValue params); @@ -47,4 +47,5 @@ private: long idmethod; PROCESS_INFORMATION processInfo; QString status; + QString useragent; }; \ No newline at end of file diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 7ad0956..e10526c 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -1,6 +1,5 @@ #include "qtcommon.h" #include "extension.h" -#include "network.h" #include "devtools.h" extern const wchar_t* TRANSLATION_ERROR; @@ -33,7 +32,7 @@ QStringList languages "Spanish: es", }; -int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1; +int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1, useragentflag = -1; std::pair Translate(const std::wstring& text, DevTools* devtools) { @@ -65,6 +64,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt docfound = -1; targetNodeId = -1; pageenabled = -1; + useragentflag = -1; } // Add spaces near ellipsis for better translation and check for quotes @@ -89,6 +89,22 @@ std::pair Translate(const std::wstring& text, DevTools* devt } pageenabled = 1; } + + // Change user-agent if in headless mode + if (useragentflag == -1) + { + QString useragent = devtools->getUserAgent(); + useragent.replace(QRegularExpression("HeadlessChrome"), "Chrome"); + if (!useragent.isEmpty()) + { + if (!devtools->SendRequest("Network.setUserAgentOverride", { {"userAgent", useragent} }, root)) + { + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + } + } + useragentflag = 1; + } + long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); long target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container" } }); From 3af42170755db637511a2b5468604e6b34821916 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Tue, 20 Oct 2020 02:08:59 +0300 Subject: [PATCH 05/14] add check for outdated doc and refactor input text modifications Added the flag for checking if webdocument structure is outdated Refactor text modifications before processing to the translator --- extensions/devtools.cpp | 6 +- extensions/devtoolsdeepltranslate.cpp | 236 ++++++++++++++------------ extensions/devtoolswrapper.cpp | 15 ++ 3 files changed, 147 insertions(+), 110 deletions(-) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index bf627e9..8c4f128 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -71,7 +71,7 @@ bool DevTools::startChrome(QString path, bool headless, int port) if (!std::filesystem::exists(path.toStdWString())) return false; DWORD exitCode = 0; - if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) && (exitCode == STILL_ACTIVE)) + if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE && exitCode == STILL_ACTIVE) return false; QString args = "--proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=" + QString::fromStdWString(std::filesystem::current_path()) @@ -237,8 +237,8 @@ void DevTools::onTextMessageReceived(QString message) { for (auto iter = mapmethod.cbegin(); iter != mapmethod.cend();) { - if ((iter->second.value("method") == root.value("method")) - && (compareJson(iter->second.value("params"), root.value("params")))) + if (iter->second.value("method") == root.value("method") + && compareJson(iter->second.value("params"), root.value("params"))) { mutex.lock(); mapmethod.erase(iter++); diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index e10526c..0b8b6fd 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -33,28 +33,48 @@ QStringList languages }; int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1, useragentflag = -1; +long update = -1; std::pair Translate(const std::wstring& text, DevTools* devtools) { QString qtext = S(text); + qtext.remove(QString(12288)); // japanese space (no need for translator) - // Check text for repeated symbols (e.g. only ellipsis) - if (qtext.length() > 2) - for (int i = 1; i < (qtext.length() - 1); i++) - { - if (qtext[i] != qtext[1]) - break; - if ((i + 2) == qtext.length() && (qtext.front() == qtext.back())) - { - return { true, text }; - } - } + // Check quotes + bool checkquote = false; + if ((qtext.front() == QString(12300) && qtext.back() == QString(12301)) // japanese quotation marks + || (qtext.front() == "\"" && qtext.back() == "\"")) + { + checkquote = true; + qtext.remove(0, 1); + qtext.chop(1); + } + if (qtext == QString(12387)) // if text consists of only one sokuon, add exclamation mark for correct translation + { + qtext += "!"; + } + + // Check ellipsis + int count = qtext.count(QString(8230)); // ellipsis + if (count == qtext.length() + || (count == (qtext.length() - 1) && qtext.back() == QString(12290))) // japanese end of a sentence + { + return { true, text }; + } + + // Put quotes back + if (checkquote) + { + qtext = "\"" + qtext + "\""; + } + + // Check status if (devtools->getStatus() == "Stopped") { return { false, FormatString(L"%s", ERROR_CHROME) }; } - if ((devtools->getStatus().startsWith("Fail")) || (devtools->getStatus().startsWith("Unconnected"))) + if (devtools->getStatus().startsWith("Fail") || devtools->getStatus().startsWith("Unconnected")) { return { false, FormatString(L"%s", ERROR_START_CHROME) }; } @@ -65,22 +85,15 @@ std::pair Translate(const std::wstring& text, DevTools* devt targetNodeId = -1; pageenabled = -1; useragentflag = -1; + update = -1; } - // Add spaces near ellipsis for better translation and check for quotes - qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230)); - qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230)); - qtext.replace(QRegularExpression("[" + QString(8230) + "]"), " " + QString(8230) + " "); - bool checkquote = false; - if ((qtext.front() == QString(12300)) && (qtext.back() == QString(12301))) - { - checkquote = true; - qtext.remove(0, 1); - qtext.chop(1); - } - QJsonObject root; + // Erase tags and reduce the number of ellipsis for better translation + qtext.remove(QRegExp("<[^>]*>")); + qtext.replace(QRegExp("(" + QString(8230) + ")+"), " " + QString(8230)); // Enable page feedback + QJsonObject root; if (pageenabled == -1) { if (!devtools->SendRequest("Page.enable", {}, root)) @@ -94,9 +107,9 @@ std::pair Translate(const std::wstring& text, DevTools* devt if (useragentflag == -1) { QString useragent = devtools->getUserAgent(); - useragent.replace(QRegularExpression("HeadlessChrome"), "Chrome"); if (!useragent.isEmpty()) { + useragent.replace("HeadlessChrome", "Chrome"); if (!devtools->SendRequest("Network.setUserAgentOverride", { {"userAgent", useragent} }, root)) { return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; @@ -107,108 +120,117 @@ std::pair Translate(const std::wstring& text, DevTools* devt long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); long target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container" } }); + if (update == -1) + { + update = devtools->methodToReceive("DOM.documentUpdated"); + } // Navigate to site QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; - if (devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) + if (!devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) { - // Wait until page is loaded - float timer = 0; - int timer_stop = 10; - while (!devtools->checkMethod(navigate) && timer < timer_stop) + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + } + + // Wait until page is loaded + float timer = 0; + int timer_stop = 10; + while (!devtools->checkMethod(navigate) && timer < timer_stop) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + timer += 0.1; + } + if (timer >= timer_stop) + { + return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; + } + + // Check if document is outdated + if (devtools->checkMethod(update)) + { + docfound = -1; + targetNodeId = -1; + update = -1; + } + + // Get document + if (docfound == -1) + { + if (!devtools->SendRequest("DOM.getDocument", {}, root)) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - timer += 0.1; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } - if (timer >= timer_stop) + docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); + } + + // Get target selector + if (targetNodeId == -1) + { + if (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "textarea.lmt__target_textarea"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) + { + docfound = -1; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + } + targetNodeId = root.value("result").toObject().value("nodeId").toInt(); + } + + // Wait for translation to appear on the web page + timer = 0; + while (!devtools->checkMethod(target) && timer < timer_stop) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + timer += 0.1; + } + + // Catch the translation + if (!devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) + { + docfound = -1; + targetNodeId = -1; + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + } + QString OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + if (OuterHTML == "
") + { + // Try to catch the notification + int noteNodeId = -1; + if (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) { return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; } + noteNodeId = root.value("result").toObject().value("nodeId").toInt(); - // Get document - if (docfound == -1) + if (devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) { - if (!devtools->SendRequest("DOM.getDocument", {}, root)) - { - docfound = -1; - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } - docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); - } - - //Get target selector - if (targetNodeId == -1) - { - if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "textarea.lmt__target_textarea"} }, root)) - || (root.value("result").toObject().value("nodeId").toInt() == 0)) - { - docfound = -1; - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } - targetNodeId = root.value("result").toObject().value("nodeId").toInt(); - } - - // Wait for translation to appear on the web page - timer = 0; - while (!devtools->checkMethod(target) && timer < timer_stop) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - timer += 0.1; - } - - // Catch the translation - devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root); - QString OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - if (OuterHTML == "
") - { - // Try to catch the notification - int noteNodeId = -1; - if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root)) - || (root.value("result").toObject().value("nodeId").toInt() == 0)) - { - return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; - } - noteNodeId = root.value("result").toObject().value("nodeId").toInt(); - - if (devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) - { - OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - } - OuterHTML.remove(QRegExp("<[^>]*>")); - OuterHTML = OuterHTML.trimmed(); - - return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; + OuterHTML = root.value("result").toObject().value("outerHTML").toString(); } OuterHTML.remove(QRegExp("<[^>]*>")); OuterHTML = OuterHTML.trimmed(); - // Check if the translator output language does not match the selected language - if (devtools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) + return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; + } + OuterHTML.remove(QRegExp("<[^>]*>")); + OuterHTML = OuterHTML.trimmed(); + + // Check if the translator output language does not match the selected language + if (devtools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) + { + QJsonObject result = root.value("result").toObject(); + QJsonArray attributes = result.value("attributes").toArray(); + for (size_t i = 0; i < attributes.size(); i++) { - QJsonObject result = root.value("result").toObject(); - QJsonArray attributes = result.value("attributes").toArray(); - for (size_t i = 0; i < attributes.size(); i++) + if (attributes[i].toString() == "lang") { - if (attributes[i].toString() == "lang") + QString targetlang = attributes[i + 1].toString().mid(0, 2); + if (targetlang != S(translateTo.Copy())) { - QString targetlang = attributes[i + 1].toString().mid(0, 2); - if (targetlang != S(translateTo.Copy())) - { - return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) }; - } + return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) }; } } } + } - // Get quotes back - if (checkquote) - { - OuterHTML = "\"" + OuterHTML + "\""; - } - return { true, S(OuterHTML) }; - } - else - { - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } + return { true, S(OuterHTML) }; } \ No newline at end of file diff --git a/extensions/devtoolswrapper.cpp b/extensions/devtoolswrapper.cpp index 5044e38..a319c78 100644 --- a/extensions/devtoolswrapper.cpp +++ b/extensions/devtoolswrapper.cpp @@ -50,6 +50,17 @@ void SaveCache() savedSize = translationCache->size(); } +void EraseControlCharacters(std::wstring& text) +{ + for (auto it = text.begin(); it!= text.end(); ++it) + { + if ((*it == '\n') || (*it == '\r') || (*it == '\t') || (int(*it) == 4) || (int(*it) == 5)) + { + text.erase(it--); + } + } +} + class Window : public QDialog { public: @@ -184,7 +195,11 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) if (auto it = translationCache->find(sentence); it != translationCache->end()) translation = it->second + L"\x200b"; } if (translation.empty() && (sentenceInfo["current select"])) + { + EraseControlCharacters(sentence); std::tie(cache, translation) = Translate(sentence, devtools); + } + if (cache) translationCache->try_emplace(sentence, translation); if (cache && translationCache->size() > savedSize + 50) SaveCache(); From 8ab1e576bd7a02f8580294668c40d2f0352fa59f Mon Sep 17 00:00:00 2001 From: zeheyler Date: Wed, 21 Oct 2020 00:10:58 +0300 Subject: [PATCH 06/14] add call queue check Added queue check for multiple calls from the main program --- extensions/devtoolsdeepltranslate.cpp | 29 ++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 0b8b6fd..57b1116 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -33,7 +33,8 @@ QStringList languages }; int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1, useragentflag = -1; -long update = -1; +long update = -1, callnumber = 0; +std::vector callqueue; std::pair Translate(const std::wstring& text, DevTools* devtools) { @@ -118,6 +119,20 @@ std::pair Translate(const std::wstring& text, DevTools* devt useragentflag = 1; } + float timer = 0; + int timer_stop = 10; + long calltag = ++callnumber; + callqueue.insert(callqueue.begin(), calltag); + while (callqueue.back() != calltag && timer < 2 * timer_stop) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + timer += 0.1; + } + if (timer >= timer_stop) + { + callqueue.pop_back(); + return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, 2 * timer_stop) }; + } long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); long target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container" } }); if (update == -1) @@ -129,12 +144,12 @@ std::pair Translate(const std::wstring& text, DevTools* devt QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; if (!devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) { + callqueue.pop_back(); return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } // Wait until page is loaded - float timer = 0; - int timer_stop = 10; + timer = 0; while (!devtools->checkMethod(navigate) && timer < timer_stop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -142,6 +157,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt } if (timer >= timer_stop) { + callqueue.pop_back(); return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; } @@ -158,6 +174,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt { if (!devtools->SendRequest("DOM.getDocument", {}, root)) { + callqueue.pop_back(); return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); @@ -170,6 +187,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt || root.value("result").toObject().value("nodeId").toInt() == 0) { docfound = -1; + callqueue.pop_back(); return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } targetNodeId = root.value("result").toObject().value("nodeId").toInt(); @@ -182,15 +200,17 @@ std::pair Translate(const std::wstring& text, DevTools* devt std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } - + // Catch the translation if (!devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) { docfound = -1; targetNodeId = -1; + callqueue.pop_back(); return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; } QString OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + callqueue.pop_back(); if (OuterHTML == "
") { // Try to catch the notification @@ -231,6 +251,5 @@ std::pair Translate(const std::wstring& text, DevTools* devt } } } - return { true, S(OuterHTML) }; } \ No newline at end of file From 93b8711bf83ac1af789f3ee73c5c9073d40101aa Mon Sep 17 00:00:00 2001 From: zeheyler Date: Thu, 22 Oct 2020 23:15:45 +0300 Subject: [PATCH 07/14] refactor errorcode Refactored error processing --- extensions/devtoolsdeepltranslate.cpp | 145 +++++++++++++------------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 57b1116..7f3a67d 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -41,7 +41,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt QString qtext = S(text); qtext.remove(QString(12288)); // japanese space (no need for translator) - // Check quotes + // Remove quotes bool checkquote = false; if ((qtext.front() == QString(12300) && qtext.back() == QString(12301)) // japanese quotation marks || (qtext.front() == "\"" && qtext.back() == "\"")) @@ -51,18 +51,18 @@ std::pair Translate(const std::wstring& text, DevTools* devt qtext.chop(1); } - if (qtext == QString(12387)) // if text consists of only one sokuon, add exclamation mark for correct translation - { - qtext += "!"; - } - - // Check ellipsis + // Check specific cases (sentence has only one japanese symbol or consists of ellipsis) int count = qtext.count(QString(8230)); // ellipsis if (count == qtext.length() || (count == (qtext.length() - 1) && qtext.back() == QString(12290))) // japanese end of a sentence { return { true, text }; } + if (count == (qtext.length() - 1)) + { + qtext.remove(QString(8230)); + qtext += QString(12290) + QString(8230); // add the end symbol for correct translation + } // Put quotes back if (checkquote) @@ -87,79 +87,69 @@ std::pair Translate(const std::wstring& text, DevTools* devt pageenabled = -1; useragentflag = -1; update = -1; + callnumber = 0; } - // Erase tags and reduce the number of ellipsis for better translation + // Remove tags and reduce the number of ellipsis for correct translation qtext.remove(QRegExp("<[^>]*>")); qtext.replace(QRegExp("(" + QString(8230) + ")+"), " " + QString(8230)); // Enable page feedback QJsonObject root; + int errorcode = 0; if (pageenabled == -1) { if (!devtools->SendRequest("Page.enable", {}, root)) - { - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } - pageenabled = 1; + errorcode = 1; + else + pageenabled = 1; } // Change user-agent if in headless mode - if (useragentflag == -1) + if (useragentflag == -1 && errorcode == 0) { QString useragent = devtools->getUserAgent(); if (!useragent.isEmpty()) { useragent.replace("HeadlessChrome", "Chrome"); if (!devtools->SendRequest("Network.setUserAgentOverride", { {"userAgent", useragent} }, root)) - { - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } + errorcode = 1; + else + useragentflag = 1; } - useragentflag = 1; } + // Increase queue counter and wait until previous calls are done float timer = 0; int timer_stop = 10; long calltag = ++callnumber; callqueue.insert(callqueue.begin(), calltag); - while (callqueue.back() != calltag && timer < 2 * timer_stop) + while (errorcode == 0 && callqueue.back() != calltag && timer < 2 * timer_stop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } if (timer >= timer_stop) - { - callqueue.pop_back(); - return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, 2 * timer_stop) }; - } + errorcode = 5; + + // Set methods to receive long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); long target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container" } }); if (update == -1) - { update = devtools->methodToReceive("DOM.documentUpdated"); - } - // Navigate to site + // Navigate to site and wait until it is loaded QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; - if (!devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) - { - callqueue.pop_back(); - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } - - // Wait until page is loaded + if (errorcode == 0 && !devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) + errorcode = 1; timer = 0; - while (!devtools->checkMethod(navigate) && timer < timer_stop) + while (errorcode == 0 && !devtools->checkMethod(navigate) && timer < timer_stop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } if (timer >= timer_stop) - { - callqueue.pop_back(); - return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; - } + errorcode = 2; // Check if document is outdated if (devtools->checkMethod(update)) @@ -170,86 +160,99 @@ std::pair Translate(const std::wstring& text, DevTools* devt } // Get document - if (docfound == -1) + if (docfound == -1 && errorcode == 0) { if (!devtools->SendRequest("DOM.getDocument", {}, root)) - { - callqueue.pop_back(); - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - } - docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); + errorcode = 1; + else + docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); } // Get target selector - if (targetNodeId == -1) + if (targetNodeId == -1 && errorcode == 0) { if (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "textarea.lmt__target_textarea"} }, root) || root.value("result").toObject().value("nodeId").toInt() == 0) { docfound = -1; - callqueue.pop_back(); - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + errorcode = 1; } - targetNodeId = root.value("result").toObject().value("nodeId").toInt(); + else + targetNodeId = root.value("result").toObject().value("nodeId").toInt(); } - // Wait for translation to appear on the web page + // Wait for the translation to appear on the web page timer = 0; - while (!devtools->checkMethod(target) && timer < timer_stop) + while (errorcode == 0 && !devtools->checkMethod(target) && timer < timer_stop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } // Catch the translation - if (!devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) + QString OuterHTML; + if (errorcode == 0 && !devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) { - docfound = -1; targetNodeId = -1; - callqueue.pop_back(); - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + errorcode = 1; + } + else + { + OuterHTML = root.value("result").toObject().value("outerHTML").toString(); } - QString OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - callqueue.pop_back(); if (OuterHTML == "
") { // Try to catch the notification int noteNodeId = -1; - if (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root) + if (errorcode == 0 && !devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root) || root.value("result").toObject().value("nodeId").toInt() == 0) { - return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) }; + errorcode = 2; } - noteNodeId = root.value("result").toObject().value("nodeId").toInt(); - - if (devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) + else { - OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + noteNodeId = root.value("result").toObject().value("nodeId").toInt(); + if (errorcode == 0 && devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) + { + OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + } + errorcode = 3; } - OuterHTML.remove(QRegExp("<[^>]*>")); - OuterHTML = OuterHTML.trimmed(); - - return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; } OuterHTML.remove(QRegExp("<[^>]*>")); OuterHTML = OuterHTML.trimmed(); // Check if the translator output language does not match the selected language - if (devtools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) + QString targetlang; + if (errorcode == 0 && devtools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) { - QJsonObject result = root.value("result").toObject(); - QJsonArray attributes = result.value("attributes").toArray(); + QJsonArray attributes = root.value("result").toObject().value("attributes").toArray(); for (size_t i = 0; i < attributes.size(); i++) { if (attributes[i].toString() == "lang") { - QString targetlang = attributes[i + 1].toString().mid(0, 2); + targetlang = attributes[i + 1].toString().mid(0, 2); if (targetlang != S(translateTo.Copy())) { - return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) }; + errorcode = 4; } } } } - return { true, S(OuterHTML) }; + + callqueue.pop_back(); + if (errorcode == 0) + return { true, S(OuterHTML) }; + else if (errorcode == 1) + return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; + else if (errorcode == 2) + return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, timer_stop) }; + else if (errorcode == 3) + return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; + else if (errorcode == 4) + return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) }; + else if (errorcode == 5) + return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, 2*timer_stop) }; + else + return { false, FormatString(L"%s", TRANSLATION_ERROR) }; } \ No newline at end of file From 4b46057800040e3fb9a614e748e179e0e21607fb Mon Sep 17 00:00:00 2001 From: zeheyler Date: Thu, 29 Oct 2020 02:08:11 +0300 Subject: [PATCH 08/14] add backup check method Added backup check method in case the first method fails --- extensions/devtools.cpp | 4 ++-- extensions/devtoolsdeepltranslate.cpp | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index 8c4f128..1b5f1b4 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -7,6 +7,8 @@ DevTools::DevTools(QObject* parent) : status("Stopped"), session(0) { + connect(&webSocket, &QWebSocket::stateChanged, this, &DevTools::stateChanged); + connect(&webSocket, &QWebSocket::textMessageReceived, this, &DevTools::onTextMessageReceived); } void DevTools::startDevTools(QString path, bool headless, int port) @@ -29,8 +31,6 @@ void DevTools::startDevTools(QString path, bool headless, int port) { useragent = doc.object().value("User-Agent").toString(); } - connect(&webSocket, &QWebSocket::stateChanged, this, &DevTools::stateChanged); - connect(&webSocket, &QWebSocket::textMessageReceived, this, &DevTools::onTextMessageReceived); webSocket.open(webSocketDebuggerUrl); session += 1; return; diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 7f3a67d..7b20d29 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -32,7 +32,7 @@ QStringList languages "Spanish: es", }; -int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1, useragentflag = -1; +int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1, useragentflag = -1, backup = -1; long update = -1, callnumber = 0; std::vector callqueue; @@ -88,6 +88,7 @@ std::pair Translate(const std::wstring& text, DevTools* devt useragentflag = -1; update = -1; callnumber = 0; + backup = -1; } // Remove tags and reduce the number of ellipsis for correct translation @@ -134,7 +135,11 @@ std::pair Translate(const std::wstring& text, DevTools* devt // Set methods to receive long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); - long target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container" } }); + long target; + if (backup == -1) + target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); + else + target = devtools->methodToReceive("DOM.childNodeCountUpdated"); if (update == -1) update = devtools->methodToReceive("DOM.documentUpdated"); @@ -204,8 +209,8 @@ std::pair Translate(const std::wstring& text, DevTools* devt { // Try to catch the notification int noteNodeId = -1; - if (errorcode == 0 && !devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) + if (errorcode == 0 && (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) && timer >= timer_stop) { errorcode = 2; } @@ -221,6 +226,8 @@ std::pair Translate(const std::wstring& text, DevTools* devt } OuterHTML.remove(QRegExp("<[^>]*>")); OuterHTML = OuterHTML.trimmed(); + if (backup == -1 && errorcode == 0 && timer >= timer_stop) + backup = 1; // Check if the translator output language does not match the selected language QString targetlang; From 9bb7fbff062149b80907fc1296be29885d430af0 Mon Sep 17 00:00:00 2001 From: zeheyler <72524723+zeheyler@users.noreply.github.com> Date: Wed, 4 Nov 2020 01:23:20 +0300 Subject: [PATCH 09/14] add source language selection and rename variables Added source language selection since autodetect fails sometimes Refactored names of variables --- extensions/devtools.cpp | 69 ++++---- extensions/devtools.h | 12 +- extensions/devtoolsdeepltranslate.cpp | 230 ++++++++++++++++---------- extensions/devtoolswrapper.cpp | 119 +++++++------ 4 files changed, 259 insertions(+), 171 deletions(-) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index 1b5f1b4..8fa4d83 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -2,8 +2,8 @@ DevTools::DevTools(QObject* parent) : QObject(parent), - idcounter(0), - idmethod(0), + idCounter(0), + idMethod(0), status("Stopped"), session(0) { @@ -29,19 +29,19 @@ void DevTools::startDevTools(QString path, bool headless, int port) { if (GetJsonfromHTTP(doc, "/json/version", port)) { - useragent = doc.object().value("User-Agent").toString(); + userAgent = doc.object().value("User-Agent").toString(); } webSocket.open(webSocketDebuggerUrl); session += 1; return; } } - status = "Failed to find chrome debug port!"; + status = "Failed to find Chrome debug port!"; emit statusChanged(status); } else { - status = "Failed to start chrome!"; + status = "Failed to start Chrome!"; emit statusChanged(status); } } @@ -58,7 +58,7 @@ QString DevTools::getStatus() QString DevTools::getUserAgent() { - return useragent; + return userAgent; } DevTools::~DevTools() @@ -136,7 +136,7 @@ bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root QJsonDocument doc(json); QString message(doc.toJson(QJsonDocument::Compact)); mutex.lock(); - mapqueue.insert(std::make_pair(id, response)); + mapQueue.insert(std::make_pair(id, response)); mutex.unlock(); webSocket.sendTextMessage(message); webSocket.flush(); @@ -171,19 +171,19 @@ long DevTools::methodToReceive(QString method, QJsonObject params) json.insert("method", method); json.insert("params", params); mutex.lock(); - mapmethod.insert(std::make_pair(id, json)); + mapMethod.insert(std::make_pair(id, json)); mutex.unlock(); return id; } long DevTools::idIncrement() { - return ++idcounter; + return ++idCounter; } long DevTools::idmIncrement() { - return ++idmethod; + return ++idMethod; } bool DevTools::isConnected() @@ -194,25 +194,30 @@ bool DevTools::isConnected() return false; } -bool DevTools::compareJson(QJsonValue storedparams, QJsonValue params) +bool DevTools::compareJson(QJsonValue stored, QJsonValue params) { - if (storedparams.isObject()) + if (stored.isObject()) { - foreach(const QString & key, storedparams.toObject().keys()) + foreach(const QString & key, stored.toObject().keys()) { - QJsonValue storedvalue = storedparams.toObject().value(key); + QJsonValue storedvalue = stored.toObject().value(key); QJsonValue value = params.toObject().value(key); if (!compareJson(storedvalue, value)) return false; } } - else if (storedparams.isArray()) + else if (stored.isArray()) { - for (int i = 0; i < storedparams.toArray().size(); i++) - if (!compareJson(storedparams.toArray()[i], params.toArray()[i])) + for (int i = 0; i < stored.toArray().size(); i++) + if (!compareJson(stored.toArray()[i], params.toArray()[i])) return false; } - else if (storedparams.toVariant() != params.toVariant()) + else if (stored.isString()) + { + if (!stored.toString().contains(params.toString())) + return false; + } + else if (stored.toVariant() != params.toVariant()) return false; return true; @@ -220,8 +225,8 @@ bool DevTools::compareJson(QJsonValue storedparams, QJsonValue params) bool DevTools::checkMethod(long id) { - MapMethod::iterator iter = mapmethod.find(id); - if (iter == mapmethod.end()) + MapMethod::iterator iter = mapMethod.find(id); + if (iter == mapMethod.end()) return true; else return false; @@ -235,13 +240,13 @@ void DevTools::onTextMessageReceived(QString message) QJsonObject root = doc.object(); if (root.contains("method")) { - for (auto iter = mapmethod.cbegin(); iter != mapmethod.cend();) + for (auto iter = mapMethod.cbegin(); iter != mapMethod.cend();) { if (iter->second.value("method") == root.value("method") && compareJson(iter->second.value("params"), root.value("params"))) { mutex.lock(); - mapmethod.erase(iter++); + mapMethod.erase(iter++); mutex.unlock(); } ++iter; @@ -251,12 +256,12 @@ void DevTools::onTextMessageReceived(QString message) if (root.contains("id")) { long id = root.value("id").toInt(); - MapResponse::iterator request = mapqueue.find(id); - if (request != mapqueue.end()) + MapResponse::iterator request = mapQueue.find(id); + if (request != mapQueue.end()) { request->second.set(root); mutex.lock(); - mapqueue.erase(request); + mapQueue.erase(request); mutex.unlock(); } return; @@ -266,20 +271,20 @@ void DevTools::onTextMessageReceived(QString message) void DevTools::closeDevTools() { - if (this->mapqueue.size() > 0) + if (this->mapQueue.size() > 0) { - MapResponse::iterator iter = mapqueue.begin(); - MapResponse::iterator iend = mapqueue.end(); + MapResponse::iterator iter = mapQueue.begin(); + MapResponse::iterator iend = mapQueue.end(); for (; iter != iend; iter++) { iter->second.set_exception("exception"); } } webSocket.close(); - mapmethod.clear(); - mapqueue.clear(); - idcounter = 0; - idmethod = 0; + mapMethod.clear(); + mapQueue.clear(); + idCounter = 0; + idMethod = 0; DWORD exitCode = 0; if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) { diff --git a/extensions/devtools.h b/extensions/devtools.h index 82641d1..c13b610 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -37,15 +37,15 @@ private: bool GetJsonfromHTTP(QJsonDocument& doc, QString object, int port = 9222); long idIncrement(); long idmIncrement(); - bool compareJson(QJsonValue storedparams, QJsonValue params); + bool compareJson(QJsonValue stored, QJsonValue params); int session; QWebSocket webSocket; std::mutex mutex; - MapResponse mapqueue; - MapMethod mapmethod; - long idcounter; - long idmethod; + MapResponse mapQueue; + MapMethod mapMethod; + long idCounter; + long idMethod; PROCESS_INFORMATION processInfo; QString status; - QString useragent; + QString userAgent; }; \ No newline at end of file diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 7b20d29..3efcf84 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -3,21 +3,21 @@ #include "devtools.h" extern const wchar_t* TRANSLATION_ERROR; -extern Synchronized translateTo; +extern Synchronized translateTo, translateFrom; -bool useCache = true, autostartchrome = false, headlesschrome = true; -int maxSentenceSize = 500, chromeport = 9222; +bool useCache = true, autoStartChrome = false, headlessChrome = true; +int maxSentenceSize = 500, chromePort = 9222; const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate"; -const wchar_t* ERROR_CHROME = L"Error: chrome not started"; -const wchar_t* ERROR_START_CHROME = L"Error: failed to start chrome or to connect to it"; +const wchar_t* ERROR_CHROME = L"Error: Chrome not started"; +const wchar_t* ERROR_START_CHROME = L"Error: failed to start Chrome or to connect to it"; const wchar_t* ERROR_GOT_TIMEOUT = L"Error: timeout (s)"; const wchar_t* ERROR_COMMAND_FAIL = L"Error: command failed"; const wchar_t* ERROR_LANGUAGE = L"Error: target languages do not match"; const wchar_t* ERROR_NOTE = L"Error: notification"; QString URL = "https://www.deepl.com/en/translator"; -QStringList languages +QStringList languagesTo { "Chinese (simplified): zh", "Dutch: nl", @@ -32,21 +32,37 @@ QStringList languages "Spanish: es", }; -int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1, useragentflag = -1, backup = -1; -long update = -1, callnumber = 0; -std::vector callqueue; +QStringList languagesFrom +{ + "Any language: ", + "Chinese: zh", + "Dutch: nl", + "English: en", + "French: fr", + "German: de", + "Italian: it", + "Japanese: ja", + "Polish: pl", + "Portuguese: pt", + "Russian: ru", + "Spanish: es", +}; -std::pair Translate(const std::wstring& text, DevTools* devtools) +int docFound = -1, targetNodeId = -1, session = -1, pageEnabled = -1, userAgentFlag = -1, backup = -1, sourceLangId = -1; +long update = -1, callNumber = 0; +std::vector callQueue; + +std::pair Translate(const std::wstring& text, DevTools* devTools) { QString qtext = S(text); qtext.remove(QString(12288)); // japanese space (no need for translator) // Remove quotes - bool checkquote = false; + bool checkQuote = false; if ((qtext.front() == QString(12300) && qtext.back() == QString(12301)) // japanese quotation marks || (qtext.front() == "\"" && qtext.back() == "\"")) { - checkquote = true; + checkQuote = true; qtext.remove(0, 1); qtext.chop(1); } @@ -65,29 +81,30 @@ std::pair Translate(const std::wstring& text, DevTools* devt } // Put quotes back - if (checkquote) + if (checkQuote) { qtext = "\"" + qtext + "\""; } // Check status - if (devtools->getStatus() == "Stopped") + if (devTools->getStatus() == "Stopped") { return { false, FormatString(L"%s", ERROR_CHROME) }; } - if (devtools->getStatus().startsWith("Fail") || devtools->getStatus().startsWith("Unconnected")) + if (devTools->getStatus().startsWith("Fail") || devTools->getStatus().startsWith("Unconnected")) { return { false, FormatString(L"%s", ERROR_START_CHROME) }; } - if (session != devtools->getSession()) + if (session != devTools->getSession()) { - session = devtools->getSession(); - docfound = -1; + session = devTools->getSession(); + docFound = -1; targetNodeId = -1; - pageenabled = -1; - useragentflag = -1; + sourceLangId = -1; + pageEnabled = -1; + userAgentFlag = -1; update = -1; - callnumber = 0; + callNumber = 0; backup = -1; } @@ -97,98 +114,112 @@ std::pair Translate(const std::wstring& text, DevTools* devt // Enable page feedback QJsonObject root; - int errorcode = 0; - if (pageenabled == -1) + int errorCode = 0; + if (pageEnabled == -1) { - if (!devtools->SendRequest("Page.enable", {}, root)) - errorcode = 1; + if (!devTools->SendRequest("Page.enable", {}, root)) + errorCode = 1; else - pageenabled = 1; + pageEnabled = 1; } // Change user-agent if in headless mode - if (useragentflag == -1 && errorcode == 0) + if (userAgentFlag == -1 && errorCode == 0) { - QString useragent = devtools->getUserAgent(); - if (!useragent.isEmpty()) + QString userAgent = devTools->getUserAgent(); + if (!userAgent.isEmpty()) { - useragent.replace("HeadlessChrome", "Chrome"); - if (!devtools->SendRequest("Network.setUserAgentOverride", { {"userAgent", useragent} }, root)) - errorcode = 1; + userAgent.replace("HeadlessChrome", "Chrome"); + if (!devTools->SendRequest("Network.setUserAgentOverride", { {"userAgent", userAgent} }, root)) + errorCode = 1; else - useragentflag = 1; + userAgentFlag = 1; } } // Increase queue counter and wait until previous calls are done float timer = 0; - int timer_stop = 10; - long calltag = ++callnumber; - callqueue.insert(callqueue.begin(), calltag); - while (errorcode == 0 && callqueue.back() != calltag && timer < 2 * timer_stop) + int timerStop = 10; + long callTag = ++callNumber; + callQueue.insert(callQueue.begin(), callTag); + while (errorCode == 0 && callQueue.back() != callTag && timer < 2 * timerStop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } - if (timer >= timer_stop) - errorcode = 5; + if (timer >= timerStop) + errorCode = 5; // Set methods to receive - long navigate = devtools->methodToReceive("Page.navigatedWithinDocument"); + long navigate = devTools->methodToReceive("Page.navigatedWithinDocument"); long target; if (backup == -1) - target = devtools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); + target = devTools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); else - target = devtools->methodToReceive("DOM.childNodeCountUpdated"); + target = devTools->methodToReceive("DOM.childNodeCountUpdated"); if (update == -1) - update = devtools->methodToReceive("DOM.documentUpdated"); + update = devTools->methodToReceive("DOM.documentUpdated"); // Navigate to site and wait until it is loaded - QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext; - if (errorcode == 0 && !devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root)) - errorcode = 1; + QString checkFrom = translateFrom.Copy().empty() ? "ja" : S(translateFrom.Copy()); + QString fullUrl = URL + "#" + checkFrom + "/" + S(translateTo.Copy()) + "/" + qtext; + if (errorCode == 0 && !devTools->SendRequest("Page.navigate", { {"url", fullUrl} }, root)) + errorCode = 1; timer = 0; - while (errorcode == 0 && !devtools->checkMethod(navigate) && timer < timer_stop) + while (errorCode == 0 && !devTools->checkMethod(navigate) && timer < timerStop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } - if (timer >= timer_stop) - errorcode = 2; + if (timer >= timerStop) + errorCode = 2; // Check if document is outdated - if (devtools->checkMethod(update)) + if (devTools->checkMethod(update)) { - docfound = -1; + docFound = -1; targetNodeId = -1; update = -1; } // Get document - if (docfound == -1 && errorcode == 0) + if (docFound == -1 && errorCode == 0) { - if (!devtools->SendRequest("DOM.getDocument", {}, root)) - errorcode = 1; + if (!devTools->SendRequest("DOM.getDocument", {}, root)) + errorCode = 1; else - docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); + docFound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); } // Get target selector - if (targetNodeId == -1 && errorcode == 0) + if (targetNodeId == -1 && errorCode == 0) { - if (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "textarea.lmt__target_textarea"} }, root) + if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "textarea.lmt__target_textarea"} }, root) || root.value("result").toObject().value("nodeId").toInt() == 0) { - docfound = -1; - errorcode = 1; + docFound = -1; + errorCode = 1; } else targetNodeId = root.value("result").toObject().value("nodeId").toInt(); } + // Get source language selector + if (sourceLangId == -1 && errorCode == 0) + { + if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__language_select--source"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) + { + docFound = -1; + errorCode = 1; + } + else + sourceLangId = root.value("result").toObject().value("nodeId").toInt(); + } + // Wait for the translation to appear on the web page timer = 0; - while (errorcode == 0 && !devtools->checkMethod(target) && timer < timer_stop) + while (errorCode == 0 && !devTools->checkMethod(target) && timer < timerStop) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; @@ -196,10 +227,10 @@ std::pair Translate(const std::wstring& text, DevTools* devt // Catch the translation QString OuterHTML; - if (errorcode == 0 && !devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) + if (errorCode == 0 && !devTools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) { targetNodeId = -1; - errorcode = 1; + errorCode = 1; } else { @@ -209,57 +240,90 @@ std::pair Translate(const std::wstring& text, DevTools* devt { // Try to catch the notification int noteNodeId = -1; - if (errorcode == 0 && (!devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) && timer >= timer_stop) + if (errorCode == 0 && (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__system_notification"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) && timer >= timerStop) { - errorcode = 2; + errorCode = 2; } else { noteNodeId = root.value("result").toObject().value("nodeId").toInt(); - if (errorcode == 0 && devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) + if (errorCode == 0 && devTools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) { OuterHTML = root.value("result").toObject().value("outerHTML").toString(); } - errorcode = 3; + errorCode = 3; } } OuterHTML.remove(QRegExp("<[^>]*>")); OuterHTML = OuterHTML.trimmed(); - if (backup == -1 && errorcode == 0 && timer >= timer_stop) + if (backup == -1 && errorCode == 0 && timer >= timerStop) backup = 1; // Check if the translator output language does not match the selected language - QString targetlang; - if (errorcode == 0 && devtools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) + QString targetLang; + if (errorCode == 0 && devTools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) { QJsonArray attributes = root.value("result").toObject().value("attributes").toArray(); for (size_t i = 0; i < attributes.size(); i++) { if (attributes[i].toString() == "lang") { - targetlang = attributes[i + 1].toString().mid(0, 2); - if (targetlang != S(translateTo.Copy())) + targetLang = attributes[i + 1].toString().mid(0, 2); + if (targetLang != S(translateTo.Copy())) { - errorcode = 4; + errorCode = 4; } } + break; + } + } + + // Check selected source language + if (errorCode == 0 && devTools->SendRequest("DOM.getAttributes", { {"nodeId", sourceLangId} }, root)) + { + QJsonArray attributes = root.value("result").toObject().value("attributes").toArray(); + for (size_t i = 0; i < attributes.size(); i++) + { + if (attributes[i].toString() == "dl-selected-lang" + && attributes[i + 1].toString().mid(0, 2) != S(translateFrom.Copy())) + { + QStringList::const_iterator constIter; + for (constIter = languagesFrom.constBegin(); constIter != languagesFrom.constEnd(); ++constIter) + { + if (constIter->contains(": " + S(translateFrom.Copy()))) + break; + } + devTools->SendRequest("Runtime.evaluate", { {"expression", + "document\ + .querySelector('div.lmt__language_select--source')\ + .querySelector('button.lmt__language_select__active')\ + .click();\ + document\ + .evaluate(\"//button[contains(text(), '" + + constIter->split(": ")[0] + "')]\", \ + document.querySelector('div.lmt__language_select__menu'),\ + null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)\ + .singleNodeValue\ + .click();" + } }, root); + } } } - callqueue.pop_back(); - if (errorcode == 0) + callQueue.pop_back(); + if (errorCode == 0) return { true, S(OuterHTML) }; - else if (errorcode == 1) + else if (errorCode == 1) return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - else if (errorcode == 2) - return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, timer_stop) }; - else if (errorcode == 3) + else if (errorCode == 2) + return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, timerStop) }; + else if (errorCode == 3) return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; - else if (errorcode == 4) - return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) }; - else if (errorcode == 5) - return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, 2*timer_stop) }; + else if (errorCode == 4) + return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetLang), S(OuterHTML)) }; + else if (errorCode == 5) + return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, 2*timerStop) }; else return { false, FormatString(L"%s", TRANSLATION_ERROR) }; } \ No newline at end of file diff --git a/extensions/devtoolswrapper.cpp b/extensions/devtoolswrapper.cpp index a319c78..546d5a2 100644 --- a/extensions/devtoolswrapper.cpp +++ b/extensions/devtoolswrapper.cpp @@ -13,33 +13,37 @@ extern const char* TRANSLATE_SELECTED_THREAD_ONLY; extern const char* USE_TRANS_CACHE; extern const char* MAX_SENTENCE_SIZE; extern const char* TRANSLATION_PROVIDER; -extern QStringList languages; -const char* PATH_TO_CHROME = u8"Path to chrome"; -const char* AUTO_START_CHROME = u8"Start chrome automatically"; +extern QStringList languagesTo; +extern QStringList languagesFrom; +const char* TRANSLATE_FROM = u8"Translate from"; +const char* NATIVE_LANGUAGE_FROM = u8"Japanese"; +const char* PATH_TO_CHROME = u8"Path to Chrome"; +const char* AUTO_START_CHROME = u8"Start Chrome automatically"; const char* HEADLESS_CHROME = u8"Start in headless mode"; const char* CHROME_DEBUG_PORT = u8"Chrome debug port"; const char* DEV_TOOLS_STATUS = u8"Status: "; const char* START_DEV_TOOLS_BUTTON = u8"Start"; -const char* START_DEV_TOOLS = u8"Start chrome"; +const char* START_DEV_TOOLS = u8"Start Chrome"; const char* STOP_DEV_TOOLS_BUTTON = u8"Stop"; -const char* STOP_DEV_TOOLS = u8"Stop chrome"; +const char* STOP_DEV_TOOLS = u8"Stop Chrome"; -extern bool useCache, autostartchrome, headlesschrome; -extern int maxSentenceSize, chromeport; +extern bool useCache, autoStartChrome, headlessChrome; +extern int maxSentenceSize, chromePort; -std::pair Translate(const std::wstring& text, DevTools* devtools); +std::pair Translate(const std::wstring& text, DevTools* devTools); -const char* LANGUAGE = u8"Language"; +const char* LANGUAGE_TO = u8"Language"; +const char* LANGUAGE_FROM = u8"Language from"; const std::string TRANSLATION_CACHE_FILE = FormatString("%s Cache.txt", TRANSLATION_PROVIDER); QFormLayout* display; -QSettings settings = openSettings(); -Synchronized translateTo = L"en"; +Settings settings; +Synchronized translateTo = L"en", translateFrom = L"ja"; Synchronized> translationCache; int savedSize; -DevTools* devtools = nullptr; -std::wstring pathtochrome = L""; +DevTools* devTools = nullptr; +std::wstring pathToChrome = L""; void SaveCache() { @@ -67,23 +71,35 @@ public: Window() : QDialog(nullptr, Qt::WindowMinMaxButtonsHint) { + localize(); display = new QFormLayout(this); settings.beginGroup(TRANSLATION_PROVIDER); - auto languageBox = new QComboBox(this); - languageBox->addItems(languages); - int language = -1; - if (settings.contains(LANGUAGE)) language = languageBox->findText(settings.value(LANGUAGE).toString(), Qt::MatchEndsWith); - if (language < 0) language = languageBox->findText(NATIVE_LANGUAGE, Qt::MatchStartsWith); - if (language < 0) language = languageBox->findText("English", Qt::MatchStartsWith); - languageBox->setCurrentIndex(language); - saveLanguage(languageBox->currentText()); - display->addRow(TRANSLATE_TO, languageBox); - connect(languageBox, &QComboBox::currentTextChanged, this, &Window::saveLanguage); + auto languageBoxTo = new QComboBox(this); + languageBoxTo->addItems(languagesTo); + int languageTo = -1; + if (settings.contains(LANGUAGE_TO)) languageTo = languageBoxTo->findText(settings.value(LANGUAGE_TO).toString(), Qt::MatchEndsWith); + if (languageTo < 0) languageTo = languageBoxTo->findText(NATIVE_LANGUAGE, Qt::MatchStartsWith); + if (languageTo < 0) languageTo = languageBoxTo->findText("English", Qt::MatchStartsWith); + languageBoxTo->setCurrentIndex(languageTo); + saveLanguage(languageBoxTo->currentText()); + display->addRow(TRANSLATE_TO, languageBoxTo); + connect(languageBoxTo, &QComboBox::currentTextChanged, this, &Window::saveLanguage); + + auto languageBoxFrom = new QComboBox(this); + languageBoxFrom->addItems(languagesFrom); + int languageFrom = -1; + if (settings.contains(LANGUAGE_FROM)) languageFrom = languageBoxFrom->findText(settings.value(LANGUAGE_FROM).toString(), Qt::MatchEndsWith); + if (languageFrom < 0) languageFrom = languageBoxFrom->findText(NATIVE_LANGUAGE_FROM, Qt::MatchStartsWith); + if (languageFrom < 0) languageFrom = languageBoxFrom->findText("Japanese", Qt::MatchStartsWith); + languageBoxFrom->setCurrentIndex(languageFrom); + saveLanguageFrom(languageBoxFrom->currentText()); + display->addRow(TRANSLATE_FROM, languageBoxFrom); + connect(languageBoxFrom, &QComboBox::currentTextChanged, this, &Window::saveLanguageFrom); for (auto [value, label] : Array{ { useCache, USE_TRANS_CACHE }, - { autostartchrome, AUTO_START_CHROME }, - //{ headlesschrome, HEADLESS_CHROME } + { autoStartChrome, AUTO_START_CHROME }, + //{ headlessChrome, HEADLESS_CHROME } }) { value = settings.value(label, value).toBool(); @@ -94,7 +110,7 @@ public: } for (auto [value, label] : Array{ { maxSentenceSize, MAX_SENTENCE_SIZE }, - { chromeport, CHROME_DEBUG_PORT }, + { chromePort, CHROME_DEBUG_PORT }, }) { value = settings.value(label, value).toInt(); @@ -106,24 +122,24 @@ public: } auto keyInput = new QLineEdit(settings.value(PATH_TO_CHROME).toString()); - pathtochrome = (S(keyInput->text())); - if (pathtochrome.empty()) + pathToChrome = (S(keyInput->text())); + if (pathToChrome.empty()) { - for (auto defaultpath : Array{ + for (auto defaultPath : Array{ { L"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe" }, { L"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" }, }) - if (std::filesystem::exists(defaultpath)) + if (std::filesystem::exists(defaultPath)) { - pathtochrome = defaultpath; - keyInput->setText(S(pathtochrome)); + pathToChrome = defaultPath; + keyInput->setText(S(pathToChrome)); } } - connect(keyInput, &QLineEdit::textChanged, [keyInput](QString key) { settings.setValue(PATH_TO_CHROME, S(pathtochrome = (S(key)))); }); + connect(keyInput, &QLineEdit::textChanged, [keyInput](QString key) { settings.setValue(PATH_TO_CHROME, S(pathToChrome = (S(key)))); }); display->addRow(PATH_TO_CHROME, keyInput); - connect(&startButton, &QPushButton::clicked, this, &Window::start); - connect(&stopButton, &QPushButton::clicked, this, &Window::stop); + connect(&startButton, &QPushButton::clicked, this, &Window::Start); + connect(&stopButton, &QPushButton::clicked, this, &Window::Stop); display->addRow(START_DEV_TOOLS, &startButton); display->addRow(STOP_DEV_TOOLS, &stopButton); @@ -144,39 +160,42 @@ public: } savedSize = translationCache->size(); - devtools = new DevTools(this); - connect(devtools, &DevTools::statusChanged, [this](QString text) + devTools = new DevTools(this); + connect(devTools, &DevTools::statusChanged, [this](QString text) { status.setText(text); }); - if (autostartchrome) - QMetaObject::invokeMethod(this, &Window::start, Qt::QueuedConnection); + if (autoStartChrome) + QMetaObject::invokeMethod(this, &Window::Start, Qt::QueuedConnection); } ~Window() { - stop(); - if (devtools != nullptr) - delete devtools; + Stop(); + if (devTools != nullptr) + delete devTools; SaveCache(); } private: - void start() + void Start() { - if (devtools->getStatus() == "Stopped") - devtools->startDevTools(S(pathtochrome), headlesschrome, chromeport); + if (devTools->getStatus() == "Stopped") + devTools->startDevTools(S(pathToChrome), headlessChrome, chromePort); } - void stop() + void Stop() { - devtools->closeDevTools(); + devTools->closeDevTools(); } - void saveLanguage(QString language) { - settings.setValue(LANGUAGE, S(translateTo->assign(S(language.split(": ")[1])))); + settings.setValue(LANGUAGE_TO, S(translateTo->assign(S(language.split(": ")[1])))); + } + void saveLanguageFrom(QString language) + { + settings.setValue(LANGUAGE_FROM, S(translateFrom->assign(S(language.split(": ")[1])))); } QPushButton startButton{ START_DEV_TOOLS_BUTTON, this }; QPushButton stopButton{ STOP_DEV_TOOLS_BUTTON, this }; @@ -197,7 +216,7 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) if (translation.empty() && (sentenceInfo["current select"])) { EraseControlCharacters(sentence); - std::tie(cache, translation) = Translate(sentence, devtools); + std::tie(cache, translation) = Translate(sentence, devTools); } if (cache) translationCache->try_emplace(sentence, translation); From 6958b22e68700ef160fb59cba005b672b6c8aaad Mon Sep 17 00:00:00 2001 From: zeheyler Date: Wed, 4 Nov 2020 20:51:23 +0300 Subject: [PATCH 10/14] fix target selector check Fixed target selector check in some cases Changed line endings from CRLF to LF --- extensions/devtoolsdeepltranslate.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 3efcf84..f227706 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -48,7 +48,7 @@ QStringList languagesFrom "Spanish: es", }; -int docFound = -1, targetNodeId = -1, session = -1, pageEnabled = -1, userAgentFlag = -1, backup = -1, sourceLangId = -1; +int docFound = -1, targetNodeId = -1, session = -1, pageEnabled = -1, userAgentFlag = -1, backup = -1, sourceLangId = -1, mobileShareId = -1; long update = -1, callNumber = 0; std::vector callQueue; @@ -101,6 +101,7 @@ std::pair Translate(const std::wstring& text, DevTools* devT docFound = -1; targetNodeId = -1; sourceLangId = -1; + mobileShareId = -1; pageEnabled = -1; userAgentFlag = -1; update = -1; @@ -153,7 +154,9 @@ std::pair Translate(const std::wstring& text, DevTools* devT // Set methods to receive long navigate = devTools->methodToReceive("Page.navigatedWithinDocument"); long target; - if (backup == -1) + if (mobileShareId != -1 && backup == -1) + target = devTools->methodToReceive("DOM.attributeModified", { { "nodeId" , mobileShareId } , { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); + else if (mobileShareId == -1 && backup == -1) target = devTools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); else target = devTools->methodToReceive("DOM.childNodeCountUpdated"); @@ -217,6 +220,19 @@ std::pair Translate(const std::wstring& text, DevTools* devT sourceLangId = root.value("result").toObject().value("nodeId").toInt(); } + // Get mobile share selector + if (mobileShareId == -1 && errorCode == 0) + { + if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__mobile_share_container"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) + { + docFound = -1; + errorCode = 1; + } + else + mobileShareId = root.value("result").toObject().value("nodeId").toInt(); + } + // Wait for the translation to appear on the web page timer = 0; while (errorCode == 0 && !devTools->checkMethod(target) && timer < timerStop) From ea6ddc7a7cf5dde7f12db7f73f6c9e401a5ce6f0 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Wed, 9 Dec 2020 14:00:57 +0300 Subject: [PATCH 11/14] small improvements Added check for the empty answer from the translator Small refactors --- extensions/devtoolsdeepltranslate.cpp | 34 +++++++++++++++++---------- extensions/devtoolswrapper.cpp | 5 ++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index f227706..f60b52a 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -15,6 +15,7 @@ const wchar_t* ERROR_GOT_TIMEOUT = L"Error: timeout (s)"; const wchar_t* ERROR_COMMAND_FAIL = L"Error: command failed"; const wchar_t* ERROR_LANGUAGE = L"Error: target languages do not match"; const wchar_t* ERROR_NOTE = L"Error: notification"; +const wchar_t* ERROR_EMPTY_ANSWER = L"Error: empty translation"; QString URL = "https://www.deepl.com/en/translator"; QStringList languagesTo @@ -56,6 +57,7 @@ std::pair Translate(const std::wstring& text, DevTools* devT { QString qtext = S(text); qtext.remove(QString(12288)); // japanese space (no need for translator) + qtext.replace(QString(12289), ","); // replace the japanese comma with the latin comma for correct translation // Remove quotes bool checkQuote = false; @@ -148,7 +150,7 @@ std::pair Translate(const std::wstring& text, DevTools* devT std::this_thread::sleep_for(std::chrono::milliseconds(100)); timer += 0.1; } - if (timer >= timerStop) + if (timer >= 2 * timerStop) errorCode = 5; // Set methods to receive @@ -243,23 +245,29 @@ std::pair Translate(const std::wstring& text, DevTools* devT // Catch the translation QString OuterHTML; - if (errorCode == 0 && !devTools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) + if (errorCode == 0) { - targetNodeId = -1; - errorCode = 1; + if (!devTools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) + { + targetNodeId = -1; + errorCode = 1; + } + else + { + OuterHTML = root.value("result").toObject().value("outerHTML").toString(); + } } - else - { - OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - } - if (OuterHTML == "
") + if (errorCode == 0 && OuterHTML == "
") { // Try to catch the notification int noteNodeId = -1; - if (errorCode == 0 && (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__system_notification"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) && timer >= timerStop) + if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__system_notification"} }, root) + || root.value("result").toObject().value("nodeId").toInt() == 0) { - errorCode = 2; + if (timer >= timerStop) + errorCode = 2; + else + errorCode = 6; } else { @@ -340,6 +348,8 @@ std::pair Translate(const std::wstring& text, DevTools* devT return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetLang), S(OuterHTML)) }; else if (errorCode == 5) return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, 2*timerStop) }; + else if (errorCode == 6) + return { false, FormatString(L"%s", ERROR_EMPTY_ANSWER) }; else return { false, FormatString(L"%s", TRANSLATION_ERROR) }; } \ No newline at end of file diff --git a/extensions/devtoolswrapper.cpp b/extensions/devtoolswrapper.cpp index 546d5a2..19fe29c 100644 --- a/extensions/devtoolswrapper.cpp +++ b/extensions/devtoolswrapper.cpp @@ -62,6 +62,11 @@ void EraseControlCharacters(std::wstring& text) { text.erase(it--); } + if (*it == '\\' && *(it + 1) == 'n') + { + text.erase((it + 1)--); + text.erase(it--); + } } } From 7eb45899c001e5dae68602948f3f8ef6b66a3610 Mon Sep 17 00:00:00 2001 From: zeheyler Date: Sun, 13 Dec 2020 00:56:00 +0300 Subject: [PATCH 12/14] fix target language check Fixed target language check --- extensions/devtoolsdeepltranslate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index f60b52a..7cb065e 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -298,8 +298,8 @@ std::pair Translate(const std::wstring& text, DevTools* devT { errorCode = 4; } + break; } - break; } } From fc81b17a3cc03cabe6879f5638e0e7e1e3b3c16b Mon Sep 17 00:00:00 2001 From: Akash Mozumdar Date: Fri, 15 Jan 2021 06:07:23 -0700 Subject: [PATCH 13/14] massive refactor, also fix newline issue and google translate throw and different number formats for process id --- GUI/CMakeLists.txt | 4 +- GUI/extenwindow.cpp | 2 - GUI/extenwindow.h | 1 - GUI/host/hookcode.cpp | 3 +- GUI/host/host.cpp | 9 +- GUI/host/textthread.cpp | 19 +- GUI/mainwindow.cpp | 20 +- extensions/CMakeLists.txt | 18 +- extensions/bingtranslate.cpp | 7 +- extensions/deepltranslate.cpp | 4 +- extensions/devtools.cpp | 414 ++++++++----------------- extensions/devtools.h | 60 +--- extensions/devtoolsdeepltranslate.cpp | 416 ++++++-------------------- extensions/devtoolswrapper.cpp | 233 --------------- extensions/extrawindow.cpp | 5 +- extensions/googletranslate.cpp | 9 +- extensions/lua.cpp | 2 +- extensions/network.cpp | 4 +- extensions/network.h | 21 +- extensions/regexfilter.cpp | 4 +- extensions/removerepeatphrase.cpp | 6 +- extensions/replacer.cpp | 5 +- extensions/threadlinker.cpp | 6 +- extensions/translatewrapper.cpp | 32 +- include/common.h | 4 +- text.cpp | 21 +- texthook/engine/match.cc | 5 +- texthook/texthook.cc | 4 +- 28 files changed, 351 insertions(+), 987 deletions(-) delete mode 100644 extensions/devtoolswrapper.cpp diff --git a/GUI/CMakeLists.txt b/GUI/CMakeLists.txt index 86ea2bf..944e492 100644 --- a/GUI/CMakeLists.txt +++ b/GUI/CMakeLists.txt @@ -14,10 +14,10 @@ add_executable(Textractor WIN32 Textractor.ico ) target_precompile_headers(Textractor REUSE_FROM pch) -target_link_libraries(${PROJECT_NAME} Qt5::Widgets shell32 winhttp) +target_link_libraries(Textractor Qt5::Widgets shell32 winhttp) if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll) if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll) - install_qt5_libs(${PROJECT_NAME}) + install_qt5_libs(Textractor) endif() endif() diff --git a/GUI/extenwindow.cpp b/GUI/extenwindow.cpp index 5925af1..019830c 100644 --- a/GUI/extenwindow.cpp +++ b/GUI/extenwindow.cpp @@ -138,8 +138,6 @@ ExtenWindow::ExtenWindow(QWidget* parent) : Sync(); } -ExtenWindow::~ExtenWindow() = default; - bool ExtenWindow::eventFilter(QObject* target, QEvent* event) { // See https://stackoverflow.com/questions/1224432/how-do-i-respond-to-an-internal-drag-and-drop-operation-using-a-qlistwidget/1528215 diff --git a/GUI/extenwindow.h b/GUI/extenwindow.h index eb52395..7561092 100644 --- a/GUI/extenwindow.h +++ b/GUI/extenwindow.h @@ -20,7 +20,6 @@ class ExtenWindow : public QMainWindow { public: explicit ExtenWindow(QWidget* parent = nullptr); - ~ExtenWindow(); private: bool eventFilter(QObject* target, QEvent* event) override; diff --git a/GUI/host/hookcode.cpp b/GUI/host/hookcode.cpp index 35fb50a..c7a176d 100644 --- a/GUI/host/hookcode.cpp +++ b/GUI/host/hookcode.cpp @@ -129,8 +129,7 @@ namespace { size_t size = 0; int value = 0; - try { value = std::stoi(HCode, &size, 16); } - catch (std::invalid_argument) {} + try { value = std::stoi(HCode, &size, 16); } catch (std::invalid_argument) {} HCode.erase(0, size); return value; }; diff --git a/GUI/host/host.cpp b/GUI/host/host.cpp index 83e25a8..54046bf 100644 --- a/GUI/host/host.cpp +++ b/GUI/host/host.cpp @@ -32,8 +32,7 @@ namespace { if (!view) return {}; std::scoped_lock lock(viewMutex); - for (auto hook : view) - if (hook.address == addr) return hook; + for (auto hook : view) if (hook.address == addr) return hook; return {}; } @@ -69,8 +68,7 @@ namespace void RemoveThreads(std::function removeIf) { std::vector threadsToRemove; - for (auto& [tp, thread] : textThreadsByParams.Acquire().contents) - if (removeIf(tp)) threadsToRemove.push_back(&thread); + for (auto& [tp, thread] : textThreadsByParams.Acquire().contents) if (removeIf(tp)) threadsToRemove.push_back(&thread); for (auto thread : threadsToRemove) { OnDestroy(*thread); @@ -250,8 +248,7 @@ namespace Host TextThread* GetThread(int64_t handle) { - for (auto& [tp, thread] : textThreadsByParams.Acquire().contents) - if (thread.handle == handle) return &thread; + for (auto& [tp, thread] : textThreadsByParams.Acquire().contents) if (thread.handle == handle) return &thread; return nullptr; } diff --git a/GUI/host/textthread.cpp b/GUI/host/textthread.cpp index 7a70c4a..c3adc2b 100644 --- a/GUI/host/textthread.cpp +++ b/GUI/host/textthread.cpp @@ -22,7 +22,7 @@ TextThread::TextThread(ThreadParam tp, HookParam hp, std::optional void TextThread::Start() { - CreateTimerQueueTimer(&timer, NULL, [](void* This, BOOLEAN) { ((TextThread*)This)->Flush(); }, this, 10, 10, WT_EXECUTELONGFUNCTION); + CreateTimerQueueTimer(&timer, NULL, [](void* This, auto) { ((TextThread*)This)->Flush(); }, this, 10, 10, WT_EXECUTELONGFUNCTION); } void TextThread::Stop() @@ -42,8 +42,21 @@ void TextThread::Push(BYTE* data, int length) BYTE doubleByteChar[2]; if (length == 1) // doublebyte characters must be processed as pairs - if (leadByte) std::tie(doubleByteChar[0], doubleByteChar[1], data, length, leadByte) = std::tuple(leadByte, data[0], doubleByteChar, 2, 0); - else if (IsDBCSLeadByteEx(hp.codepage ? hp.codepage : Host::defaultCodepage, data[0])) std::tie(leadByte, length) = std::tuple(data[0], 0); + { + if (leadByte) + { + doubleByteChar[0] = leadByte; + doubleByteChar[1] = data[0]; + data = doubleByteChar; + length = 2; + leadByte = 0; + } + else if (IsDBCSLeadByteEx(hp.codepage ? hp.codepage : Host::defaultCodepage, data[0])) + { + leadByte = data[0]; + length = 0; + } + } if (hp.type & HEX_DUMP) for (int i = 0; i < length; i += sizeof(short)) buffer.append(FormatString(L"%04hX ", *(short*)(data + i))); else if (hp.type & USING_UNICODE) buffer.append((wchar_t*)data, length / sizeof(wchar_t)); diff --git a/GUI/mainwindow.cpp b/GUI/mainwindow.cpp index 5cfb22c..abb577f 100644 --- a/GUI/mainwindow.cpp +++ b/GUI/mainwindow.cpp @@ -17,7 +17,7 @@ extern const char* ATTACH; extern const char* LAUNCH; -extern const char* GAME_CONFIG; +extern const char* CONFIG; extern const char* DETACH; extern const char* FORGET; extern const char* ADD_HOOK; @@ -198,14 +198,14 @@ namespace CloseHandle(info.hThread); } - void OpenProcessConfig() + void ConfigureProcess() { if (auto processName = GetModuleFilename(selectedProcessId)) if (int last = processName->rfind(L'\\') + 1) { std::wstring configFile = std::wstring(processName.value()).replace(last, std::string::npos, GAME_CONFIG_FILE); if (!std::filesystem::exists(configFile)) QTextFile(S(configFile), QFile::WriteOnly).write("see https://github.com/Artikash/Textractor/wiki/Game-configuration-file"); if (std::filesystem::exists(configFile)) _wspawnlp(_P_DETACH, L"notepad", L"notepad", configFile.c_str(), NULL); - else QMessageBox::critical(This, GAME_CONFIG, QString(FAILED_TO_CREATE_CONFIG_FILE).arg(S(configFile))); + else QMessageBox::critical(This, CONFIG, QString(FAILED_TO_CREATE_CONFIG_FILE).arg(S(configFile))); } } @@ -258,8 +258,7 @@ namespace hookList->setAttribute(Qt::WA_DeleteOnClose); hookList->setMinimumSize({ 300, 50 }); hookList->setWindowTitle(DOUBLE_CLICK_TO_REMOVE_HOOK); - for (auto [address, hp] : hooks) - new QListWidgetItem(QString(hp.name) + "@" + QString::number(address, 16), hookList); + for (auto [address, hp] : hooks) new QListWidgetItem(QString(hp.name) + "@" + QString::number(address, 16), hookList); QObject::connect(hookList, &QListWidget::itemDoubleClicked, [processId, hookList](QListWidgetItem* item) { try @@ -335,8 +334,7 @@ namespace layout.addRow(&confirm); if (!dialog.exec()) return; wcsncpy_s(sp.text, S(textInput.text()).c_str(), PATTERN_SIZE - 1); - try { Host::FindHooks(selectedProcessId, sp); } - catch (std::out_of_range) {} + try { Host::FindHooks(selectedProcessId, sp); } catch (std::out_of_range) {} return; } @@ -408,8 +406,7 @@ namespace catch (std::out_of_range) { return; } std::thread([hooks] { - for (int lastSize = 0; hooks->size() == 0 || hooks->size() != lastSize; Sleep(2000)) - lastSize = hooks->size(); + for (int lastSize = 0; hooks->size() == 0 || hooks->size() != lastSize; Sleep(2000)) lastSize = hooks->size(); QString saveFileName; QMetaObject::invokeMethod(This, [&] @@ -562,6 +559,7 @@ namespace bool SentenceReceived(TextThread& thread, std::wstring& sentence) { + for (int i = 0; i < sentence.size(); ++i) if (sentence[i] == '\r' && sentence[i + 1] == '\n') sentence[i] = 0x200b; // for some reason \r appears as newline - no need to double if (!DispatchSentenceToExtensions(sentence, GetSentenceInfo(thread).data())) return false; sentence += L'\n'; if (&thread == current) QMetaObject::invokeMethod(This, [sentence = S(sentence)]() mutable @@ -598,7 +596,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent) for (auto [text, slot] : Array{ { ATTACH, AttachProcess }, { LAUNCH, LaunchProcess }, - { GAME_CONFIG, OpenProcessConfig }, + { CONFIG, ConfigureProcess }, { DETACH, DetachProcess }, { FORGET, ForgetProcess }, { ADD_HOOK, AddHook }, @@ -647,7 +645,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent) for (int i = 0; i < argc; ++i) if (std::wstring arg = argv[i]; arg[0] == L'/' || arg[0] == L'-') if (arg[1] == L'p' || arg[1] == L'P') - if (DWORD processId = _wtoi(arg.substr(2).c_str())) Host::InjectProcess(processId); + if (DWORD processId = wcstoul(arg.substr(2).c_str(), nullptr, 0)) Host::InjectProcess(processId); else for (auto [processId, processName] : processes) if (processName.value_or(L"").find(L"\\" + arg.substr(2)) != std::string::npos) Host::InjectProcess(processId); diff --git a/extensions/CMakeLists.txt b/extensions/CMakeLists.txt index e7367d0..b1bef90 100644 --- a/extensions/CMakeLists.txt +++ b/extensions/CMakeLists.txt @@ -1,16 +1,15 @@ include(QtUtils) msvc_registry_search() find_qt5(Core Widgets WebSockets) -set(CMAKE_AUTOMOC ON) cmake_policy(SET CMP0037 OLD) add_library(Bing\ Translate MODULE bingtranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) add_library(Copy\ to\ Clipboard MODULE copyclipboard.cpp extensionimpl.cpp) add_library(DeepL\ Translate MODULE deepltranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) +add_library(DevTools\ DeepL\ Translate MODULE devtoolsdeepltranslate.cpp devtools.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) add_library(Extra\ Newlines MODULE extranewlines.cpp extensionimpl.cpp) add_library(Extra\ Window MODULE extrawindow.cpp extensionimpl.cpp) add_library(Google\ Translate MODULE googletranslate.cpp translatewrapper.cpp network.cpp extensionimpl.cpp) -add_library(DevTools\ DeepL\ Translate MODULE devtoolsdeepltranslate.cpp devtoolswrapper.cpp devtools.cpp network.cpp extensionimpl.cpp) add_library(Lua MODULE lua.cpp extensionimpl.cpp) add_library(Regex\ Filter MODULE regexfilter.cpp extensionimpl.cpp) add_library(Remove\ Repeated\ Characters MODULE removerepeatchar.cpp extensionimpl.cpp) @@ -23,10 +22,10 @@ add_library(Thread\ Linker MODULE threadlinker.cpp extensionimpl.cpp) target_precompile_headers(Bing\ Translate REUSE_FROM pch) target_precompile_headers(Copy\ to\ Clipboard REUSE_FROM pch) target_precompile_headers(DeepL\ Translate REUSE_FROM pch) +target_precompile_headers(DevTools\ DeepL\ Translate REUSE_FROM pch) target_precompile_headers(Extra\ Newlines REUSE_FROM pch) target_precompile_headers(Extra\ Window REUSE_FROM pch) target_precompile_headers(Google\ Translate REUSE_FROM pch) -target_precompile_headers(DevTools\ DeepL\ Translate REUSE_FROM pch) target_precompile_headers(Lua REUSE_FROM pch) target_precompile_headers(Regex\ Filter REUSE_FROM pch) target_precompile_headers(Remove\ Repeated\ Characters REUSE_FROM pch) @@ -38,18 +37,15 @@ target_precompile_headers(Thread\ Linker REUSE_FROM pch) target_link_libraries(Bing\ Translate winhttp Qt5::Widgets) target_link_libraries(DeepL\ Translate winhttp Qt5::Widgets) +target_link_libraries(DevTools\ DeepL\ Translate shell32 winhttp Qt5::Widgets Qt5::WebSockets) target_link_libraries(Extra\ Window Qt5::Widgets) target_link_libraries(Google\ Translate winhttp Qt5::Widgets) -target_link_libraries(DevTools\ DeepL\ Translate winhttp Qt5::Widgets Qt5::WebSockets) target_link_libraries(Lua lua53 Qt5::Widgets) target_link_libraries(Regex\ Filter Qt5::Widgets) target_link_libraries(Thread\ Linker Qt5::Widgets) if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll) - add_custom_command(TARGET DevTools\ DeepL\ Translate - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" - COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin - COMMAND Qt5::windeployqt --dir $ "$/DevTools\ DeepL\ Translate.dll" - ) -endif() \ No newline at end of file + if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll) + install_qt5_libs(DevTools\ DeepL\ Translate) + endif() +endif() diff --git a/extensions/bingtranslate.cpp b/extensions/bingtranslate.cpp index cb0983f..6571ad1 100644 --- a/extensions/bingtranslate.cpp +++ b/extensions/bingtranslate.cpp @@ -1,5 +1,4 @@ -#include "extension.h" -#include "network.h" +#include "network.h" #include extern const wchar_t* TRANSLATION_ERROR; @@ -81,7 +80,7 @@ QStringList languages }; bool translateSelectedOnly = false, rateLimitAll = true, rateLimitSelected = false, useCache = true; -int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 500; +int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 1000; std::pair Translate(const std::wstring& text) { @@ -94,10 +93,8 @@ std::pair Translate(const std::wstring& text) FormatString(R"([{"text":"%s"}])", JSON::Escape(WideStringToString(text))), FormatString(L"Content-Type: application/json; charset=UTF-8\r\nOcp-Apim-Subscription-Key:%s", apiKey.Copy()).c_str() }) - { if (auto translation = Copy(JSON::Parse(httpRequest.response)[0][L"translations"][0][L"text"].String())) return { true, translation.value() }; else return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, httpRequest.response) }; - } else return { false, FormatString(L"%s (code=%u)", TRANSLATION_ERROR, httpRequest.errorCode) }; if (HttpRequest httpRequest{ diff --git a/extensions/deepltranslate.cpp b/extensions/deepltranslate.cpp index 831d9ad..9a91786 100644 --- a/extensions/deepltranslate.cpp +++ b/extensions/deepltranslate.cpp @@ -1,5 +1,4 @@ #include "qtcommon.h" -#include "extension.h" #include "network.h" #include @@ -26,7 +25,7 @@ QStringList languages }; bool translateSelectedOnly = true, rateLimitAll = true, rateLimitSelected = true, useCache = true; -int tokenCount = 10, tokenRestoreDelay = 60000, maxSentenceSize = 500; +int tokenCount = 10, tokenRestoreDelay = 60000, maxSentenceSize = 1000; enum KeyType { CAT, REST }; int keyType = CAT; @@ -90,6 +89,7 @@ std::pair Translate(const std::wstring& text) L"/jsonrpc", body, L"Host: www2.deepl.com\r\nAccept-Language: en-US,en;q=0.5\r\nContent-type: application/json; charset=utf-8\r\nOrigin: https://www.deepl.com\r\nTE: Trailers", + INTERNET_DEFAULT_PORT, L"https://www.deepl.com/translator", WINHTTP_FLAG_SECURE }) diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index 8fa4d83..5c1ba36 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -1,310 +1,154 @@ #include "devtools.h" +#include +#include +#include -DevTools::DevTools(QObject* parent) : - QObject(parent), - idCounter(0), - idMethod(0), - status("Stopped"), - session(0) +namespace { - connect(&webSocket, &QWebSocket::stateChanged, this, &DevTools::stateChanged); - connect(&webSocket, &QWebSocket::textMessageReceived, this, &DevTools::onTextMessageReceived); + std::function OnStatusChanged = Swallow; + PROCESS_INFORMATION processInfo = {}; + std::atomic idCounter = 0, idMethod = 0; + std::mutex devToolsMutex; + QWebSocket webSocket; + std::unordered_map>> mapQueue; + std::unordered_map>> mapMethod; + auto _ = ([] + { + QObject::connect(&webSocket, &QWebSocket::stateChanged, + [](QAbstractSocket::SocketState state) { OnStatusChanged(QMetaEnum::fromType().valueToKey(state)); } + ); + QObject::connect(&webSocket, &QWebSocket::textMessageReceived, [](QString message) + { + auto result = JSON::Parse(S(message)); + std::scoped_lock lock(devToolsMutex); + if (auto method = result[L"method"].String()) + for (auto& [listenTo, results] : mapMethod) if (*method == listenTo) results.push_back(result[L"params"]); + if (auto id = result[L"id"].Number()) + if (auto request = mapQueue.find((int)*id); request != mapQueue.end()) + request->second.set(result), mapQueue.erase(request); + }); + }(), 0); } -void DevTools::startDevTools(QString path, bool headless, int port) +namespace DevTools { - if (startChrome(path, headless, port)) + void Start(const std::wstring& path, std::function statusChanged, bool headless, int port) { - QJsonDocument doc; - QString webSocketDebuggerUrl; - if (GetJsonfromHTTP(doc, "/json/list", port)) + OnStatusChanged = statusChanged; + DWORD exitCode = 0; + auto args = FormatString( + L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=%d", + path, + std::filesystem::current_path().wstring(), + port + ); + if (headless) args += L"--headless"; + STARTUPINFOW DUMMY = { sizeof(DUMMY) }; + if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) || + CreateProcessW(NULL, args.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &DUMMY, &processInfo) + ) { - for (const auto obj : doc.array()) - if (obj.toObject().value("type") == "page") + if (HttpRequest httpRequest{ + L"Mozilla/5.0 Textractor", + L"127.0.0.1", + L"POST", + L"/json/list", + "", + NULL, + (DWORD)port, + NULL, + WINHTTP_FLAG_ESCAPE_DISABLE + }) + { + if (auto list = Copy(JSON::Parse(httpRequest.response).Array())) if (auto it = std::find_if( + list->begin(), + list->end(), + [](const JSON::Value& object) { return object[L"type"].String() && *object[L"type"].String() == L"page" && object[L"webSocketDebuggerUrl"].String(); } + ); it != list->end()) { - webSocketDebuggerUrl.append(obj.toObject().value("webSocketDebuggerUrl").toString()); - break; + std::scoped_lock lock(devToolsMutex); + webSocket.open(S(*(*it)[L"webSocketDebuggerUrl"].String())); + if (HttpRequest httpRequest{ + L"Mozilla/5.0 Textractor", + L"127.0.0.1", + L"POST", + L"/json/version", + "", + NULL, + (DWORD)port, + NULL, + WINHTTP_FLAG_ESCAPE_DISABLE + }) + if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String())) + if (userAgent->find(L"Headless") != std::string::npos) + SendRequest(L"Network.setUserAgentOverride", FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L""))); + return; } - if (!webSocketDebuggerUrl.isEmpty()) - { - if (GetJsonfromHTTP(doc, "/json/version", port)) - { - userAgent = doc.object().value("User-Agent").toString(); - } - webSocket.open(webSocketDebuggerUrl); - session += 1; - return; } + OnStatusChanged("Failed Connection"); } - status = "Failed to find Chrome debug port!"; - emit statusChanged(status); - } - else - { - status = "Failed to start Chrome!"; - emit statusChanged(status); - } -} - -int DevTools::getSession() -{ - return session; -} - -QString DevTools::getStatus() -{ - return status; -} - -QString DevTools::getUserAgent() -{ - return userAgent; -} - -DevTools::~DevTools() -{ - closeDevTools(); -} - -bool DevTools::startChrome(QString path, bool headless, int port) -{ - if (!std::filesystem::exists(path.toStdWString())) - return false; - DWORD exitCode = 0; - if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE && exitCode == STILL_ACTIVE) - return false; - QString args = "--proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=" - + QString::fromStdWString(std::filesystem::current_path()) - + "\\devtoolscache --remote-debugging-port=" - + QString::number(port); - if (headless) - args += " --headless"; - STARTUPINFOW dummy = { sizeof(dummy) }; - if (!CreateProcessW(NULL, (wchar_t*)(path + " " + args).utf16(), nullptr, nullptr, - FALSE, 0, nullptr, nullptr, &dummy, &processInfo)) - return false; - else - return true; -} - -bool DevTools::GetJsonfromHTTP(QJsonDocument& doc, QString object, int port) -{ - if (HttpRequest httpRequest{ - L"Mozilla/5.0 Textractor", - L"127.0.0.1", - L"POST", - object.toStdWString().c_str(), - "", - NULL, - NULL, - WINHTTP_FLAG_ESCAPE_DISABLE, - NULL, - NULL, - DWORD(port) - }) - { - QString qtString = QString::fromStdWString(httpRequest.response); - doc = QJsonDocument::fromJson(qtString.toUtf8()); - if (!doc.isEmpty()) - return true; else - return false; - } - else - return false; - -} - -void DevTools::stateChanged(QAbstractSocket::SocketState state) -{ - QMetaEnum metaenum = QMetaEnum::fromType(); - status = metaenum.valueToKey(state); - emit statusChanged(status); -} - -bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root) -{ - if (!isConnected()) - return false; - root = QJsonObject(); - QJsonObject json; - task_completion_event response; - long id = idIncrement(); - json.insert("id", id); - json.insert("method", method); - json.insert("params", params); - QJsonDocument doc(json); - QString message(doc.toJson(QJsonDocument::Compact)); - mutex.lock(); - mapQueue.insert(std::make_pair(id, response)); - mutex.unlock(); - webSocket.sendTextMessage(message); - webSocket.flush(); - try - { - root = create_task(response).get(); - } - catch (const std::exception& ex) - { - response.set_exception(ex); - return false; - } - if (!root.isEmpty()) - { - if (root.contains("error")) { - return false; - } - else if (root.contains("result")) - return true; - else - return false; - } - else - return false; -} - -long DevTools::methodToReceive(QString method, QJsonObject params) -{ - QJsonObject json; - long id = idmIncrement(); - json.insert("method", method); - json.insert("params", params); - mutex.lock(); - mapMethod.insert(std::make_pair(id, json)); - mutex.unlock(); - return id; -} - -long DevTools::idIncrement() -{ - return ++idCounter; -} - -long DevTools::idmIncrement() -{ - return ++idMethod; -} - -bool DevTools::isConnected() -{ - if (webSocket.state() == QAbstractSocket::ConnectedState) - return true; - else - return false; -} - -bool DevTools::compareJson(QJsonValue stored, QJsonValue params) -{ - if (stored.isObject()) - { - foreach(const QString & key, stored.toObject().keys()) - { - QJsonValue storedvalue = stored.toObject().value(key); - QJsonValue value = params.toObject().value(key); - if (!compareJson(storedvalue, value)) - return false; + OnStatusChanged("Failed Startup"); } } - else if (stored.isArray()) - { - for (int i = 0; i < stored.toArray().size(); i++) - if (!compareJson(stored.toArray()[i], params.toArray()[i])) - return false; - } - else if (stored.isString()) - { - if (!stored.toString().contains(params.toString())) - return false; - } - else if (stored.toVariant() != params.toVariant()) - return false; - return true; -} - -bool DevTools::checkMethod(long id) -{ - MapMethod::iterator iter = mapMethod.find(id); - if (iter == mapMethod.end()) - return true; - else - return false; -} - -void DevTools::onTextMessageReceived(QString message) -{ - QJsonDocument doc = QJsonDocument::fromJson(message.toUtf8()); - if (doc.isObject()) + void Close() { - QJsonObject root = doc.object(); - if (root.contains("method")) - { - for (auto iter = mapMethod.cbegin(); iter != mapMethod.cend();) - { - if (iter->second.value("method") == root.value("method") - && compareJson(iter->second.value("params"), root.value("params"))) - { - mutex.lock(); - mapMethod.erase(iter++); - mutex.unlock(); - } - ++iter; - } - return; - } - if (root.contains("id")) - { - long id = root.value("id").toInt(); - MapResponse::iterator request = mapQueue.find(id); - if (request != mapQueue.end()) - { - request->second.set(root); - mutex.lock(); - mapQueue.erase(request); - mutex.unlock(); - } - return; - } - } -} - -void DevTools::closeDevTools() -{ - if (this->mapQueue.size() > 0) - { - MapResponse::iterator iter = mapQueue.begin(); - MapResponse::iterator iend = mapQueue.end(); - for (; iter != iend; iter++) - { - iter->second.set_exception("exception"); - } - } - webSocket.close(); - mapMethod.clear(); - mapQueue.clear(); - idCounter = 0; - idMethod = 0; - DWORD exitCode = 0; - if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE) - { - if (exitCode == STILL_ACTIVE) + std::scoped_lock lock(devToolsMutex); + for (const auto& [_, task] : mapQueue) task.set_exception(std::runtime_error("closed")); + webSocket.close(); + mapMethod.clear(); + mapQueue.clear(); + DWORD exitCode = 0; + if (GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) { TerminateProcess(processInfo.hProcess, 0); WaitForSingleObject(processInfo.hProcess, 100); CloseHandle(processInfo.hProcess); CloseHandle(processInfo.hThread); } - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - try - { - std::filesystem::remove_all(L"devtoolscache"); - } - catch (const std::exception&) - { - - } + try { std::filesystem::remove_all(L"devtoolscache"); } catch (std::filesystem::filesystem_error) {} + OnStatusChanged("Stopped"); } - status = "Stopped"; - emit statusChanged(status); -} \ No newline at end of file + + bool Connected() + { + std::scoped_lock lock(devToolsMutex); + return webSocket.state() == QAbstractSocket::ConnectedState; + } + + JSON::Value SendRequest(const std::wstring& method, const std::wstring& params) + { + if (webSocket.state() != QAbstractSocket::ConnectedState) return {}; + concurrency::task_completion_event> response; + int id = idCounter +=1; + auto message = FormatString(LR"({"id":%d,"method":"%s","params":%s})", id, method, params); + { + std::scoped_lock lock(devToolsMutex); + mapQueue.try_emplace(id, response); + webSocket.sendTextMessage(S(message)); + webSocket.flush(); + } + try { if (auto result = create_task(response).get()[L"result"]) return result; } catch (...) {} + return {}; + } + + void StartListening(const std::wstring& method) + { + std::scoped_lock lock(devToolsMutex); + mapMethod.try_emplace(method); + } + + std::vector> ListenResults(const std::wstring& method) + { + std::scoped_lock lock(devToolsMutex); + return mapMethod[method]; + } + + void StopListening(const std::wstring& method) + { + std::scoped_lock lock(devToolsMutex); + mapMethod.erase(method); + } +} diff --git a/extensions/devtools.h b/extensions/devtools.h index c13b610..ae7d6dd 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -1,51 +1,13 @@ -#include -#include -#include +#include "qtcommon.h" #include "network.h" -using namespace Concurrency; - -typedef std::map> MapResponse; -typedef std::map MapMethod; - -class DevTools : public QObject { - Q_OBJECT -public: - explicit DevTools(QObject* parent = nullptr); - ~DevTools(); - -Q_SIGNALS: - void statusChanged(const QString&); - -private Q_SLOTS: - void stateChanged(QAbstractSocket::SocketState state); - void onTextMessageReceived(QString message); - -public: - void startDevTools(QString path, bool headless = false, int port = 9222); - void closeDevTools(); - bool checkMethod(long id); - int getSession(); - bool SendRequest(QString method, QJsonObject params, QJsonObject& root); - long methodToReceive(QString method, QJsonObject params = {}); - QString getStatus(); - QString getUserAgent(); - -private: - bool isConnected(); - bool startChrome(QString path, bool headless = false, int port = 9222); - bool GetJsonfromHTTP(QJsonDocument& doc, QString object, int port = 9222); - long idIncrement(); - long idmIncrement(); - bool compareJson(QJsonValue stored, QJsonValue params); - int session; - QWebSocket webSocket; - std::mutex mutex; - MapResponse mapQueue; - MapMethod mapMethod; - long idCounter; - long idMethod; - PROCESS_INFORMATION processInfo; - QString status; - QString userAgent; -}; \ No newline at end of file +namespace DevTools +{ + void Start(const std::wstring& path, std::function statusChanged, bool headless, int port); + void Close(); + bool Connected(); + JSON::Value SendRequest(const std::wstring& method, const std::wstring& params = L"{}"); + void StartListening(const std::wstring& method); + std::vector> ListenResults(const std::wstring& method); + void StopListening(const std::wstring& method); +} diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 7cb065e..743b4e1 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -1,24 +1,26 @@ #include "qtcommon.h" -#include "extension.h" #include "devtools.h" +#include extern const wchar_t* TRANSLATION_ERROR; -extern Synchronized translateTo, translateFrom; - -bool useCache = true, autoStartChrome = false, headlessChrome = true; -int maxSentenceSize = 500, chromePort = 9222; +extern Synchronized translateTo; +extern QFormLayout* display; +extern Settings settings; const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate"; -const wchar_t* ERROR_CHROME = L"Error: Chrome not started"; -const wchar_t* ERROR_START_CHROME = L"Error: failed to start Chrome or to connect to it"; -const wchar_t* ERROR_GOT_TIMEOUT = L"Error: timeout (s)"; -const wchar_t* ERROR_COMMAND_FAIL = L"Error: command failed"; -const wchar_t* ERROR_LANGUAGE = L"Error: target languages do not match"; -const wchar_t* ERROR_NOTE = L"Error: notification"; -const wchar_t* ERROR_EMPTY_ANSWER = L"Error: empty translation"; +const char* GET_API_KEY_FROM = nullptr; +bool translateSelectedOnly = true, rateLimitAll = false, rateLimitSelected = false, useCache = true; +int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 10000; -QString URL = "https://www.deepl.com/en/translator"; -QStringList languagesTo +extern const char* CHROME_LOCATION; +extern const char* START_DEVTOOLS; +extern const char* STOP_DEVTOOLS; +extern const char* HEADLESS_MODE; +extern const char* DEVTOOLS_STATUS; +extern const char* AUTO_START; +extern const wchar_t* ERROR_START_CHROME; + +QStringList languages { "Chinese (simplified): zh", "Dutch: nl", @@ -33,323 +35,97 @@ QStringList languagesTo "Spanish: es", }; -QStringList languagesFrom +BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) { - "Any language: ", - "Chinese: zh", - "Dutch: nl", - "English: en", - "French: fr", - "German: de", - "Italian: it", - "Japanese: ja", - "Polish: pl", - "Portuguese: pt", - "Russian: ru", - "Spanish: es", -}; - -int docFound = -1, targetNodeId = -1, session = -1, pageEnabled = -1, userAgentFlag = -1, backup = -1, sourceLangId = -1, mobileShareId = -1; -long update = -1, callNumber = 0; -std::vector callQueue; - -std::pair Translate(const std::wstring& text, DevTools* devTools) -{ - QString qtext = S(text); - qtext.remove(QString(12288)); // japanese space (no need for translator) - qtext.replace(QString(12289), ","); // replace the japanese comma with the latin comma for correct translation - - // Remove quotes - bool checkQuote = false; - if ((qtext.front() == QString(12300) && qtext.back() == QString(12301)) // japanese quotation marks - || (qtext.front() == "\"" && qtext.back() == "\"")) + switch (ul_reason_for_call) { - checkQuote = true; - qtext.remove(0, 1); - qtext.chop(1); - } - - // Check specific cases (sentence has only one japanese symbol or consists of ellipsis) - int count = qtext.count(QString(8230)); // ellipsis - if (count == qtext.length() - || (count == (qtext.length() - 1) && qtext.back() == QString(12290))) // japanese end of a sentence + case DLL_PROCESS_ATTACH: { - return { true, text }; - } - if (count == (qtext.length() - 1)) - { - qtext.remove(QString(8230)); - qtext += QString(12290) + QString(8230); // add the end symbol for correct translation - } - - // Put quotes back - if (checkQuote) - { - qtext = "\"" + qtext + "\""; - } - - // Check status - if (devTools->getStatus() == "Stopped") - { - return { false, FormatString(L"%s", ERROR_CHROME) }; - } - if (devTools->getStatus().startsWith("Fail") || devTools->getStatus().startsWith("Unconnected")) - { - return { false, FormatString(L"%s", ERROR_START_CHROME) }; - } - if (session != devTools->getSession()) - { - session = devTools->getSession(); - docFound = -1; - targetNodeId = -1; - sourceLangId = -1; - mobileShareId = -1; - pageEnabled = -1; - userAgentFlag = -1; - update = -1; - callNumber = 0; - backup = -1; - } - - // Remove tags and reduce the number of ellipsis for correct translation - qtext.remove(QRegExp("<[^>]*>")); - qtext.replace(QRegExp("(" + QString(8230) + ")+"), " " + QString(8230)); - - // Enable page feedback - QJsonObject root; - int errorCode = 0; - if (pageEnabled == -1) - { - if (!devTools->SendRequest("Page.enable", {}, root)) - errorCode = 1; - else - pageEnabled = 1; - } - - // Change user-agent if in headless mode - if (userAgentFlag == -1 && errorCode == 0) - { - QString userAgent = devTools->getUserAgent(); - if (!userAgent.isEmpty()) + QString chromePath = settings.value(CHROME_LOCATION).toString(); + wchar_t programFiles[MAX_PATH + 100] = {}; + if (chromePath.isEmpty()) for (auto folder : { CSIDL_PROGRAM_FILESX86, CSIDL_PROGRAM_FILES, CSIDL_LOCAL_APPDATA }) { - userAgent.replace("HeadlessChrome", "Chrome"); - if (!devTools->SendRequest("Network.setUserAgentOverride", { {"userAgent", userAgent} }, root)) - errorCode = 1; - else - userAgentFlag = 1; + SHGetFolderPathW(NULL, folder, NULL, SHGFP_TYPE_CURRENT, programFiles); + wcscat_s(programFiles, L"/Google/Chrome/Application/chrome.exe"); + if (std::filesystem::exists(programFiles)) chromePath = S(programFiles); } + auto chromePathEdit = new QLineEdit(chromePath); + QObject::connect(chromePathEdit, &QLineEdit::textChanged, [chromePathEdit](QString path) { settings.setValue(CHROME_LOCATION, path); }); + display->addRow(CHROME_LOCATION, chromePathEdit); + auto statusLabel = new QLabel("Stopped"); + auto startButton = new QPushButton(START_DEVTOOLS), stopButton = new QPushButton(STOP_DEVTOOLS); + auto headlessCheckBox = new QCheckBox(HEADLESS_MODE); + headlessCheckBox->setChecked(settings.value(HEADLESS_MODE, true).toBool()); + QObject::connect(headlessCheckBox, &QCheckBox::clicked, [](bool headless) { settings.setValue(HEADLESS_MODE, headless); }); + QObject::connect(startButton, &QPushButton::clicked, [statusLabel, chromePathEdit, headlessCheckBox] { + DevTools::Start( + S(chromePathEdit->text()), + [statusLabel](QString status) { QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status)); }, + headlessCheckBox->isChecked(), + 9222 + ); + if (!DevTools::SendRequest(L"Network.enable")) DevTools::Close(); + }); + QObject::connect(stopButton, &QPushButton::clicked, &DevTools::Close); + auto buttons = new QHBoxLayout(); + buttons->addWidget(startButton); + buttons->addWidget(stopButton); + display->addRow(buttons); + display->addRow(headlessCheckBox); + auto autoStartButton = new QCheckBox(AUTO_START); + autoStartButton->setChecked(settings.value(AUTO_START, false).toBool()); + QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) {settings.setValue(AUTO_START, autoStart); }); + display->addRow(autoStartButton); + statusLabel->setFrameStyle(QFrame::Panel | QFrame::Sunken); + display->addRow(DEVTOOLS_STATUS, statusLabel); + if (autoStartButton->isChecked()) startButton->click(); } - - // Increase queue counter and wait until previous calls are done - float timer = 0; - int timerStop = 10; - long callTag = ++callNumber; - callQueue.insert(callQueue.begin(), callTag); - while (errorCode == 0 && callQueue.back() != callTag && timer < 2 * timerStop) + break; + case DLL_PROCESS_DETACH: { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - timer += 0.1; + DevTools::Close(); } - if (timer >= 2 * timerStop) - errorCode = 5; + break; + } + return TRUE; +} - // Set methods to receive - long navigate = devTools->methodToReceive("Page.navigatedWithinDocument"); - long target; - if (mobileShareId != -1 && backup == -1) - target = devTools->methodToReceive("DOM.attributeModified", { { "nodeId" , mobileShareId } , { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); - else if (mobileShareId == -1 && backup == -1) - target = devTools->methodToReceive("DOM.attributeModified", { { "value" , "lmt__mobile_share_container lmt--mobile-hidden" } }); - else - target = devTools->methodToReceive("DOM.childNodeCountUpdated"); - if (update == -1) - update = devTools->methodToReceive("DOM.documentUpdated"); +std::pair Translate(const std::wstring& text) +{ + if (!DevTools::Connected()) return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, ERROR_START_CHROME) }; + + // DevTools can't handle concurrent translations yet + static std::mutex translationMutex; + std::scoped_lock lock(translationMutex); // Navigate to site and wait until it is loaded - QString checkFrom = translateFrom.Copy().empty() ? "ja" : S(translateFrom.Copy()); - QString fullUrl = URL + "#" + checkFrom + "/" + S(translateTo.Copy()) + "/" + qtext; - if (errorCode == 0 && !devTools->SendRequest("Page.navigate", { {"url", fullUrl} }, root)) - errorCode = 1; - timer = 0; - while (errorCode == 0 && !devTools->checkMethod(navigate) && timer < timerStop) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - timer += 0.1; - } - if (timer >= timerStop) - errorCode = 2; + DevTools::StartListening(L"Network.responseReceived"); + DevTools::SendRequest(L"Page.navigate", FormatString(LR"({"url":"https://www.deepl.com/translator#any/%s/%s"})", translateTo.Copy(), Escape(text))); + for (int retry = 0; ++retry < 50; Sleep(100)) + for (const auto& result : DevTools::ListenResults(L"Network.responseReceived")) + if (auto URL = result[L"response"][L"url"].String()) + if (URL->find(L"deepl.com/jsonrpc") != std::string::npos) break; + DevTools::StopListening(L"Network.responseReceived"); - // Check if document is outdated - if (devTools->checkMethod(update)) + // Extract translation from site + auto RemoveTags = [](const std::wstring& HTML) { - docFound = -1; - targetNodeId = -1; - update = -1; - } - - // Get document - if (docFound == -1 && errorCode == 0) - { - if (!devTools->SendRequest("DOM.getDocument", {}, root)) - errorCode = 1; - else - docFound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt(); - } - - // Get target selector - if (targetNodeId == -1 && errorCode == 0) - { - if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "textarea.lmt__target_textarea"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) - { - docFound = -1; - errorCode = 1; - } - else - targetNodeId = root.value("result").toObject().value("nodeId").toInt(); - } - - // Get source language selector - if (sourceLangId == -1 && errorCode == 0) - { - if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__language_select--source"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) - { - docFound = -1; - errorCode = 1; - } - else - sourceLangId = root.value("result").toObject().value("nodeId").toInt(); - } - - // Get mobile share selector - if (mobileShareId == -1 && errorCode == 0) - { - if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__mobile_share_container"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) - { - docFound = -1; - errorCode = 1; - } - else - mobileShareId = root.value("result").toObject().value("nodeId").toInt(); - } - - // Wait for the translation to appear on the web page - timer = 0; - while (errorCode == 0 && !devTools->checkMethod(target) && timer < timerStop) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - timer += 0.1; - } - - // Catch the translation - QString OuterHTML; - if (errorCode == 0) - { - if (!devTools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root)) - { - targetNodeId = -1; - errorCode = 1; - } - else - { - OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - } - } - if (errorCode == 0 && OuterHTML == "
") - { - // Try to catch the notification - int noteNodeId = -1; - if (!devTools->SendRequest("DOM.querySelector", { {"nodeId", docFound}, {"selector", "div.lmt__system_notification"} }, root) - || root.value("result").toObject().value("nodeId").toInt() == 0) - { - if (timer >= timerStop) - errorCode = 2; - else - errorCode = 6; - } - else - { - noteNodeId = root.value("result").toObject().value("nodeId").toInt(); - if (errorCode == 0 && devTools->SendRequest("DOM.getOuterHTML", { {"nodeId", noteNodeId} }, root)) - { - OuterHTML = root.value("result").toObject().value("outerHTML").toString(); - } - errorCode = 3; - } - } - OuterHTML.remove(QRegExp("<[^>]*>")); - OuterHTML = OuterHTML.trimmed(); - if (backup == -1 && errorCode == 0 && timer >= timerStop) - backup = 1; - - // Check if the translator output language does not match the selected language - QString targetLang; - if (errorCode == 0 && devTools->SendRequest("DOM.getAttributes", { {"nodeId", targetNodeId} }, root)) - { - QJsonArray attributes = root.value("result").toObject().value("attributes").toArray(); - for (size_t i = 0; i < attributes.size(); i++) - { - if (attributes[i].toString() == "lang") - { - targetLang = attributes[i + 1].toString().mid(0, 2); - if (targetLang != S(translateTo.Copy())) - { - errorCode = 4; - } - break; - } - } - } - - // Check selected source language - if (errorCode == 0 && devTools->SendRequest("DOM.getAttributes", { {"nodeId", sourceLangId} }, root)) - { - QJsonArray attributes = root.value("result").toObject().value("attributes").toArray(); - for (size_t i = 0; i < attributes.size(); i++) - { - if (attributes[i].toString() == "dl-selected-lang" - && attributes[i + 1].toString().mid(0, 2) != S(translateFrom.Copy())) - { - QStringList::const_iterator constIter; - for (constIter = languagesFrom.constBegin(); constIter != languagesFrom.constEnd(); ++constIter) - { - if (constIter->contains(": " + S(translateFrom.Copy()))) - break; - } - devTools->SendRequest("Runtime.evaluate", { {"expression", - "document\ - .querySelector('div.lmt__language_select--source')\ - .querySelector('button.lmt__language_select__active')\ - .click();\ - document\ - .evaluate(\"//button[contains(text(), '" - + constIter->split(": ")[0] + "')]\", \ - document.querySelector('div.lmt__language_select__menu'),\ - null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)\ - .singleNodeValue\ - .click();" - } }, root); - } - } - } - - callQueue.pop_back(); - if (errorCode == 0) - return { true, S(OuterHTML) }; - else if (errorCode == 1) - return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) }; - else if (errorCode == 2) - return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, timerStop) }; - else if (errorCode == 3) - return { false, FormatString(L"%s: %s", ERROR_NOTE, S(OuterHTML)) }; - else if (errorCode == 4) - return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetLang), S(OuterHTML)) }; - else if (errorCode == 5) - return { false, FormatString(L"%s: %d", ERROR_GOT_TIMEOUT, 2*timerStop) }; - else if (errorCode == 6) - return { false, FormatString(L"%s", ERROR_EMPTY_ANSWER) }; - else - return { false, FormatString(L"%s", TRANSLATION_ERROR) }; + std::wstring result; + for (unsigned i = 0; i < HTML.size(); ++i) + if (HTML[i] == '<') i = HTML.find('>', i); + else result.push_back(HTML[i]); + return result; + }; + if (auto document = Copy(DevTools::SendRequest(L"DOM.getDocument")[L"root"][L"nodeId"].Number())) + if (auto target = Copy(DevTools::SendRequest( + L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"#target-dummydiv"})", (int)document.value()) + )[L"nodeId"].Number())) + if (auto outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String())) + if (auto translation = RemoveTags(outerHTML.value()); !translation.empty()) return { true, translation }; + else if (target = Copy(DevTools::SendRequest( + L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"div.lmt__system_notification"})", (int)document.value()) + )[L"nodeId"].Number())) + if (outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String())) + return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, RemoveTags(outerHTML.value())) }; + return { false, TRANSLATION_ERROR }; } \ No newline at end of file diff --git a/extensions/devtoolswrapper.cpp b/extensions/devtoolswrapper.cpp deleted file mode 100644 index 19fe29c..0000000 --- a/extensions/devtoolswrapper.cpp +++ /dev/null @@ -1,233 +0,0 @@ -#include "qtcommon.h" -#include "extension.h" -#include "blockmarkup.h" -#include "network.h" -#include -#include -#include -#include "devtools.h" - -extern const char* NATIVE_LANGUAGE; -extern const char* TRANSLATE_TO; -extern const char* TRANSLATE_SELECTED_THREAD_ONLY; -extern const char* USE_TRANS_CACHE; -extern const char* MAX_SENTENCE_SIZE; -extern const char* TRANSLATION_PROVIDER; -extern QStringList languagesTo; -extern QStringList languagesFrom; -const char* TRANSLATE_FROM = u8"Translate from"; -const char* NATIVE_LANGUAGE_FROM = u8"Japanese"; -const char* PATH_TO_CHROME = u8"Path to Chrome"; -const char* AUTO_START_CHROME = u8"Start Chrome automatically"; -const char* HEADLESS_CHROME = u8"Start in headless mode"; -const char* CHROME_DEBUG_PORT = u8"Chrome debug port"; -const char* DEV_TOOLS_STATUS = u8"Status: "; -const char* START_DEV_TOOLS_BUTTON = u8"Start"; -const char* START_DEV_TOOLS = u8"Start Chrome"; -const char* STOP_DEV_TOOLS_BUTTON = u8"Stop"; -const char* STOP_DEV_TOOLS = u8"Stop Chrome"; - -extern bool useCache, autoStartChrome, headlessChrome; -extern int maxSentenceSize, chromePort; - -std::pair Translate(const std::wstring& text, DevTools* devTools); - -const char* LANGUAGE_TO = u8"Language"; -const char* LANGUAGE_FROM = u8"Language from"; -const std::string TRANSLATION_CACHE_FILE = FormatString("%s Cache.txt", TRANSLATION_PROVIDER); - -QFormLayout* display; -Settings settings; -Synchronized translateTo = L"en", translateFrom = L"ja"; -Synchronized> translationCache; - -int savedSize; -DevTools* devTools = nullptr; -std::wstring pathToChrome = L""; - -void SaveCache() -{ - std::wstring allTranslations(L"\xfeff"); - for (const auto& [sentence, translation] : translationCache.Acquire().contents) - allTranslations.append(L"|SENTENCE|").append(sentence).append(L"|TRANSLATION|").append(translation).append(L"|END|\r\n"); - std::ofstream(TRANSLATION_CACHE_FILE, std::ios::binary | std::ios::trunc).write((const char*)allTranslations.c_str(), allTranslations.size() * sizeof(wchar_t)); - savedSize = translationCache->size(); -} - -void EraseControlCharacters(std::wstring& text) -{ - for (auto it = text.begin(); it!= text.end(); ++it) - { - if ((*it == '\n') || (*it == '\r') || (*it == '\t') || (int(*it) == 4) || (int(*it) == 5)) - { - text.erase(it--); - } - if (*it == '\\' && *(it + 1) == 'n') - { - text.erase((it + 1)--); - text.erase(it--); - } - } -} - -class Window : public QDialog -{ -public: - Window() : - QDialog(nullptr, Qt::WindowMinMaxButtonsHint) - { - localize(); - display = new QFormLayout(this); - settings.beginGroup(TRANSLATION_PROVIDER); - - auto languageBoxTo = new QComboBox(this); - languageBoxTo->addItems(languagesTo); - int languageTo = -1; - if (settings.contains(LANGUAGE_TO)) languageTo = languageBoxTo->findText(settings.value(LANGUAGE_TO).toString(), Qt::MatchEndsWith); - if (languageTo < 0) languageTo = languageBoxTo->findText(NATIVE_LANGUAGE, Qt::MatchStartsWith); - if (languageTo < 0) languageTo = languageBoxTo->findText("English", Qt::MatchStartsWith); - languageBoxTo->setCurrentIndex(languageTo); - saveLanguage(languageBoxTo->currentText()); - display->addRow(TRANSLATE_TO, languageBoxTo); - connect(languageBoxTo, &QComboBox::currentTextChanged, this, &Window::saveLanguage); - - auto languageBoxFrom = new QComboBox(this); - languageBoxFrom->addItems(languagesFrom); - int languageFrom = -1; - if (settings.contains(LANGUAGE_FROM)) languageFrom = languageBoxFrom->findText(settings.value(LANGUAGE_FROM).toString(), Qt::MatchEndsWith); - if (languageFrom < 0) languageFrom = languageBoxFrom->findText(NATIVE_LANGUAGE_FROM, Qt::MatchStartsWith); - if (languageFrom < 0) languageFrom = languageBoxFrom->findText("Japanese", Qt::MatchStartsWith); - languageBoxFrom->setCurrentIndex(languageFrom); - saveLanguageFrom(languageBoxFrom->currentText()); - display->addRow(TRANSLATE_FROM, languageBoxFrom); - connect(languageBoxFrom, &QComboBox::currentTextChanged, this, &Window::saveLanguageFrom); - for (auto [value, label] : Array{ - { useCache, USE_TRANS_CACHE }, - { autoStartChrome, AUTO_START_CHROME }, - //{ headlessChrome, HEADLESS_CHROME } - }) - { - value = settings.value(label, value).toBool(); - auto checkBox = new QCheckBox(this); - checkBox->setChecked(value); - display->addRow(label, checkBox); - connect(checkBox, &QCheckBox::clicked, [label, &value](bool checked) { settings.setValue(label, value = checked); }); - } - for (auto [value, label] : Array{ - { maxSentenceSize, MAX_SENTENCE_SIZE }, - { chromePort, CHROME_DEBUG_PORT }, - }) - { - value = settings.value(label, value).toInt(); - auto spinBox = new QSpinBox(this); - spinBox->setRange(0, INT_MAX); - spinBox->setValue(value); - display->addRow(label, spinBox); - connect(spinBox, qOverload(&QSpinBox::valueChanged), [label, &value](int newValue) { settings.setValue(label, value = newValue); }); - } - - auto keyInput = new QLineEdit(settings.value(PATH_TO_CHROME).toString()); - pathToChrome = (S(keyInput->text())); - if (pathToChrome.empty()) - { - for (auto defaultPath : Array{ - { L"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe" }, - { L"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" }, - }) - if (std::filesystem::exists(defaultPath)) - { - pathToChrome = defaultPath; - keyInput->setText(S(pathToChrome)); - } - } - connect(keyInput, &QLineEdit::textChanged, [keyInput](QString key) { settings.setValue(PATH_TO_CHROME, S(pathToChrome = (S(key)))); }); - display->addRow(PATH_TO_CHROME, keyInput); - - connect(&startButton, &QPushButton::clicked, this, &Window::Start); - connect(&stopButton, &QPushButton::clicked, this, &Window::Stop); - display->addRow(START_DEV_TOOLS, &startButton); - display->addRow(STOP_DEV_TOOLS, &stopButton); - - status.setFrameStyle(QFrame::Panel | QFrame::Sunken); - display->addRow(DEV_TOOLS_STATUS, &status); - - setWindowTitle(TRANSLATION_PROVIDER); - QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection); - - std::ifstream stream(TRANSLATION_CACHE_FILE, std::ios::binary); - BlockMarkupIterator savedTranslations(stream, Array{ L"|SENTENCE|", L"|TRANSLATION|" }); - auto translationCache = ::translationCache.Acquire(); - - while (auto read = savedTranslations.Next()) - { - auto& [sentence, translation] = read.value(); - translationCache->try_emplace(std::move(sentence), std::move(translation)); - } - savedSize = translationCache->size(); - - devTools = new DevTools(this); - connect(devTools, &DevTools::statusChanged, [this](QString text) - { - status.setText(text); - }); - - if (autoStartChrome) - QMetaObject::invokeMethod(this, &Window::Start, Qt::QueuedConnection); - } - - ~Window() - { - Stop(); - if (devTools != nullptr) - delete devTools; - SaveCache(); - } - -private: - void Start() - { - if (devTools->getStatus() == "Stopped") - devTools->startDevTools(S(pathToChrome), headlessChrome, chromePort); - } - void Stop() - { - devTools->closeDevTools(); - } - - void saveLanguage(QString language) - { - settings.setValue(LANGUAGE_TO, S(translateTo->assign(S(language.split(": ")[1])))); - } - void saveLanguageFrom(QString language) - { - settings.setValue(LANGUAGE_FROM, S(translateFrom->assign(S(language.split(": ")[1])))); - } - QPushButton startButton{ START_DEV_TOOLS_BUTTON, this }; - QPushButton stopButton{ STOP_DEV_TOOLS_BUTTON, this }; - QLabel status{ "Stopped" }; -} window; - -bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) -{ - if (sentenceInfo["text number"] == 0 || sentence.size() > maxSentenceSize) return false; - - bool cache = false; - std::wstring translation; - if (useCache) - { - auto translationCache = ::translationCache.Acquire(); - if (auto it = translationCache->find(sentence); it != translationCache->end()) translation = it->second + L"\x200b"; - } - if (translation.empty() && (sentenceInfo["current select"])) - { - EraseControlCharacters(sentence); - std::tie(cache, translation) = Translate(sentence, devTools); - } - - if (cache) translationCache->try_emplace(sentence, translation); - if (cache && translationCache->size() > savedSize + 50) SaveCache(); - - JSON::Unescape(translation); - sentence += L"\n" + translation; - return true; -} \ No newline at end of file diff --git a/extensions/extrawindow.cpp b/extensions/extrawindow.cpp index a602595..01b2776 100644 --- a/extensions/extrawindow.cpp +++ b/extensions/extrawindow.cpp @@ -46,7 +46,7 @@ struct PrettyWindow : QDialog { PrettyWindow(const char* name) { - localize(); + Localize(); ui.setupUi(this); ui.display->setGraphicsEffect(&outliner); setWindowFlags(Qt::FramelessWindowHint); @@ -439,8 +439,7 @@ private: { std::vector results; for (auto [it, end] = std::equal_range(dictionary.begin(), dictionary.end(), DictionaryEntry{ term.toUtf8() }); it != end; ++it) - if (foundDefinitions.emplace(it->definition).second) - results.push_back({ term, it->definition, inflectionsUsed }); + if (foundDefinitions.emplace(it->definition).second) results.push_back({ term, it->definition, inflectionsUsed }); for (const auto& inflection : inflections) if (auto match = inflection.inflectsTo.match(term); match.hasMatch()) { QStringList currentInflectionsUsed = inflectionsUsed; diff --git a/extensions/googletranslate.cpp b/extensions/googletranslate.cpp index 95abfd2..9998f36 100644 --- a/extensions/googletranslate.cpp +++ b/extensions/googletranslate.cpp @@ -1,5 +1,4 @@ #include "qtcommon.h" -#include "extension.h" #include "network.h" #include @@ -123,7 +122,7 @@ QStringList languages }; bool translateSelectedOnly = false, rateLimitAll = true, rateLimitSelected = false, useCache = true; -int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 500; +int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 1000; std::pair Translate(const std::wstring& text) { @@ -153,9 +152,11 @@ std::pair Translate(const std::wstring& text) if (auto blob = Copy(JSON::Parse(httpRequest.response.substr(start))[0][2].String())) if (auto translations = Copy(JSON::Parse(blob.value())[1][0].Array())) { std::wstring translation; - if (translations->size() == 1 && (translations = Copy(translations.value()[0][5].Array()))) + if (translations->size() == 1) { - for (const auto& sentence : translations.value()) if (sentence[0].String()) (translation += *sentence[0].String()) += L" "; + if (translations = Copy(translations.value()[0][5].Array())) + for (const auto& sentence : translations.value()) + if (sentence[0].String()) (translation += *sentence[0].String()) += L" "; } else { diff --git a/extensions/lua.cpp b/extensions/lua.cpp index 0999774..4ab62f0 100644 --- a/extensions/lua.cpp +++ b/extensions/lua.cpp @@ -47,7 +47,7 @@ public: Window() : QDialog(nullptr, Qt::WindowMinMaxButtonsHint) { - localize(); + Localize(); connect(&loadButton, &QPushButton::clicked, this, &Window::LoadScript); if (scriptEditor.toPlainText().isEmpty()) scriptEditor.setPlainText(LUA_INTRO); diff --git a/extensions/network.cpp b/extensions/network.cpp index 8a3f99f..e14d9cc 100644 --- a/extensions/network.cpp +++ b/extensions/network.cpp @@ -7,11 +7,11 @@ HttpRequest::HttpRequest( const wchar_t* objectName, std::string body, const wchar_t* headers, + DWORD port, const wchar_t* referrer, DWORD requestFlags, const wchar_t* httpVersion, - const wchar_t** acceptTypes, - DWORD port + const wchar_t** acceptTypes ) { static std::atomic internet = NULL; diff --git a/extensions/network.h b/extensions/network.h index 045ab08..5a0db8c 100644 --- a/extensions/network.h +++ b/extensions/network.h @@ -14,11 +14,11 @@ struct HttpRequest const wchar_t* objectName, std::string body = "", const wchar_t* headers = NULL, + DWORD port = INTERNET_DEFAULT_PORT, const wchar_t* referrer = NULL, DWORD requestFlags = WINHTTP_FLAG_SECURE | WINHTTP_FLAG_ESCAPE_DISABLE, const wchar_t* httpVersion = NULL, - const wchar_t** acceptTypes = NULL, - DWORD port = INTERNET_DEFAULT_PORT + const wchar_t** acceptTypes = NULL ); operator bool() { return errorCode == ERROR_SUCCESS; } @@ -38,7 +38,7 @@ namespace JSON std::basic_string Escape(std::basic_string text) { int oldSize = text.size(); - text.resize(text.size() + std::count_if(text.begin(), text.end(), [](auto ch) { return ch == '\n' || ch == '\r' || ch == '\t' || ch == '\\' || ch == '"'; })); + text.resize(text.size() + std::count_if(text.begin(), text.end(), [](C ch) { return ch == '\n' || ch == '\r' || ch == '\t' || ch == '\\' || ch == '"'; })); auto out = text.rbegin(); for (int i = oldSize - 1; i >= 0; --i) { @@ -60,13 +60,13 @@ namespace JSON template struct UTF {}; template <> struct UTF { - inline static std::wstring FromCodepoint(int codepoint) { return { (wchar_t)codepoint }; } // TODO: surrogate pairs + inline static std::wstring FromCodepoint(unsigned codepoint) { return { (wchar_t)codepoint }; } // TODO: surrogate pairs }; template - struct Value : private std::variant, std::vector>, std::unordered_map, Value>> + struct Value : private std::variant, std::vector>, std::unordered_map, Value>> { - using std::variant, std::vector>, std::unordered_map, Value>>::variant; + using std::variant, std::vector>, std::unordered_map, Value>>::variant; explicit operator bool() const { return index(); } bool IsNull() const { return index() == 1; } @@ -78,17 +78,18 @@ namespace JSON const Value& operator[](std::basic_string key) const { - static const Value failure; if (auto object = Object()) if (auto it = object->find(key); it != object->end()) return it->second; return failure; } const Value& operator[](int i) const { - static const Value failure; if (auto array = Array()) if (i < array->size()) return array->at(i); return failure; } + + static const Value failure; }; + template const Value Value::failure; template Value Parse(const std::basic_string& text, int64_t& i, int depth) @@ -116,7 +117,7 @@ namespace JSON if (ch == 'u' && isxdigit(text[i + 2]) && isxdigit(text[i + 3]) && isxdigit(text[i + 4]) && isxdigit(text[i + 5])) { char charCode[] = { text[i + 2], text[i + 3], text[i + 4], text[i + 5], 0 }; - unescaped += UTF::FromCodepoint(strtol(charCode, nullptr, 16)); + unescaped += UTF::FromCodepoint(strtoul(charCode, nullptr, 16)); i += 5; continue; } @@ -137,7 +138,7 @@ namespace JSON static C nullStr[] = { 'n', 'u', 'l', 'l' }, trueStr[] = { 't', 'r', 'u', 'e' }, falseStr[] = { 'f', 'a', 'l', 's', 'e' }; if (ch == nullStr[0]) - if (std::char_traits::compare(text.data() + i, nullStr, std::size(nullStr)) == 0) return i += std::size(nullStr), std::nullopt; + if (std::char_traits::compare(text.data() + i, nullStr, std::size(nullStr)) == 0) return i += std::size(nullStr), nullptr; else return {}; if (ch == trueStr[0]) if (std::char_traits::compare(text.data() + i, trueStr, std::size(trueStr)) == 0) return i += std::size(trueStr), true; diff --git a/extensions/regexfilter.cpp b/extensions/regexfilter.cpp index e4178ba..ac472f2 100644 --- a/extensions/regexfilter.cpp +++ b/extensions/regexfilter.cpp @@ -21,7 +21,7 @@ public: Window() : QDialog(nullptr, Qt::WindowMinMaxButtonsHint) { - localize(); + Localize(); ui.setupUi(this); connect(ui.input, &QLineEdit::textEdited, this, &Window::setRegex); @@ -34,7 +34,7 @@ public: void setRegex(QString regex) { ui.input->setText(regex); - std::lock_guard l(m); + std::scoped_lock lock(m); if (!regex.isEmpty()) try { ::regex = S(regex); } catch (std::regex_error) { return ui.output->setText(INVALID_REGEX); } else ::regex = std::nullopt; diff --git a/extensions/removerepeatphrase.cpp b/extensions/removerepeatphrase.cpp index f8dc5ee..e24a98b 100644 --- a/extensions/removerepeatphrase.cpp +++ b/extensions/removerepeatphrase.cpp @@ -59,14 +59,12 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) { std::wstring substring(sentence, suffixArray[i], commonPrefixLength); bool substringCharMap[0x10000] = {}; - for (auto ch : substring) - substringCharMap[ch] = true; + for (auto ch : substring) substringCharMap[ch] = true; for (int regionSize = 0, j = 0; j <= sentence.size(); ++j) if (substringCharMap[sentence[j]]) regionSize += 1; else if (regionSize >= commonPrefixLength * 2) - while (regionSize > 0) - sentence[j - regionSize--] = ERASED; + while (regionSize > 0) sentence[j - regionSize--] = ERASED; else regionSize = 0; if (!wcsstr(sentence.c_str(), substring.c_str())) std::copy(substring.begin(), substring.end(), sentence.begin() + max(suffixArray[i], suffixArray[i + 1])); diff --git a/extensions/replacer.cpp b/extensions/replacer.cpp index 3488e9d..d317316 100644 --- a/extensions/replacer.cpp +++ b/extensions/replacer.cpp @@ -87,7 +87,7 @@ void UpdateReplacements() try { if (replaceFileLastWrite.exchange(std::filesystem::last_write_time(REPLACE_SAVE_FILE)) == std::filesystem::last_write_time(REPLACE_SAVE_FILE)) return; - std::scoped_lock l(m); + std::scoped_lock lock(m); trie = Trie(std::ifstream(REPLACE_SAVE_FILE, std::ios::binary)); } catch (std::filesystem::filesystem_error) { replaceFileLastWrite.store({}); } @@ -103,7 +103,8 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved if (trie.Empty()) { auto file = std::ofstream(REPLACE_SAVE_FILE, std::ios::binary) << "\xff\xfe"; - for (auto ch : std::wstring_view(REPLACER_INSTRUCTIONS)) file << (ch == L'\n' ? std::string_view("\r\0\n", 4) : std::string_view((char*)&ch, 2)); + for (auto ch : std::wstring_view(REPLACER_INSTRUCTIONS)) + file << (ch == L'\n' ? std::string_view("\r\0\n", 4) : std::string_view((char*)&ch, 2)); _spawnlp(_P_DETACH, "notepad", "notepad", REPLACE_SAVE_FILE, NULL); // show file to user } } diff --git a/extensions/threadlinker.cpp b/extensions/threadlinker.cpp index ed7751a..aa7b93b 100644 --- a/extensions/threadlinker.cpp +++ b/extensions/threadlinker.cpp @@ -17,7 +17,7 @@ public: Window() : QDialog(nullptr, Qt::WindowMinMaxButtonsHint) { - localize(); + Localize(); connect(&linkButton, &QPushButton::clicked, this, &Window::Link); layout.addWidget(&linkList); @@ -35,7 +35,7 @@ private: int to = QInputDialog::getText(this, THREAD_LINK_TO, HEXADECIMAL, QLineEdit::Normal, "", &ok3, Qt::WindowCloseButtonHint).toInt(&ok4, 16); if (ok1 && ok2 && ok3 && ok4) { - std::lock_guard l(m); + std::scoped_lock lock(m); linkedTextHandles[from].insert(to); linkList.addItem(QString::number(from, 16) + "->" + QString::number(to, 16)); } @@ -47,7 +47,7 @@ private: { QStringList link = linkList.currentItem()->text().split("->"); linkList.takeItem(linkList.currentRow()); - std::lock_guard l(m); + std::scoped_lock lock(m); linkedTextHandles[link[0].toInt(nullptr, 16)].erase(link[1].toInt(nullptr, 16)); } } diff --git a/extensions/translatewrapper.cpp b/extensions/translatewrapper.cpp index dd4613d..f1f984c 100644 --- a/extensions/translatewrapper.cpp +++ b/extensions/translatewrapper.cpp @@ -32,16 +32,18 @@ QFormLayout* display; Settings settings; Synchronized translateTo = L"en", apiKey; -Synchronized> translationCache; -int savedSize; - -void SaveCache() +namespace { - std::wstring allTranslations(L"\xfeff"); - for (const auto& [sentence, translation] : translationCache.Acquire().contents) - allTranslations.append(L"|SENTENCE|").append(sentence).append(L"|TRANSLATION|").append(translation).append(L"|END|\r\n"); - std::ofstream(TRANSLATION_CACHE_FILE, std::ios::binary | std::ios::trunc).write((const char*)allTranslations.c_str(), allTranslations.size() * sizeof(wchar_t)); - savedSize = translationCache->size(); + Synchronized> translationCache; + int savedSize; + void SaveCache() + { + std::wstring allTranslations(L"\xfeff"); + for (const auto& [sentence, translation] : translationCache.Acquire().contents) + allTranslations.append(L"|SENTENCE|").append(sentence).append(L"|TRANSLATION|").append(translation).append(L"|END|\r\n"); + std::ofstream(TRANSLATION_CACHE_FILE, std::ios::binary | std::ios::trunc).write((const char*)allTranslations.c_str(), allTranslations.size() * sizeof(wchar_t)); + savedSize = translationCache->size(); + } } class Window : public QDialog @@ -50,7 +52,7 @@ public: Window() : QDialog(nullptr, Qt::WindowMinMaxButtonsHint) { - localize(); + Localize(); display = new QFormLayout(this); settings.beginGroup(TRANSLATION_PROVIDER); @@ -93,7 +95,7 @@ public: } if (GET_API_KEY_FROM) { - auto keyInput = new QLineEdit(settings.value(API_KEY).toString()); + auto keyInput = new QLineEdit(settings.value(API_KEY).toString(), this); apiKey->assign(S(keyInput->text())); QObject::connect(keyInput, &QLineEdit::textChanged, [](QString key) { settings.setValue(API_KEY, S(apiKey->assign(S(key)))); }); auto keyLabel = new QLabel(QString("%2").arg(GET_API_KEY_FROM, API_KEY), this); @@ -147,8 +149,15 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) Synchronized> tokens; } rateLimiter; + auto StripWhitespace = [](std::wstring& text) + { + text.erase(text.begin(), std::find_if_not(text.begin(), text.end(), iswspace)); + text.erase(std::find_if_not(text.rbegin(), text.rend(), iswspace).base(), text.end()); + }; + bool cache = false; std::wstring translation; + StripWhitespace(sentence); if (useCache) { auto translationCache = ::translationCache.Acquire(); @@ -157,6 +166,7 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) if (translation.empty() && (!translateSelectedOnly || sentenceInfo["current select"])) if (rateLimiter.Request() || !rateLimitAll || (!rateLimitSelected && sentenceInfo["current select"])) std::tie(cache, translation) = Translate(sentence); else translation = TOO_MANY_TRANS_REQUESTS; + StripWhitespace(translation); if (cache) translationCache->try_emplace(sentence, translation); if (cache && translationCache->size() > savedSize + 50) SaveCache(); diff --git a/include/common.h b/include/common.h index e316111..fe0ced0 100644 --- a/include/common.h +++ b/include/common.h @@ -82,6 +82,8 @@ static struct // should be inline but MSVC (linker) is bugged template operator T*() { static_assert(sizeof(T) < sizeof(DUMMY)); return (T*)DUMMY; } } DUMMY; +inline auto Swallow = [](auto&&...) {}; + template std::optional> Copy(T* ptr) { if (ptr) return *ptr; return {}; } template inline auto FormatArg(T arg) { return arg; } @@ -134,7 +136,7 @@ inline void TEXTRACTOR_MESSAGE(const wchar_t* format, const Args&... args) { Mes template inline void TEXTRACTOR_DEBUG(const wchar_t* format, const Args&... args) { std::thread([=] { TEXTRACTOR_MESSAGE(format, args...); }).detach(); } -void localize(); +void Localize(); #ifdef _DEBUG #define TEST(...) static auto _ = CreateThread(nullptr, 0, [](auto) { __VA_ARGS__; return 0UL; }, NULL, 0, nullptr); diff --git a/text.cpp b/text.cpp index fe65c5a..34d39b8 100644 --- a/text.cpp +++ b/text.cpp @@ -22,7 +22,7 @@ const char* NATIVE_LANGUAGE = "English"; const char* ATTACH = u8"Attach to game"; const char* LAUNCH = u8"Launch game"; -const char* GAME_CONFIG = u8"Configure game"; +const char* CONFIG = u8"Configure game"; const char* DETACH = u8"Detach from game"; const char* FORGET = u8"Forget game"; const char* ADD_HOOK = u8"Add hook"; @@ -144,6 +144,13 @@ const wchar_t* TOO_MANY_TRANS_REQUESTS = L"Rate limit exceeded: refuse to make m const wchar_t* TRANSLATION_ERROR = L"Error while translating"; const char* USE_PREV_SENTENCE_CONTEXT = u8"Use previous sentence as context"; const char* API_KEY = u8"API key"; +const char* CHROME_LOCATION = "Google Chrome location"; +const char* START_DEVTOOLS = u8"Start DevTools"; +const char* STOP_DEVTOOLS = u8"Stop DevTools"; +const char* HEADLESS_MODE = u8"Headless mode"; +const char* DEVTOOLS_STATUS = u8"DevTools status"; +const char* AUTO_START = u8"Start automatically"; +const wchar_t* ERROR_START_CHROME = L"failed to start Chrome or to connect to it"; const char* EXTRA_WINDOW_INFO = u8R"(Right click to change settings Click and drag on window edges to move, or the bottom right corner to resize)"; const char* SENTENCE_TOO_BIG = u8"Sentence too large to display"; @@ -215,7 +222,7 @@ const char* THREAD_LINK_FROM = u8"Thread number to link from"; const char* THREAD_LINK_TO = u8"Thread number to link to"; const char* HEXADECIMAL = u8"Hexadecimal"; -void localize() +void Localize() { #ifdef TURKISH NATIVE_LANGUAGE = "Turkish"; @@ -316,7 +323,7 @@ Clic y arrastra los bordes de la ventana para moverla, o en la esquina inferior NATIVE_LANGUAGE = "Chinese (simplified)"; ATTACH = u8"附加到游戏"; LAUNCH = u8"启动游戏"; - GAME_CONFIG = u8"配置游戏"; + CONFIG = u8"配置游戏"; DETACH = u8"从游戏分离"; FORGET = u8"移除游戏"; ADD_HOOK = u8"添加钩子"; @@ -469,7 +476,7 @@ end)"; NATIVE_LANGUAGE = "Russian"; ATTACH = u8"Присоединить к игре"; LAUNCH = u8"Запустить игру"; - GAME_CONFIG = u8"Настройки игры"; + CONFIG = u8"Настройки игры"; DETACH = u8"Отсоединить от игры"; FORGET = u8"Забыть игру"; ADD_HOOK = u8"Добавить хук"; @@ -725,7 +732,7 @@ Klik dan tarik pinggiran jendela untuk memindahkan, atau sudut kanan bawah untuk NATIVE_LANGUAGE = "Italian"; ATTACH = u8"Collega al gioco"; LAUNCH = u8"Avvia gioco"; - GAME_CONFIG = u8"Configura gioco"; + CONFIG = u8"Configura gioco"; DETACH = u8"Scollega dal gioco"; FORGET = u8"Dimentica gioco"; ADD_HOOK = u8"Aggiungi gancio"; @@ -1145,7 +1152,7 @@ original_text의 빈공간은 무시되지만, replacement_text는 공백과 엔 NATIVE_LANGUAGE = "French"; ATTACH = u8"Attacher le jeu"; LAUNCH = u8"Lancer le jeu"; - GAME_CONFIG = u8"Configure le jeu"; + CONFIG = u8"Configure le jeu"; DETACH = u8"Detacher du jeu"; FORGET = u8"Oublier le jeu"; ADD_HOOK = u8"Ajouter un hook"; @@ -1336,4 +1343,4 @@ Ce fichier doit être encodé en Unicode (UTF-16 Little Endian).)"; #endif // FRENCH }; -static auto _ = (localize(), 0); +static auto _ = (Localize(), 0); diff --git a/texthook/engine/match.cc b/texthook/engine/match.cc index e7d49ad..817595c 100644 --- a/texthook/engine/match.cc +++ b/texthook/engine/match.cc @@ -35,7 +35,7 @@ namespace Engine void Hijack() { - static auto _ = [] + static auto _ = ([] { GetModuleFileNameW(nullptr, processPath, MAX_PATH); processName = wcsrchr(processPath, L'\\') + 1; @@ -68,8 +68,7 @@ namespace Engine ConsoleOutput("Textractor: hijacking process located from 0x%p to 0x%p", processStartAddress, processStopAddress); DetermineEngineType(); - return NULL; - }(); + }(), 0); } bool ShouldMonoHook(const char* name) diff --git a/texthook/texthook.cc b/texthook/texthook.cc index d2dceb7..66f63d3 100644 --- a/texthook/texthook.cc +++ b/texthook/texthook.cc @@ -30,7 +30,7 @@ namespace { // unnamed 0xff, 0xd3, // call ebx 0x9d, // popfd 0x61, // popad - 0x9d, // popfd + 0x9d, // popfd 0x68, 0,0,0,0, // push @original 0xc3 // ret ; basically absolute jmp to @original }; @@ -86,7 +86,7 @@ namespace { // unnamed 0x5b, // pop rbx 0x58, // pop rax 0x9d, // pop rflags - 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [0] ; relative to next instruction (i.e. jmp @original) + 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [rip] ; relative to next instruction (i.e. jmp @original) 0,0,0,0,0,0,0,0 // @original }; int this_offset = 50, send_offset = 60, original_offset = 126; From c62c586a5981877f667448141aa52911bde36eca Mon Sep 17 00:00:00 2001 From: Akash Mozumdar Date: Fri, 15 Jan 2021 09:32:23 -0700 Subject: [PATCH 14/14] fix bugs and autoload lua --- GUI/CMakeLists.txt | 11 ++-- cmake/QtUtils.cmake | 20 ------- extensions/CMakeLists.txt | 13 +++-- extensions/devtools.cpp | 66 +++++------------------ extensions/devtools.h | 5 +- extensions/devtoolsdeepltranslate.cpp | 77 ++++++++++++--------------- extensions/lua.cpp | 2 + 7 files changed, 67 insertions(+), 127 deletions(-) diff --git a/GUI/CMakeLists.txt b/GUI/CMakeLists.txt index 944e492..be99474 100644 --- a/GUI/CMakeLists.txt +++ b/GUI/CMakeLists.txt @@ -16,8 +16,11 @@ add_executable(Textractor WIN32 target_precompile_headers(Textractor REUSE_FROM pch) target_link_libraries(Textractor Qt5::Widgets shell32 winhttp) -if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll) - if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll) - install_qt5_libs(Textractor) - endif() +if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll AND NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll) + add_custom_command(TARGET Textractor + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" + COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin + COMMAND Qt5::windeployqt --dir ${CMAKE_FINAL_OUTPUT_DIRECTORY} "${CMAKE_FINAL_OUTPUT_DIRECTORY}/Textractor.exe" + ) endif() diff --git a/cmake/QtUtils.cmake b/cmake/QtUtils.cmake index d2523d0..2c5fd3d 100644 --- a/cmake/QtUtils.cmake +++ b/cmake/QtUtils.cmake @@ -87,23 +87,3 @@ macro(find_qt5) message(FATAL_ERROR "Cannot find QT5!") endif() endmacro(find_qt5) - -# Copies required DLLs to directory with target -# Optionally can provide QML directory as second argument -function(install_qt5_libs target) - if(TARGET Qt5::windeployqt) - set(EXTRA "") - if(EXISTS ${ARGV1}) - message("QML directory to be scanned=${ARGV1}") - list(APPEND EXTRA --qmldir ${ARGV1}) - endif() - - # execute windeployqt in a tmp directory after build - add_custom_command(TARGET ${target} - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" - COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin - COMMAND Qt5::windeployqt --dir $ "$/$" ${EXTRA} - ) - endif() -endfunction(install_qt5_libs) diff --git a/extensions/CMakeLists.txt b/extensions/CMakeLists.txt index b1bef90..b6ebbbc 100644 --- a/extensions/CMakeLists.txt +++ b/extensions/CMakeLists.txt @@ -44,8 +44,11 @@ target_link_libraries(Lua lua53 Qt5::Widgets) target_link_libraries(Regex\ Filter Qt5::Widgets) target_link_libraries(Thread\ Linker Qt5::Widgets) -if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll) - if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll) - install_qt5_libs(DevTools\ DeepL\ Translate) - endif() -endif() +if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll AND NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll) + add_custom_command(TARGET DevTools\ DeepL\ Translate + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" + COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin + COMMAND Qt5::windeployqt --dir ${CMAKE_FINAL_OUTPUT_DIRECTORY} "${CMAKE_FINAL_OUTPUT_DIRECTORY}/DevTools\ DeepL\ Translate.dll" + ) +endif() diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index 5c1ba36..a2bc204 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -7,11 +7,10 @@ namespace { std::function OnStatusChanged = Swallow; PROCESS_INFORMATION processInfo = {}; - std::atomic idCounter = 0, idMethod = 0; + std::atomic idCounter = 0; std::mutex devToolsMutex; QWebSocket webSocket; std::unordered_map>> mapQueue; - std::unordered_map>> mapMethod; auto _ = ([] { QObject::connect(&webSocket, &QWebSocket::stateChanged, @@ -21,28 +20,27 @@ namespace { auto result = JSON::Parse(S(message)); std::scoped_lock lock(devToolsMutex); - if (auto method = result[L"method"].String()) - for (auto& [listenTo, results] : mapMethod) if (*method == listenTo) results.push_back(result[L"params"]); - if (auto id = result[L"id"].Number()) - if (auto request = mapQueue.find((int)*id); request != mapQueue.end()) - request->second.set(result), mapQueue.erase(request); + if (auto id = result[L"id"].Number()) if (auto request = mapQueue.find((int)*id); request != mapQueue.end()) + { + request->second.set(result); + mapQueue.erase(request); + } }); }(), 0); } namespace DevTools { - void Start(const std::wstring& path, std::function statusChanged, bool headless, int port) + void Start(const std::wstring& path, std::function statusChanged, bool headless) { OnStatusChanged = statusChanged; DWORD exitCode = 0; auto args = FormatString( - L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=%d", + L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=9222", path, - std::filesystem::current_path().wstring(), - port + std::filesystem::current_path().wstring() ); - if (headless) args += L"--headless"; + if (headless) args += L" --headless"; STARTUPINFOW DUMMY = { sizeof(DUMMY) }; if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) || CreateProcessW(NULL, args.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &DUMMY, &processInfo) @@ -55,7 +53,7 @@ namespace DevTools L"/json/list", "", NULL, - (DWORD)port, + 9222, NULL, WINHTTP_FLAG_ESCAPE_DISABLE }) @@ -68,29 +66,12 @@ namespace DevTools { std::scoped_lock lock(devToolsMutex); webSocket.open(S(*(*it)[L"webSocketDebuggerUrl"].String())); - if (HttpRequest httpRequest{ - L"Mozilla/5.0 Textractor", - L"127.0.0.1", - L"POST", - L"/json/version", - "", - NULL, - (DWORD)port, - NULL, - WINHTTP_FLAG_ESCAPE_DISABLE - }) - if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String())) - if (userAgent->find(L"Headless") != std::string::npos) - SendRequest(L"Network.setUserAgentOverride", FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L""))); return; } } OnStatusChanged("Failed Connection"); } - else - { - OnStatusChanged("Failed Startup"); - } + else OnStatusChanged("Failed Startup"); } void Close() @@ -98,7 +79,6 @@ namespace DevTools std::scoped_lock lock(devToolsMutex); for (const auto& [_, task] : mapQueue) task.set_exception(std::runtime_error("closed")); webSocket.close(); - mapMethod.clear(); mapQueue.clear(); DWORD exitCode = 0; if (GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) @@ -120,12 +100,12 @@ namespace DevTools JSON::Value SendRequest(const std::wstring& method, const std::wstring& params) { - if (webSocket.state() != QAbstractSocket::ConnectedState) return {}; concurrency::task_completion_event> response; - int id = idCounter +=1; + int id = idCounter += 1; auto message = FormatString(LR"({"id":%d,"method":"%s","params":%s})", id, method, params); { std::scoped_lock lock(devToolsMutex); + if (webSocket.state() != QAbstractSocket::ConnectedState) return {}; mapQueue.try_emplace(id, response); webSocket.sendTextMessage(S(message)); webSocket.flush(); @@ -133,22 +113,4 @@ namespace DevTools try { if (auto result = create_task(response).get()[L"result"]) return result; } catch (...) {} return {}; } - - void StartListening(const std::wstring& method) - { - std::scoped_lock lock(devToolsMutex); - mapMethod.try_emplace(method); - } - - std::vector> ListenResults(const std::wstring& method) - { - std::scoped_lock lock(devToolsMutex); - return mapMethod[method]; - } - - void StopListening(const std::wstring& method) - { - std::scoped_lock lock(devToolsMutex); - mapMethod.erase(method); - } } diff --git a/extensions/devtools.h b/extensions/devtools.h index ae7d6dd..81c996f 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -3,11 +3,8 @@ namespace DevTools { - void Start(const std::wstring& path, std::function statusChanged, bool headless, int port); + void Start(const std::wstring& path, std::function statusChanged, bool headless); void Close(); bool Connected(); JSON::Value SendRequest(const std::wstring& method, const std::wstring& params = L"{}"); - void StartListening(const std::wstring& method); - std::vector> ListenResults(const std::wstring& method); - void StopListening(const std::wstring& method); } diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 743b4e1..1247d71 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -10,7 +10,7 @@ extern Settings settings; const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate"; const char* GET_API_KEY_FROM = nullptr; bool translateSelectedOnly = true, rateLimitAll = false, rateLimitSelected = false, useCache = true; -int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 10000; +int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 2500; extern const char* CHROME_LOCATION; extern const char* START_DEVTOOLS; @@ -54,31 +54,50 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved display->addRow(CHROME_LOCATION, chromePathEdit); auto statusLabel = new QLabel("Stopped"); auto startButton = new QPushButton(START_DEVTOOLS), stopButton = new QPushButton(STOP_DEVTOOLS); - auto headlessCheckBox = new QCheckBox(HEADLESS_MODE); + auto headlessCheckBox = new QCheckBox(); headlessCheckBox->setChecked(settings.value(HEADLESS_MODE, true).toBool()); QObject::connect(headlessCheckBox, &QCheckBox::clicked, [](bool headless) { settings.setValue(HEADLESS_MODE, headless); }); QObject::connect(startButton, &QPushButton::clicked, [statusLabel, chromePathEdit, headlessCheckBox] { DevTools::Start( S(chromePathEdit->text()), - [statusLabel](QString status) { QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status)); }, - headlessCheckBox->isChecked(), - 9222 + [statusLabel](QString status) + { + QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status)); + if (status == "ConnectedState") std::thread([] + { + if (HttpRequest httpRequest{ + L"Mozilla/5.0 Textractor", + L"127.0.0.1", + L"POST", + L"/json/version", + "", + NULL, + 9222, + NULL, + WINHTTP_FLAG_ESCAPE_DISABLE + }) + if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String())) + if (userAgent->find(L"Headless") != std::string::npos) + DevTools::SendRequest(L"Network.setUserAgentOverride", + FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L""))); + }).detach(); + }, + headlessCheckBox->isChecked() ); - if (!DevTools::SendRequest(L"Network.enable")) DevTools::Close(); }); QObject::connect(stopButton, &QPushButton::clicked, &DevTools::Close); auto buttons = new QHBoxLayout(); buttons->addWidget(startButton); buttons->addWidget(stopButton); - display->addRow(buttons); - display->addRow(headlessCheckBox); - auto autoStartButton = new QCheckBox(AUTO_START); + display->addRow(HEADLESS_MODE, headlessCheckBox); + auto autoStartButton = new QCheckBox(); autoStartButton->setChecked(settings.value(AUTO_START, false).toBool()); - QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) {settings.setValue(AUTO_START, autoStart); }); - display->addRow(autoStartButton); + QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) { settings.setValue(AUTO_START, autoStart); }); + display->addRow(AUTO_START, autoStartButton); + display->addRow(buttons); statusLabel->setFrameStyle(QFrame::Panel | QFrame::Sunken); display->addRow(DEVTOOLS_STATUS, statusLabel); - if (autoStartButton->isChecked()) startButton->click(); + if (autoStartButton->isChecked()) QMetaObject::invokeMethod(startButton, &QPushButton::click, Qt::QueuedConnection); } break; case DLL_PROCESS_DETACH: @@ -93,39 +112,13 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved std::pair Translate(const std::wstring& text) { if (!DevTools::Connected()) return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, ERROR_START_CHROME) }; - // DevTools can't handle concurrent translations yet static std::mutex translationMutex; std::scoped_lock lock(translationMutex); - - // Navigate to site and wait until it is loaded - DevTools::StartListening(L"Network.responseReceived"); DevTools::SendRequest(L"Page.navigate", FormatString(LR"({"url":"https://www.deepl.com/translator#any/%s/%s"})", translateTo.Copy(), Escape(text))); - for (int retry = 0; ++retry < 50; Sleep(100)) - for (const auto& result : DevTools::ListenResults(L"Network.responseReceived")) - if (auto URL = result[L"response"][L"url"].String()) - if (URL->find(L"deepl.com/jsonrpc") != std::string::npos) break; - DevTools::StopListening(L"Network.responseReceived"); - - // Extract translation from site - auto RemoveTags = [](const std::wstring& HTML) - { - std::wstring result; - for (unsigned i = 0; i < HTML.size(); ++i) - if (HTML[i] == '<') i = HTML.find('>', i); - else result.push_back(HTML[i]); - return result; - }; - if (auto document = Copy(DevTools::SendRequest(L"DOM.getDocument")[L"root"][L"nodeId"].Number())) - if (auto target = Copy(DevTools::SendRequest( - L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"#target-dummydiv"})", (int)document.value()) - )[L"nodeId"].Number())) - if (auto outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String())) - if (auto translation = RemoveTags(outerHTML.value()); !translation.empty()) return { true, translation }; - else if (target = Copy(DevTools::SendRequest( - L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"div.lmt__system_notification"})", (int)document.value()) - )[L"nodeId"].Number())) - if (outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String())) - return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, RemoveTags(outerHTML.value())) }; + for (int retry = 0; ++retry < 100; Sleep(100)) + if (auto translation = Copy(DevTools::SendRequest( + L"Runtime.evaluate", LR"({"expression":"document.querySelector('#target-dummydiv').innerHTML","returnByValue":true})")[L"result"][L"value"].String()) + ) if (!translation->empty()) return { true, translation.value() }; return { false, TRANSLATION_ERROR }; } \ No newline at end of file diff --git a/extensions/lua.cpp b/extensions/lua.cpp index 4ab62f0..9d5c272 100644 --- a/extensions/lua.cpp +++ b/extensions/lua.cpp @@ -57,6 +57,8 @@ public: resize(800, 600); setWindowTitle("Lua"); QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection); + + LoadScript(); } ~Window()