From c62c586a5981877f667448141aa52911bde36eca Mon Sep 17 00:00:00 2001 From: Akash Mozumdar Date: Fri, 15 Jan 2021 09:32:23 -0700 Subject: [PATCH] fix bugs and autoload lua --- GUI/CMakeLists.txt | 11 ++-- cmake/QtUtils.cmake | 20 ------- extensions/CMakeLists.txt | 13 +++-- extensions/devtools.cpp | 66 +++++------------------ extensions/devtools.h | 5 +- extensions/devtoolsdeepltranslate.cpp | 77 ++++++++++++--------------- extensions/lua.cpp | 2 + 7 files changed, 67 insertions(+), 127 deletions(-) diff --git a/GUI/CMakeLists.txt b/GUI/CMakeLists.txt index 944e492..be99474 100644 --- a/GUI/CMakeLists.txt +++ b/GUI/CMakeLists.txt @@ -16,8 +16,11 @@ add_executable(Textractor WIN32 target_precompile_headers(Textractor REUSE_FROM pch) target_link_libraries(Textractor Qt5::Widgets shell32 winhttp) -if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll) - if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll) - install_qt5_libs(Textractor) - endif() +if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll AND NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll) + add_custom_command(TARGET Textractor + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" + COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin + COMMAND Qt5::windeployqt --dir ${CMAKE_FINAL_OUTPUT_DIRECTORY} "${CMAKE_FINAL_OUTPUT_DIRECTORY}/Textractor.exe" + ) endif() diff --git a/cmake/QtUtils.cmake b/cmake/QtUtils.cmake index d2523d0..2c5fd3d 100644 --- a/cmake/QtUtils.cmake +++ b/cmake/QtUtils.cmake @@ -87,23 +87,3 @@ macro(find_qt5) message(FATAL_ERROR "Cannot find QT5!") endif() endmacro(find_qt5) - -# Copies required DLLs to directory with target -# Optionally can provide QML directory as second argument -function(install_qt5_libs target) - if(TARGET Qt5::windeployqt) - set(EXTRA "") - if(EXISTS ${ARGV1}) - message("QML directory to be scanned=${ARGV1}") - list(APPEND EXTRA --qmldir ${ARGV1}) - endif() - - # execute windeployqt in a tmp directory after build - add_custom_command(TARGET ${target} - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" - COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin - COMMAND Qt5::windeployqt --dir $ "$/$" ${EXTRA} - ) - endif() -endfunction(install_qt5_libs) diff --git a/extensions/CMakeLists.txt b/extensions/CMakeLists.txt index b1bef90..b6ebbbc 100644 --- a/extensions/CMakeLists.txt +++ b/extensions/CMakeLists.txt @@ -44,8 +44,11 @@ target_link_libraries(Lua lua53 Qt5::Widgets) target_link_libraries(Regex\ Filter Qt5::Widgets) target_link_libraries(Thread\ Linker Qt5::Widgets) -if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll) - if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll) - install_qt5_libs(DevTools\ DeepL\ Translate) - endif() -endif() +if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll AND NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll) + add_custom_command(TARGET DevTools\ DeepL\ Translate + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt" + COMMAND set PATH=%PATH%$${qt5_install_prefix}/bin + COMMAND Qt5::windeployqt --dir ${CMAKE_FINAL_OUTPUT_DIRECTORY} "${CMAKE_FINAL_OUTPUT_DIRECTORY}/DevTools\ DeepL\ Translate.dll" + ) +endif() diff --git a/extensions/devtools.cpp b/extensions/devtools.cpp index 5c1ba36..a2bc204 100644 --- a/extensions/devtools.cpp +++ b/extensions/devtools.cpp @@ -7,11 +7,10 @@ namespace { std::function OnStatusChanged = Swallow; PROCESS_INFORMATION processInfo = {}; - std::atomic idCounter = 0, idMethod = 0; + std::atomic idCounter = 0; std::mutex devToolsMutex; QWebSocket webSocket; std::unordered_map>> mapQueue; - std::unordered_map>> mapMethod; auto _ = ([] { QObject::connect(&webSocket, &QWebSocket::stateChanged, @@ -21,28 +20,27 @@ namespace { auto result = JSON::Parse(S(message)); std::scoped_lock lock(devToolsMutex); - if (auto method = result[L"method"].String()) - for (auto& [listenTo, results] : mapMethod) if (*method == listenTo) results.push_back(result[L"params"]); - if (auto id = result[L"id"].Number()) - if (auto request = mapQueue.find((int)*id); request != mapQueue.end()) - request->second.set(result), mapQueue.erase(request); + if (auto id = result[L"id"].Number()) if (auto request = mapQueue.find((int)*id); request != mapQueue.end()) + { + request->second.set(result); + mapQueue.erase(request); + } }); }(), 0); } namespace DevTools { - void Start(const std::wstring& path, std::function statusChanged, bool headless, int port) + void Start(const std::wstring& path, std::function statusChanged, bool headless) { OnStatusChanged = statusChanged; DWORD exitCode = 0; auto args = FormatString( - L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=%d", + L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=9222", path, - std::filesystem::current_path().wstring(), - port + std::filesystem::current_path().wstring() ); - if (headless) args += L"--headless"; + if (headless) args += L" --headless"; STARTUPINFOW DUMMY = { sizeof(DUMMY) }; if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) || CreateProcessW(NULL, args.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &DUMMY, &processInfo) @@ -55,7 +53,7 @@ namespace DevTools L"/json/list", "", NULL, - (DWORD)port, + 9222, NULL, WINHTTP_FLAG_ESCAPE_DISABLE }) @@ -68,29 +66,12 @@ namespace DevTools { std::scoped_lock lock(devToolsMutex); webSocket.open(S(*(*it)[L"webSocketDebuggerUrl"].String())); - if (HttpRequest httpRequest{ - L"Mozilla/5.0 Textractor", - L"127.0.0.1", - L"POST", - L"/json/version", - "", - NULL, - (DWORD)port, - NULL, - WINHTTP_FLAG_ESCAPE_DISABLE - }) - if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String())) - if (userAgent->find(L"Headless") != std::string::npos) - SendRequest(L"Network.setUserAgentOverride", FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L""))); return; } } OnStatusChanged("Failed Connection"); } - else - { - OnStatusChanged("Failed Startup"); - } + else OnStatusChanged("Failed Startup"); } void Close() @@ -98,7 +79,6 @@ namespace DevTools std::scoped_lock lock(devToolsMutex); for (const auto& [_, task] : mapQueue) task.set_exception(std::runtime_error("closed")); webSocket.close(); - mapMethod.clear(); mapQueue.clear(); DWORD exitCode = 0; if (GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) @@ -120,12 +100,12 @@ namespace DevTools JSON::Value SendRequest(const std::wstring& method, const std::wstring& params) { - if (webSocket.state() != QAbstractSocket::ConnectedState) return {}; concurrency::task_completion_event> response; - int id = idCounter +=1; + int id = idCounter += 1; auto message = FormatString(LR"({"id":%d,"method":"%s","params":%s})", id, method, params); { std::scoped_lock lock(devToolsMutex); + if (webSocket.state() != QAbstractSocket::ConnectedState) return {}; mapQueue.try_emplace(id, response); webSocket.sendTextMessage(S(message)); webSocket.flush(); @@ -133,22 +113,4 @@ namespace DevTools try { if (auto result = create_task(response).get()[L"result"]) return result; } catch (...) {} return {}; } - - void StartListening(const std::wstring& method) - { - std::scoped_lock lock(devToolsMutex); - mapMethod.try_emplace(method); - } - - std::vector> ListenResults(const std::wstring& method) - { - std::scoped_lock lock(devToolsMutex); - return mapMethod[method]; - } - - void StopListening(const std::wstring& method) - { - std::scoped_lock lock(devToolsMutex); - mapMethod.erase(method); - } } diff --git a/extensions/devtools.h b/extensions/devtools.h index ae7d6dd..81c996f 100644 --- a/extensions/devtools.h +++ b/extensions/devtools.h @@ -3,11 +3,8 @@ namespace DevTools { - void Start(const std::wstring& path, std::function statusChanged, bool headless, int port); + void Start(const std::wstring& path, std::function statusChanged, bool headless); void Close(); bool Connected(); JSON::Value SendRequest(const std::wstring& method, const std::wstring& params = L"{}"); - void StartListening(const std::wstring& method); - std::vector> ListenResults(const std::wstring& method); - void StopListening(const std::wstring& method); } diff --git a/extensions/devtoolsdeepltranslate.cpp b/extensions/devtoolsdeepltranslate.cpp index 743b4e1..1247d71 100644 --- a/extensions/devtoolsdeepltranslate.cpp +++ b/extensions/devtoolsdeepltranslate.cpp @@ -10,7 +10,7 @@ extern Settings settings; const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate"; const char* GET_API_KEY_FROM = nullptr; bool translateSelectedOnly = true, rateLimitAll = false, rateLimitSelected = false, useCache = true; -int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 10000; +int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 2500; extern const char* CHROME_LOCATION; extern const char* START_DEVTOOLS; @@ -54,31 +54,50 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved display->addRow(CHROME_LOCATION, chromePathEdit); auto statusLabel = new QLabel("Stopped"); auto startButton = new QPushButton(START_DEVTOOLS), stopButton = new QPushButton(STOP_DEVTOOLS); - auto headlessCheckBox = new QCheckBox(HEADLESS_MODE); + auto headlessCheckBox = new QCheckBox(); headlessCheckBox->setChecked(settings.value(HEADLESS_MODE, true).toBool()); QObject::connect(headlessCheckBox, &QCheckBox::clicked, [](bool headless) { settings.setValue(HEADLESS_MODE, headless); }); QObject::connect(startButton, &QPushButton::clicked, [statusLabel, chromePathEdit, headlessCheckBox] { DevTools::Start( S(chromePathEdit->text()), - [statusLabel](QString status) { QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status)); }, - headlessCheckBox->isChecked(), - 9222 + [statusLabel](QString status) + { + QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status)); + if (status == "ConnectedState") std::thread([] + { + if (HttpRequest httpRequest{ + L"Mozilla/5.0 Textractor", + L"127.0.0.1", + L"POST", + L"/json/version", + "", + NULL, + 9222, + NULL, + WINHTTP_FLAG_ESCAPE_DISABLE + }) + if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String())) + if (userAgent->find(L"Headless") != std::string::npos) + DevTools::SendRequest(L"Network.setUserAgentOverride", + FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L""))); + }).detach(); + }, + headlessCheckBox->isChecked() ); - if (!DevTools::SendRequest(L"Network.enable")) DevTools::Close(); }); QObject::connect(stopButton, &QPushButton::clicked, &DevTools::Close); auto buttons = new QHBoxLayout(); buttons->addWidget(startButton); buttons->addWidget(stopButton); - display->addRow(buttons); - display->addRow(headlessCheckBox); - auto autoStartButton = new QCheckBox(AUTO_START); + display->addRow(HEADLESS_MODE, headlessCheckBox); + auto autoStartButton = new QCheckBox(); autoStartButton->setChecked(settings.value(AUTO_START, false).toBool()); - QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) {settings.setValue(AUTO_START, autoStart); }); - display->addRow(autoStartButton); + QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) { settings.setValue(AUTO_START, autoStart); }); + display->addRow(AUTO_START, autoStartButton); + display->addRow(buttons); statusLabel->setFrameStyle(QFrame::Panel | QFrame::Sunken); display->addRow(DEVTOOLS_STATUS, statusLabel); - if (autoStartButton->isChecked()) startButton->click(); + if (autoStartButton->isChecked()) QMetaObject::invokeMethod(startButton, &QPushButton::click, Qt::QueuedConnection); } break; case DLL_PROCESS_DETACH: @@ -93,39 +112,13 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved std::pair Translate(const std::wstring& text) { if (!DevTools::Connected()) return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, ERROR_START_CHROME) }; - // DevTools can't handle concurrent translations yet static std::mutex translationMutex; std::scoped_lock lock(translationMutex); - - // Navigate to site and wait until it is loaded - DevTools::StartListening(L"Network.responseReceived"); DevTools::SendRequest(L"Page.navigate", FormatString(LR"({"url":"https://www.deepl.com/translator#any/%s/%s"})", translateTo.Copy(), Escape(text))); - for (int retry = 0; ++retry < 50; Sleep(100)) - for (const auto& result : DevTools::ListenResults(L"Network.responseReceived")) - if (auto URL = result[L"response"][L"url"].String()) - if (URL->find(L"deepl.com/jsonrpc") != std::string::npos) break; - DevTools::StopListening(L"Network.responseReceived"); - - // Extract translation from site - auto RemoveTags = [](const std::wstring& HTML) - { - std::wstring result; - for (unsigned i = 0; i < HTML.size(); ++i) - if (HTML[i] == '<') i = HTML.find('>', i); - else result.push_back(HTML[i]); - return result; - }; - if (auto document = Copy(DevTools::SendRequest(L"DOM.getDocument")[L"root"][L"nodeId"].Number())) - if (auto target = Copy(DevTools::SendRequest( - L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"#target-dummydiv"})", (int)document.value()) - )[L"nodeId"].Number())) - if (auto outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String())) - if (auto translation = RemoveTags(outerHTML.value()); !translation.empty()) return { true, translation }; - else if (target = Copy(DevTools::SendRequest( - L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"div.lmt__system_notification"})", (int)document.value()) - )[L"nodeId"].Number())) - if (outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String())) - return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, RemoveTags(outerHTML.value())) }; + for (int retry = 0; ++retry < 100; Sleep(100)) + if (auto translation = Copy(DevTools::SendRequest( + L"Runtime.evaluate", LR"({"expression":"document.querySelector('#target-dummydiv').innerHTML","returnByValue":true})")[L"result"][L"value"].String()) + ) if (!translation->empty()) return { true, translation.value() }; return { false, TRANSLATION_ERROR }; } \ No newline at end of file diff --git a/extensions/lua.cpp b/extensions/lua.cpp index 4ab62f0..9d5c272 100644 --- a/extensions/lua.cpp +++ b/extensions/lua.cpp @@ -57,6 +57,8 @@ public: resize(800, 600); setWindowTitle("Lua"); QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection); + + LoadScript(); } ~Window()