chenx221/Textractor_test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix bugs and autoload lua

Browse Source
This commit is contained in:
Akash Mozumdar 2021-01-15 09:32:23 -07:00
parent fc81b17a3c
commit c62c586a59
7 changed files with 67 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
GUI/CMakeLists.txt
View File

@ -16,8 +16,11 @@ add_executable(Textractor WIN32
target_precompile_headers(Textractor REUSE_FROM pch)
target_link_libraries(Textractor Qt5::Widgets shell32 winhttp)
if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll)
if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll)
install_qt5_libs(Textractor)
endif()
if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Core.dll AND NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5Cored.dll)
add_custom_command(TARGET Textractor
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt"
COMMAND set PATH=%PATH%$<SEMICOLON>${qt5_install_prefix}/bin
COMMAND Qt5::windeployqt --dir ${CMAKE_FINAL_OUTPUT_DIRECTORY} "${CMAKE_FINAL_OUTPUT_DIRECTORY}/Textractor.exe"
)
endif()

20
cmake/QtUtils.cmake
View File

@ -87,23 +87,3 @@ macro(find_qt5)
message(FATAL_ERROR "Cannot find QT5!")
endif()
endmacro(find_qt5)
# Copies required DLLs to directory with target
# Optionally can provide QML directory as second argument
function(install_qt5_libs target)
if(TARGET Qt5::windeployqt)
set(EXTRA "")
if(EXISTS ${ARGV1})
message("QML directory to be scanned=${ARGV1}")
list(APPEND EXTRA --qmldir ${ARGV1})
endif()
# execute windeployqt in a tmp directory after build
add_custom_command(TARGET ${target}
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt"
COMMAND set PATH=%PATH%$<SEMICOLON>${qt5_install_prefix}/bin
COMMAND Qt5::windeployqt --dir $<TARGET_FILE_DIR:${target}> "$<TARGET_FILE_DIR:${target}>/$<TARGET_FILE_NAME:${target}>" ${EXTRA}
)
endif()
endfunction(install_qt5_libs)

13
extensions/CMakeLists.txt
View File

@ -44,8 +44,11 @@ target_link_libraries(Lua lua53 Qt5::Widgets)
target_link_libraries(Regex\ Filter Qt5::Widgets)
target_link_libraries(Thread\ Linker Qt5::Widgets)
if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll)
if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll)
install_qt5_libs(DevTools\ DeepL\ Translate)
endif()
endif()
if (NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSockets.dll AND NOT EXISTS ${CMAKE_FINAL_OUTPUT_DIRECTORY}/Qt5WebSocketsd.dll)
add_custom_command(TARGET DevTools\ DeepL\ Translate
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/windeployqt"
COMMAND set PATH=%PATH%$<SEMICOLON>${qt5_install_prefix}/bin
COMMAND Qt5::windeployqt --dir ${CMAKE_FINAL_OUTPUT_DIRECTORY} "${CMAKE_FINAL_OUTPUT_DIRECTORY}/DevTools\ DeepL\ Translate.dll"
)
endif()

66
extensions/devtools.cpp
View File

@ -7,11 +7,10 @@ namespace
{
std::function<void(QString)> OnStatusChanged = Swallow;
PROCESS_INFORMATION processInfo = {};
std::atomic<int> idCounter = 0, idMethod = 0;
std::atomic<int> idCounter = 0;
std::mutex devToolsMutex;
QWebSocket webSocket;
std::unordered_map<int, concurrency::task_completion_event<JSON::Value<wchar_t>>> mapQueue;
std::unordered_map<std::wstring, std::vector<JSON::Value<wchar_t>>> mapMethod;
auto _ = ([]
{
QObject::connect(&webSocket, &QWebSocket::stateChanged,
@ -21,28 +20,27 @@ namespace
{
auto result = JSON::Parse(S(message));
std::scoped_lock lock(devToolsMutex);
if (auto method = result[L"method"].String())
for (auto& [listenTo, results] : mapMethod) if (*method == listenTo) results.push_back(result[L"params"]);
if (auto id = result[L"id"].Number())
if (auto request = mapQueue.find((int)*id); request != mapQueue.end())
request->second.set(result), mapQueue.erase(request);
if (auto id = result[L"id"].Number()) if (auto request = mapQueue.find((int)*id); request != mapQueue.end())
{
request->second.set(result);
mapQueue.erase(request);
}
});
}(), 0);
}
namespace DevTools
{
void Start(const std::wstring& path, std::function<void(QString)> statusChanged, bool headless, int port)
void Start(const std::wstring& path, std::function<void(QString)> statusChanged, bool headless)
{
OnStatusChanged = statusChanged;
DWORD exitCode = 0;
auto args = FormatString(
L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=%d",
L"%s --proxy-server=direct:// --disable-extensions --disable-gpu --user-data-dir=%s\\devtoolscache --remote-debugging-port=9222",
path,
std::filesystem::current_path().wstring(),
port
std::filesystem::current_path().wstring()
);
if (headless) args += L"--headless";
if (headless) args += L" --headless";
STARTUPINFOW DUMMY = { sizeof(DUMMY) };
if ((GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE) ||
CreateProcessW(NULL, args.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &DUMMY, &processInfo)
@ -55,7 +53,7 @@ namespace DevTools
L"/json/list",
"",
NULL,
(DWORD)port,
9222,
NULL,
WINHTTP_FLAG_ESCAPE_DISABLE
})
@ -68,29 +66,12 @@ namespace DevTools
{
std::scoped_lock lock(devToolsMutex);
webSocket.open(S(*(*it)[L"webSocketDebuggerUrl"].String()));
if (HttpRequest httpRequest{
L"Mozilla/5.0 Textractor",
L"127.0.0.1",
L"POST",
L"/json/version",
"",
NULL,
(DWORD)port,
NULL,
WINHTTP_FLAG_ESCAPE_DISABLE
})
if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String()))
if (userAgent->find(L"Headless") != std::string::npos)
SendRequest(L"Network.setUserAgentOverride", FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L"")));
return;
}
}
OnStatusChanged("Failed Connection");
}
else
{
OnStatusChanged("Failed Startup");
}
else OnStatusChanged("Failed Startup");
}
void Close()
@ -98,7 +79,6 @@ namespace DevTools
std::scoped_lock lock(devToolsMutex);
for (const auto& [_, task] : mapQueue) task.set_exception(std::runtime_error("closed"));
webSocket.close();
mapMethod.clear();
mapQueue.clear();
DWORD exitCode = 0;
if (GetExitCodeProcess(processInfo.hProcess, &exitCode) && exitCode == STILL_ACTIVE)
@ -120,12 +100,12 @@ namespace DevTools
JSON::Value<wchar_t> SendRequest(const std::wstring& method, const std::wstring& params)
{
if (webSocket.state() != QAbstractSocket::ConnectedState) return {};
concurrency::task_completion_event<JSON::Value<wchar_t>> response;
int id = idCounter +=1;
int id = idCounter += 1;
auto message = FormatString(LR"({"id":%d,"method":"%s","params":%s})", id, method, params);
{
std::scoped_lock lock(devToolsMutex);
if (webSocket.state() != QAbstractSocket::ConnectedState) return {};
mapQueue.try_emplace(id, response);
webSocket.sendTextMessage(S(message));
webSocket.flush();
@ -133,22 +113,4 @@ namespace DevTools
try { if (auto result = create_task(response).get()[L"result"]) return result; } catch (...) {}
return {};
}
void StartListening(const std::wstring& method)
{
std::scoped_lock lock(devToolsMutex);
mapMethod.try_emplace(method);
}
std::vector<JSON::Value<wchar_t>> ListenResults(const std::wstring& method)
{
std::scoped_lock lock(devToolsMutex);
return mapMethod[method];
}
void StopListening(const std::wstring& method)
{
std::scoped_lock lock(devToolsMutex);
mapMethod.erase(method);
}
}

5
extensions/devtools.h
View File

@ -3,11 +3,8 @@
namespace DevTools
{
void Start(const std::wstring& path, std::function<void(QString)> statusChanged, bool headless, int port);
void Start(const std::wstring& path, std::function<void(QString)> statusChanged, bool headless);
void Close();
bool Connected();
JSON::Value<wchar_t> SendRequest(const std::wstring& method, const std::wstring& params = L"{}");
void StartListening(const std::wstring& method);
std::vector<JSON::Value<wchar_t>> ListenResults(const std::wstring& method);
void StopListening(const std::wstring& method);
}

77
extensions/devtoolsdeepltranslate.cpp
View File

@ -10,7 +10,7 @@ extern Settings settings;
const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate";
const char* GET_API_KEY_FROM = nullptr;
bool translateSelectedOnly = true, rateLimitAll = false, rateLimitSelected = false, useCache = true;
int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 10000;
int tokenCount = 30, tokenRestoreDelay = 60000, maxSentenceSize = 2500;
extern const char* CHROME_LOCATION;
extern const char* START_DEVTOOLS;
@ -54,31 +54,50 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved
display->addRow(CHROME_LOCATION, chromePathEdit);
auto statusLabel = new QLabel("Stopped");
auto startButton = new QPushButton(START_DEVTOOLS), stopButton = new QPushButton(STOP_DEVTOOLS);
auto headlessCheckBox = new QCheckBox(HEADLESS_MODE);
auto headlessCheckBox = new QCheckBox();
headlessCheckBox->setChecked(settings.value(HEADLESS_MODE, true).toBool());
QObject::connect(headlessCheckBox, &QCheckBox::clicked, [](bool headless) { settings.setValue(HEADLESS_MODE, headless); });
QObject::connect(startButton, &QPushButton::clicked, [statusLabel, chromePathEdit, headlessCheckBox] {
DevTools::Start(
S(chromePathEdit->text()),
[statusLabel](QString status) { QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status)); },
headlessCheckBox->isChecked(),
9222
[statusLabel](QString status)
{
QMetaObject::invokeMethod(statusLabel, std::bind(&QLabel::setText, statusLabel, status));
if (status == "ConnectedState") std::thread([]
{
if (HttpRequest httpRequest{
L"Mozilla/5.0 Textractor",
L"127.0.0.1",
L"POST",
L"/json/version",
"",
NULL,
9222,
NULL,
WINHTTP_FLAG_ESCAPE_DISABLE
})
if (auto userAgent = Copy(JSON::Parse(httpRequest.response)[L"User-Agent"].String()))
if (userAgent->find(L"Headless") != std::string::npos)
DevTools::SendRequest(L"Network.setUserAgentOverride",
FormatString(LR"({"userAgent":"%s"})", userAgent->replace(userAgent->find(L"Headless"), 8, L"")));
}).detach();
},
headlessCheckBox->isChecked()
);
if (!DevTools::SendRequest(L"Network.enable")) DevTools::Close();
});
QObject::connect(stopButton, &QPushButton::clicked, &DevTools::Close);
auto buttons = new QHBoxLayout();
buttons->addWidget(startButton);
buttons->addWidget(stopButton);
display->addRow(buttons);
display->addRow(headlessCheckBox);
auto autoStartButton = new QCheckBox(AUTO_START);
display->addRow(HEADLESS_MODE, headlessCheckBox);
auto autoStartButton = new QCheckBox();
autoStartButton->setChecked(settings.value(AUTO_START, false).toBool());
QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) {settings.setValue(AUTO_START, autoStart); });
display->addRow(autoStartButton);
QObject::connect(autoStartButton, &QCheckBox::clicked, [](bool autoStart) { settings.setValue(AUTO_START, autoStart); });
display->addRow(AUTO_START, autoStartButton);
display->addRow(buttons);
statusLabel->setFrameStyle(QFrame::Panel | QFrame::Sunken);
display->addRow(DEVTOOLS_STATUS, statusLabel);
if (autoStartButton->isChecked()) startButton->click();
if (autoStartButton->isChecked()) QMetaObject::invokeMethod(startButton, &QPushButton::click, Qt::QueuedConnection);
}
break;
case DLL_PROCESS_DETACH:
@ -93,39 +112,13 @@ BOOL WINAPI DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved
std::pair<bool, std::wstring> Translate(const std::wstring& text)
{
if (!DevTools::Connected()) return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, ERROR_START_CHROME) };
// DevTools can't handle concurrent translations yet
static std::mutex translationMutex;
std::scoped_lock lock(translationMutex);
// Navigate to site and wait until it is loaded
DevTools::StartListening(L"Network.responseReceived");
DevTools::SendRequest(L"Page.navigate", FormatString(LR"({"url":"https://www.deepl.com/translator#any/%s/%s"})", translateTo.Copy(), Escape(text)));
for (int retry = 0; ++retry < 50; Sleep(100))
for (const auto& result : DevTools::ListenResults(L"Network.responseReceived"))
if (auto URL = result[L"response"][L"url"].String())
if (URL->find(L"deepl.com/jsonrpc") != std::string::npos) break;
DevTools::StopListening(L"Network.responseReceived");
// Extract translation from site
auto RemoveTags = [](const std::wstring& HTML)
{
std::wstring result;
for (unsigned i = 0; i < HTML.size(); ++i)
if (HTML[i] == '<') i = HTML.find('>', i);
else result.push_back(HTML[i]);
return result;
};
if (auto document = Copy(DevTools::SendRequest(L"DOM.getDocument")[L"root"][L"nodeId"].Number()))
if (auto target = Copy(DevTools::SendRequest(
L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"#target-dummydiv"})", (int)document.value())
)[L"nodeId"].Number()))
if (auto outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String()))
if (auto translation = RemoveTags(outerHTML.value()); !translation.empty()) return { true, translation };
else if (target = Copy(DevTools::SendRequest(
L"DOM.querySelector", FormatString(LR"({"nodeId":%d,"selector":"div.lmt__system_notification"})", (int)document.value())
)[L"nodeId"].Number()))
if (outerHTML = Copy(DevTools::SendRequest(L"DOM.getOuterHTML", FormatString(LR"({"nodeId":%d})", (int)target.value()))[L"outerHTML"].String()))
return { false, FormatString(L"%s: %s", TRANSLATION_ERROR, RemoveTags(outerHTML.value())) };
for (int retry = 0; ++retry < 100; Sleep(100))
if (auto translation = Copy(DevTools::SendRequest(
L"Runtime.evaluate", LR"({"expression":"document.querySelector('#target-dummydiv').innerHTML","returnByValue":true})")[L"result"][L"value"].String())
) if (!translation->empty()) return { true, translation.value() };
return { false, TRANSLATION_ERROR };
}

2
extensions/lua.cpp
View File

@ -57,6 +57,8 @@ public:
resize(800, 600);
setWindowTitle("Lua");
QMetaObject::invokeMethod(this, &QWidget::show, Qt::QueuedConnection);
LoadScript();
}
~Window()