Public-Mirror/Textractor
1
0
Fork 1
mirror of https://github.com/Artikash/Textractor.git synced 2024-12-23 08:54:12 +08:00
Code Issues Projects Releases Wiki Activity

add event catch method and refactor error messages

Browse Source 
Added method that catches specified events from the page
Refactored error messages
This commit is contained in:
zeheyler 2020-10-15 02:32:58 +03:00
parent 6b54ec0733
commit a3ebaf0023
3 changed files with 103 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
extensions/devtools.cpp
View File

@ -3,10 +3,11 @@
DevTools::DevTools(QObject* parent) :
QObject(parent),
idcounter(0),
idmethod(0),
pagenavigated(false),
translateready(false),
status("Stopped"),
session(1)
session(0)
{
}
@ -116,30 +117,6 @@ void DevTools::stateChanged(QAbstractSocket::SocketState state)
emit statusChanged(status);
}
void DevTools::setNavigated(bool value)
{
mutex.lock();
pagenavigated = value;
mutex.unlock();
}
bool DevTools::getNavigated()
{
return pagenavigated;
}
void DevTools::setTranslate(bool value)
{
mutex.lock();
translateready = value;
mutex.unlock();
}
bool DevTools::getTranslate()
{
return translateready;
}
bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root)
{
if (!isConnected())
@ -183,11 +160,28 @@ bool DevTools::SendRequest(QString method, QJsonObject params, QJsonObject& root
return false;
}
long DevTools::methodToReceive(QString method, QJsonObject params)
{
QJsonObject json;
long id = idmIncrement();
json.insert("method", method);
json.insert("params", params);
mutex.lock();
mapmethod.insert(std::make_pair(id, json));
mutex.unlock();
return id;
}
long DevTools::idIncrement()
{
return ++idcounter;
}
long DevTools::idmIncrement()
{
return ++idmethod;
}
bool DevTools::isConnected()
{
if (webSocket.state() == QAbstractSocket::ConnectedState)
@ -196,6 +190,29 @@ bool DevTools::isConnected()
return false;
}
bool DevTools::compareJson(QJsonObject storedparams, QJsonObject params)
{
foreach(const QString & key, storedparams.keys())
{
if (storedparams.value(key).isArray())
return false;
if (storedparams.value(key) != params.value(key))
return false;
if (!compareJson(storedparams.value(key).toObject(), params.value(key).toObject()))
return false;
}
return true;
}
bool DevTools::checkMethod(long id)
{
MapMethod::iterator iter = mapmethod.find(id);
if (iter == mapmethod.end())
return true;
else
return false;
}
void DevTools::onTextMessageReceived(QString message)
{
QJsonDocument doc = QJsonDocument::fromJson(message.toUtf8());
@ -204,23 +221,19 @@ void DevTools::onTextMessageReceived(QString message)
QJsonObject root = doc.object();
if (root.contains("method"))
{
if (root.value("method").toString() == "Page.navigatedWithinDocument")
for (auto iter = mapmethod.cbegin(); iter != mapmethod.cend();)
{
if ((iter->second.value("method") == root.value("method"))
&& ((iter->second.value("params").toObject().isEmpty())
|| (compareJson(iter->second.value("params").toObject(), root.value("params").toObject()))))
{
mutex.lock();
pagenavigated = true;
mutex.unlock();
}
if (root.value("method").toString() == "DOM.attributeModified")
{
if (root.value("params").toObject().value("value") == "lmt__mobile_share_container")
{
mutex.lock();
translateready = true;
mapmethod.erase(iter++);
mutex.unlock();
}
++iter;
}
return;
}
if (root.contains("id"))
{
@ -242,17 +255,18 @@ void DevTools::closeDevTools()
{
if (this->mapqueue.size() > 0)
{
MapResponse::iterator iter = this->mapqueue.begin();
MapResponse::iterator iend = this->mapqueue.end();
MapResponse::iterator iter = mapqueue.begin();
MapResponse::iterator iend = mapqueue.end();
for (; iter != iend; iter++)
{
iter->second.set_exception("exception");
}
}
webSocket.close();
mapmethod.clear();
mapqueue.clear();
idcounter = 0;
idmethod = 0;
DWORD exitCode = 0;
if (GetExitCodeProcess(processInfo.hProcess, &exitCode) != FALSE)
{

13
extensions/devtools.h
View File

@ -7,6 +7,7 @@
using namespace Concurrency;
typedef std::map<long, task_completion_event<QJsonObject>> MapResponse;
typedef std::map<long, QJsonObject> MapMethod;
class DevTools : public QObject {
Q_OBJECT
@ -24,12 +25,10 @@ private Q_SLOTS:
public:
void startDevTools(QString path, bool headless = false, int port = 9222);
void closeDevTools();
void setNavigated(bool value);
bool getNavigated();
void setTranslate(bool value);
bool getTranslate();
bool checkMethod(long id);
int getSession();
bool SendRequest(QString command, QJsonObject params, QJsonObject& result);
bool SendRequest(QString method, QJsonObject params, QJsonObject& root);
long methodToReceive(QString method, QJsonObject params);
QString getStatus();
private:
@ -37,13 +36,17 @@ private:
bool startChrome(QString path, bool headless = false, int port = 9222);
bool GetwebSocketDebuggerUrl(QString& url, int port = 9222);
long idIncrement();
long idmIncrement();
bool compareJson(QJsonObject storedparams, QJsonObject params);
int session;
QWebSocket webSocket;
std::mutex mutex;
MapResponse mapqueue;
MapMethod mapmethod;
bool pagenavigated;
bool translateready;
long idcounter;
long idmethod;
PROCESS_INFORMATION processInfo;
QString status;
};

75
extensions/devtoolsdeepltranslate.cpp
View File

@ -10,6 +10,12 @@ bool useCache = true, autostartchrome = false, headlesschrome = true;
int maxSentenceSize = 500, chromeport = 9222;
const char* TRANSLATION_PROVIDER = "DevTools DeepL Translate";
const wchar_t* ERROR_CHROME = L"Error: chrome not started";
const wchar_t* ERROR_START_CHROME = L"Error: failed to start chrome or to connect to it";
const wchar_t* ERROR_GOT_TIMEOUT = L"Error: timeout (s)";
const wchar_t* ERROR_COMMAND_FAIL = L"Error: command failed";
const wchar_t* ERROR_LANGUAGE = L"Error: target languages do not match";
QString URL = "https://www.deepl.com/en/translator";
QStringList languages
{
@ -26,27 +32,10 @@ QStringList languages
"Spanish: es",
};
int docfound = -1;
int targetNodeId = -1;
int session = -1;
int docfound = -1, targetNodeId = -1, session = -1, pageenabled = -1;
std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devtools)
{
if (devtools->getStatus() == "Stopped")
{
return { false, FormatString(L"Error: chrome not started") };
}
if ((devtools->getStatus().startsWith("Fail")) || (devtools->getStatus().startsWith("Unconnected")))
{
return { false, FormatString(L"Error: %s", S(devtools->getStatus())) };
}
if (session != devtools->getSession())
{
session = devtools->getSession();
docfound = -1;
targetNodeId = -1;
}
QString qtext = S(text);
// Check text for repeated symbols (e.g. only ellipsis)
@ -57,10 +46,26 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
break;
if ((i + 2) == qtext.length() && (qtext.front() == qtext.back()))
{
return { false, text };
return { true, text };
}
}
if (devtools->getStatus() == "Stopped")
{
return { false, FormatString(L"%s", ERROR_CHROME) };
}
if ((devtools->getStatus().startsWith("Fail")) || (devtools->getStatus().startsWith("Unconnected")))
{
return { false, FormatString(L"%s", ERROR_START_CHROME) };
}
if (session != devtools->getSession())
{
session = devtools->getSession();
docfound = -1;
targetNodeId = -1;
pageenabled = -1;
}
// Add spaces near ellipsis for better translation and check for quotes
qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230));
qtext.replace(QRegularExpression("[" + QString(8230) + "]" + "[" + QString(8230) + "]"), QString(8230));
@ -73,31 +78,34 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
qtext.chop(1);
}
QJsonObject root;
QJsonObject result;
// Enable page feedback
if (pageenabled == -1)
{
if (!devtools->SendRequest("Page.enable", {}, root))
{
return { false, FormatString(L"Error: page enable failed! %s", TRANSLATION_ERROR) };
return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) };
}
pageenabled = 1;
}
long navigate = devtools->methodToReceive("Page.navigatedWithinDocument", {});
long target = devtools->methodToReceive("DOM.attributeModified", { {"value" , "lmt__mobile_share_container"} });
// Navigate to site
QString fullurl = URL + "#ja/" + S(translateTo.Copy()) + "/" + qtext;
devtools->setNavigated(false);
devtools->setTranslate(false);
if (devtools->SendRequest("Page.navigate", { {"url", fullurl} }, root))
{
// Wait until page is loaded
float timer = 0;
int timer_stop = 10;
while (!devtools->getNavigated() && timer < timer_stop)
while (!devtools->checkMethod(navigate) && timer < timer_stop)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
timer += 0.1;
}
if (timer >= timer_stop)
{
return { false, FormatString(L"Error: page load timeout %d s! %s", timer_stop, TRANSLATION_ERROR) };
return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) };
}
QString OuterHTML("<div></div>");
@ -107,7 +115,7 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
if (!devtools->SendRequest("DOM.getDocument", {}, root))
{
docfound = -1;
return { false, FormatString(L"Error: getDocument failed! %s", TRANSLATION_ERROR) };
return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) };
}
docfound = root.value("result").toObject().value("root").toObject().value("nodeId").toInt();
}
@ -119,14 +127,14 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
|| (root.value("result").toObject().value("nodeId").toInt() == 0))
{
docfound = -1;
return { false, FormatString(L"Error: querySelector result failed! %s", TRANSLATION_ERROR) };
return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) };
}
targetNodeId = root.value("result").toObject().value("nodeId").toInt();
}
// Wait for translation to appear on the web page
timer = 0;
while (!devtools->getTranslate() && timer < timer_stop)
while (!devtools->checkMethod(target) && timer < timer_stop)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
timer += 0.1;
@ -138,7 +146,7 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
if (!(devtools->SendRequest("DOM.querySelector", { {"nodeId", docfound}, {"selector", "div.lmt__system_notification"} }, root))
|| (root.value("result").toObject().value("nodeId").toInt() == 0))
{
return { false, FormatString(L"Error: result timeout %d s! %s", timer_stop, TRANSLATION_ERROR) };
return { false, FormatString(L"%s: %d ", ERROR_GOT_TIMEOUT, timer_stop) };
}
noteNodeId = root.value("result").toObject().value("nodeId").toInt();
@ -152,14 +160,13 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
{
OuterHTML = "Could not get notification";
}
return { false, FormatString(L"Error: got notification from translator: %s", S(OuterHTML)) };
return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) };
}
// Catch the translation
devtools->SendRequest("DOM.getOuterHTML", { {"nodeId", targetNodeId + 1} }, root);
result = root.value("result").toObject();
OuterHTML = result.value("outerHTML").toString();
OuterHTML = root.value("result").toObject().value("outerHTML").toString();
OuterHTML.remove(QRegExp("<[^>]*>"));
OuterHTML = OuterHTML.trimmed();
@ -175,7 +182,7 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
QString targetlang = attributes[i + 1].toString().mid(0, 2);
if (targetlang != S(translateTo.Copy()))
{
return { false, FormatString(L"Error: target langs do not match (%s): %s", S(targetlang), S(OuterHTML)) };
return { false, FormatString(L"%s (%s): %s", ERROR_LANGUAGE, S(targetlang), S(OuterHTML)) };
}
}
}
@ -190,6 +197,6 @@ std::pair<bool, std::wstring> Translate(const std::wstring& text, DevTools* devt
}
else
{
return { false, FormatString(L"Error: navigate failed! %s", TRANSLATION_ERROR) };
return { false, FormatString(L"%s", ERROR_COMMAND_FAIL) };
}
}