From 7cd7c77848288794658865b978784725701a8c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=81=8D=E5=85=AE=E6=83=9A=E5=85=AE?= <101191390+HIllya51@users.noreply.github.com> Date: Mon, 4 Nov 2024 23:10:41 +0800 Subject: [PATCH] many (#1134) --- .gitignore | 6 + .gitmodules | 3 + README.md | 2 +- docs/other/README_en.md | 2 +- docs/other/README_ru.md | 2 +- src/LunaTranslator/ocrengines/local.py | 72 +- src/build.py | 3 - src/plugins/CMakeLists.txt | 5 +- src/plugins/LunaOCR/CMakeLists.txt | 49 ++ src/plugins/LunaOCR/OCR.cpp | 702 ++++++++++++++++++ .../applicationloopbackaudio/CMakeLists.txt | 2 +- .../applicationloopbackaudio/runer.cpp | 6 +- src/plugins/libs/Clipper2 | 1 + src/plugins/libs/libs.cmake | 28 +- .../OnnxRuntimeWrapper.cmake | 16 + .../opencv-static/OpenCVWrapperConfig.cmake | 15 + src/plugins/libs/tinymp3 | 2 +- src/plugins/pch.h | 2 +- src/plugins/scripts/copytarget.py | 2 + src/plugins/scripts/fetchwebview2.py | 33 +- src/plugins/shareddllproxy/CMakeLists.txt | 23 +- src/plugins/shareddllproxy/aspatch.cpp | 2 + src/plugins/wcocr/wcocr.cpp | 8 +- src/plugins/winsharedutils/AreoAcrylic.cpp | 6 +- src/plugins/winsharedutils/CMakeLists.txt | 3 +- src/plugins/winsharedutils/clipboard.cpp | 6 +- src/plugins/winsharedutils/define.h | 2 +- .../winsharedutils/globalmessagelistener.cpp | 4 +- src/plugins/winsharedutils/hwnd.cpp | 12 +- src/plugins/winsharedutils/mp3enc.cpp | 2 +- src/plugins/winsharedutils/sapi_dll.cpp | 2 +- src/plugins/winsharedutils/screenshot.cpp | 4 +- src/plugins/winsharedutils/theme.cpp | 6 +- src/plugins/winsharedutils/webview2_extra.cpp | 12 +- 34 files changed, 930 insertions(+), 115 deletions(-) create mode 100644 src/plugins/LunaOCR/CMakeLists.txt create mode 100644 src/plugins/LunaOCR/OCR.cpp create mode 160000 src/plugins/libs/Clipper2 create mode 100644 src/plugins/libs/onnxruntime-static/OnnxRuntimeWrapper.cmake create mode 100644 src/plugins/libs/opencv-static/OpenCVWrapperConfig.cmake diff --git a/.gitignore b/.gitignore index 11200999..2d61b740 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,9 @@ src/run3732.bat src/plugins/builds src/plugins/libs/webview2 src/plugins/.vscode/settings.json +src/plugins/libs/opencv-static/windows-x86 +src/plugins/libs/opencv-static/windows-x64 +src/plugins/libs/onnxruntime-static/windows-x86 +src/plugins/libs/onnxruntime-static/windows-x64 +src/plugins/libs/opencv-static.zip +src/plugins/libs/onnxruntime-static.zip diff --git a/.gitmodules b/.gitmodules index 89f5631e..30bec105 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "src/plugins/libs/Detours"] path = src/plugins/libs/Detours url = https://github.com/microsoft/Detours +[submodule "src/plugins/libs/Clipper2"] + path = src/plugins/libs/Clipper2 + url = https://github.com/AngusJohnson/Clipper2 diff --git a/README.md b/README.md index 4647eac3..94dcc165 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ - **HOOK** 支持使用[HOOK](https://github.com/HIllya51/LunaHook)方式获取文本，支持使用特殊码，支持自动保存游戏及HOOK、自动加载HOOK等。对于部分引擎，还支持内嵌翻译。对于不支持或支持不好的游戏，请[提交反馈](https://lunatranslator.org/Resource/game_support) -- **OCR** 支持 **离线OCR** ( 除[内置OCR引擎](https://github.com/HIllya51/LunaOCR)外，还支持WindowsOCR、Tessearact5、manga-ocr、WeChat/QQ OCR ) 和 **在线OCR** ( 百度、有道、飞书、讯飞、Google Lens、Google Cloud Vision、docsumo、ocrspace、Gemini、ChatGPT兼容接口 ) +- **OCR** 支持 **离线OCR** ( 除内置OCR引擎外，还支持WindowsOCR、Tessearact5、manga-ocr、WeChat/QQ OCR ) 和 **在线OCR** ( 百度、有道、飞书、讯飞、Google Lens、Google Cloud Vision、docsumo、ocrspace、Gemini、ChatGPT兼容接口 ) - **剪贴板** 支持从剪贴板中获取文本进行翻译 diff --git a/docs/other/README_en.md b/docs/other/README_en.md index 9c03ef4a..646deb87 100644 --- a/docs/other/README_en.md +++ b/docs/other/README_en.md @@ -20,7 +20,7 @@ - **HOOK** Supports obtaining text using [HOOK](https://github.com/HIllya51/LunaHook) methods, supports the use of special codes, supports automatic saving of games and HOOKs, automatic loading of HOOKs, etc. For some engines, it also supports embedded translation. For games that are not supported or not well supported, please [submit feedback](https://lunatranslator.org/Resource/game_support) -- **OCR** supports **offline OCR** (in addition to the [built-in OCR engine](https://github.com/HIllya51/LunaOCR), it also supports WindowsOCR, Tesseract5, manga-ocr, WeChat/QQ OCR) and **online OCR** (Baidu, Youdao, Feishu, iFlytek, Google Lens, Google Cloud Vision, docsumo, ocrspace, Gemini, ChatGPT-compatible interfaces). +- **OCR** supports **offline OCR** (in addition to the built-in OCR engine, it also supports WindowsOCR, Tesseract5, manga-ocr, WeChat/QQ OCR) and **online OCR** (Baidu, Youdao, Feishu, iFlytek, Google Lens, Google Cloud Vision, docsumo, ocrspace, Gemini, ChatGPT-compatible interfaces). - **Clipboard** Supports obtaining text from the clipboard for translation diff --git a/docs/other/README_ru.md b/docs/other/README_ru.md index 96075547..a9d6c95d 100644 --- a/docs/other/README_ru.md +++ b/docs/other/README_ru.md @@ -21,7 +21,7 @@ - **HOOK** Поддерживает получение текста с использованием метода [HOOK](https://github.com/HIllya51/LunaHook), поддерживает использование специальных кодов, поддерживает автоматическое сохранение игр и HOOK, автоматическое загрузка HOOK и т.д. Для некоторых движков также поддерживается встроенная трансляция. Для игр, которые не поддерживаются или плохо поддерживаются, пожалуйста, [отправьте обратную связь](https://lunatranslator.org/Resource/game_support) -- **OCR** поддерживает **офлайн OCR** (помимо [встроенного OCR-движка](https://github.com/HIllya51/LunaOCR), также поддерживает WindowsOCR, Tesseract5, manga-ocr, WeChat/QQ OCR) и **онлайн OCR** (Baidu, Youdao, Feishu, iFlytek, Google Lens, Google Cloud Vision, docsumo, ocrspace, Gemini, совместимые интерфейсы ChatGPT). +- **OCR** поддерживает **офлайн OCR** (помимо встроенного OCR-движка, также поддерживает WindowsOCR, Tesseract5, manga-ocr, WeChat/QQ OCR) и **онлайн OCR** (Baidu, Youdao, Feishu, iFlytek, Google Lens, Google Cloud Vision, docsumo, ocrspace, Gemini, совместимые интерфейсы ChatGPT). - **Буфер обмена** Поддерживает получение текста из буфера обмена для перевода diff --git a/src/LunaTranslator/ocrengines/local.py b/src/LunaTranslator/ocrengines/local.py index 268598dc..68e14b22 100644 --- a/src/LunaTranslator/ocrengines/local.py +++ b/src/LunaTranslator/ocrengines/local.py @@ -1,6 +1,6 @@ import os, zipfile from myutils.utils import dynamiclink -from myutils.config import _TR, getlang_inner2show +from myutils.config import _TR, getlang_inner2show, globalconfig from ocrengines.baseocrclass import baseocr from ctypes import ( CDLL, @@ -12,6 +12,9 @@ from ctypes import ( Structure, pointer, c_char_p, + c_wchar_p, + c_bool, + CFUNCTYPE, ) import os import gobject, functools @@ -40,69 +43,42 @@ class ocrwrapper: self.pOcrObj = None self.__OcrInit(det, rec, key) - def __OcrInit(self, szDetModel, szRecModel, szKeyPath, szClsModel="", nThreads=4): + def __OcrInit(self, szDetModel, szRecModel, szKeyPath, nThreads=4): _OcrInit = self.dll.OcrInit _OcrInit.restype = c_void_p - self.pOcrObj = _OcrInit( - c_char_p(szDetModel.encode("utf8")), - c_char_p(szClsModel.encode("utf8")), - c_char_p(szRecModel.encode("utf8")), - c_char_p(szKeyPath.encode("utf8")), - nThreads, - ) + _OcrInit.argtypes = c_wchar_p, c_wchar_p, c_wchar_p, c_int32 + self.pOcrObj = _OcrInit(szDetModel, szRecModel, szKeyPath, nThreads) + + def __OcrDetect(self, data: bytes, rotate: bool): + + texts = [] + pss = [] + + def cb(ps: ocrpoints, text: bytes): + pss.append((ps.x1, ps.y1, ps.x2, ps.y2, ps.x3, ps.y3, ps.x4, ps.y4)) + texts.append(text.decode("utf8")) - def __OcrDetect(self, data: bytes, angle): _OcrDetect = self.dll.OcrDetect _OcrDetect.argtypes = ( c_void_p, c_void_p, c_size_t, - c_int32, - POINTER(c_int32), - POINTER(POINTER(ocrpoints)), - POINTER(POINTER(c_char_p)), + c_bool, + c_void_p, ) - - _OcrFreeptr = self.dll.OcrFreeptr - _OcrFreeptr.argtypes = c_int32, c_void_p, c_void_p - - num = c_int32() - ps = POINTER(ocrpoints)() - chars = POINTER(c_char_p)() - res = _OcrDetect( + _OcrDetect( self.pOcrObj, data, len(data), - c_int32(angle), - pointer(num), - pointer(ps), - pointer(chars), + rotate, + CFUNCTYPE(None, ocrpoints, c_char_p)(cb), ) - if not res: - return [], [] - texts = [] - pss = [] - for i in range((num.value)): - texts.append(chars[i].decode("utf8")) - pss.append( - ( - ps[i].x1, - ps[i].y1, - ps[i].x2, - ps[i].y2, - ps[i].x3, - ps[i].y3, - ps[i].x4, - ps[i].y4, - ) - ) - _OcrFreeptr(num, ps, chars) return pss, texts - def ocr(self, data, angle=0): + def ocr(self, data, rotate=False): try: - return self.__OcrDetect(data, angle) + return self.__OcrDetect(data, rotate) except: print_exc() return [], [] @@ -226,6 +202,6 @@ class OCR(baseocr): pss, texts = self._ocr.ocr( imagebinary, - 0, + globalconfig["verticalocr"] == 1, ) return {"box": pss, "text": texts} diff --git a/src/build.py b/src/build.py index 57ee3421..a03c356f 100644 --- a/src/build.py +++ b/src/build.py @@ -29,7 +29,6 @@ mylinks = { "ja.zip": "https://github.com/test123456654321/RESOURCES/releases/download/ocr_models/ja.zip", }, "mecab.zip": "https://github.com/HIllya51/RESOURCES/releases/download/common/mecab.zip", - "ocr.zip": "https://github.com/HIllya51/RESOURCES/releases/download/common/ocr.zip", "magpie.zip": "https://github.com/HIllya51/RESOURCES/releases/download/common/magpie.zip", "themes.zip": "https://github.com/HIllya51/RESOURCES/releases/download/common/themes.zip", } @@ -121,8 +120,6 @@ def downloadcommon(): downloadlr() subprocess.run(f"curl -LO {mylinks['mecab.zip']}") subprocess.run(f"7z x mecab.zip -oALL") - subprocess.run(f"curl -LO {mylinks['ocr.zip']}") - subprocess.run(f"7z x ocr.zip -oALL") subprocess.run(f"curl -LO {mylinks['magpie.zip']}") subprocess.run(f"7z x magpie.zip -oALL") diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 52aade4f..28a0bbe3 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -29,7 +29,7 @@ include(generate_product_version) set(VERSION_MAJOR 5) set(VERSION_MINOR 55) -set(VERSION_PATCH 2) +set(VERSION_PATCH 3) add_library(pch pch.cpp) target_precompile_headers(pch PUBLIC pch.h) @@ -41,4 +41,5 @@ add_subdirectory(winsharedutils) add_subdirectory(hookmagpie) add_subdirectory(shareddllproxy) add_subdirectory(applicationloopbackaudio) -add_subdirectory(wcocr) \ No newline at end of file +add_subdirectory(wcocr) +add_subdirectory(LunaOCR) \ No newline at end of file diff --git a/src/plugins/LunaOCR/CMakeLists.txt b/src/plugins/LunaOCR/CMakeLists.txt new file mode 100644 index 00000000..fa969fd6 --- /dev/null +++ b/src/plugins/LunaOCR/CMakeLists.txt @@ -0,0 +1,49 @@ +project(LunaOCR) + +generate_product_version( + versioninfo + NAME "LunaTranslator OCR" + COMPANY_COPYRIGHT "HIllya51 (C) 2024" + VERSION_MAJOR ${VERSION_MAJOR} + VERSION_MINOR ${VERSION_MINOR} + VERSION_PATCH ${VERSION_PATCH} +) + + +set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + +include(${CMAKE_CURRENT_SOURCE_DIR}/../libs/onnxruntime-static/OnnxRuntimeWrapper.cmake) + +find_package(OnnxRuntime REQUIRED) +if (OnnxRuntime_FOUND) + message(STATUS "OnnxRuntime_LIBS: ${OnnxRuntime_LIBS}") + message(STATUS "OnnxRuntime_INCLUDE_DIRS: ${OnnxRuntime_INCLUDE_DIRS}") +else () + message(FATAL_ERROR "onnxruntime Not Found!") +endif (OnnxRuntime_FOUND) + +# OpenCV +set(BUILD_SHARED_LIBS false) +if (CMAKE_CL_64) + set(OpenCV_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../libs/opencv-static/windows-x64") + link_directories(${OpenCV_DIR}/x64/vc16/staticlib) + include(${OpenCV_DIR}/x64/vc16/staticlib/OpenCVConfig.cmake) + +else () + set(OpenCV_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../libs/opencv-static/windows-x86") + link_directories(${OpenCV_DIR}/x86/vc16/staticlib) + include(${OpenCV_DIR}/x86/vc16/staticlib/OpenCVConfig.cmake) +endif () + +set(OpenCV_LIBS opencv_core;opencv_features2d;opencv_flann;opencv_highgui;opencv_imgcodecs;opencv_imgproc;opencv_photo;opencv_video) + +add_library(LunaOCR SHARED OCR.cpp ${versioninfo}) +target_link_libraries(LunaOCR ${OnnxRuntime_LIBS} ${OpenCV_LIBS} Clipper2Lib) +target_include_directories(LunaOCR PRIVATE ${OpenCV_DIR}/include) +target_precompile_headers(LunaOCR REUSE_FROM pch) + +if(${CMAKE_SIZEOF_VOID_P} EQUAL 8) + set_target_properties(LunaOCR PROPERTIES OUTPUT_NAME "LunaOCR64") +else() + set_target_properties(LunaOCR PROPERTIES OUTPUT_NAME "LunaOCR32") +endif() \ No newline at end of file diff --git a/src/plugins/LunaOCR/OCR.cpp b/src/plugins/LunaOCR/OCR.cpp new file mode 100644 index 00000000..21f60434 --- /dev/null +++ b/src/plugins/LunaOCR/OCR.cpp @@ -0,0 +1,702 @@ +#include +#include +#include +#include +#include +typedef std::vector TextBox; +typedef std::string TextLine; +typedef std::pair TextBlock; + +struct ScaleParam { + int srcWidth; + int srcHeight; + int dstWidth; + int dstHeight; + float ratioWidth; + float ratioHeight; +}; +#define getinputoutputNames(Func1, vec, Func2) \ + do \ + { \ + Ort::AllocatorWithDefaultOptions allocator; \ + const size_t numInputNodes = session->Func1(); \ + \ + vec.reserve(numInputNodes); \ + std::vector input_node_dims; \ + \ + for (size_t i = 0; i < numInputNodes; i++) \ + { \ + auto inputName = session->Func2(i, allocator); \ + vec.push_back(std::move(inputName)); \ + } \ + } while (0); + +class CommonOnnxModel +{ + std::vector inputNamesPtr; + std::vector outputNamesPtr; + std::unique_ptr session; + Ort::Env env = Ort::Env(ORT_LOGGING_LEVEL_ERROR); + Ort::SessionOptions sessionOptions = Ort::SessionOptions(); + const std::array meanValues; + const std::array normValues; + + std::vector substractMeanNormalize(cv::Mat &src, const float *meanVals, const float *normVals) + { + auto inputTensorSize = src.cols * src.rows * src.channels(); + std::vector inputTensorValues(inputTensorSize); + size_t numChannels = src.channels(); + size_t imageSize = src.cols * src.rows; + + for (size_t pid = 0; pid < imageSize; pid++) + { + for (size_t ch = 0; ch < numChannels; ++ch) + { + float data = (float)(src.data[pid * numChannels + ch] * normVals[ch] - meanVals[ch] * normVals[ch]); + inputTensorValues[ch * imageSize + pid] = data; + } + } + return inputTensorValues; + } + + void setNumThread(int numOfThread) + { + sessionOptions.SetInterOpNumThreads(numOfThread); + sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED); + } + +public: + std::pair, std::vector> RunSession(cv::Mat src) + { + auto inputTensorValues = substractMeanNormalize(src, meanValues.data(), normValues.data()); + std::array inputShape{1, src.channels(), src.rows, src.cols}; + auto memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + Ort::Value inputTensor = Ort::Value::CreateTensor(memoryInfo, inputTensorValues.data(), + inputTensorValues.size(), inputShape.data(), + inputShape.size()); + assert(inputTensor.IsTensor()); + std::vector inputNames = {inputNamesPtr.data()->get()}; + std::vector outputNames = {outputNamesPtr.data()->get()}; + auto outputTensor = session->Run(Ort::RunOptions{nullptr}, inputNames.data(), &inputTensor, + inputNames.size(), outputNames.data(), outputNames.size()); + assert(outputTensor.size() == 1 && outputTensor.front().IsTensor()); + std::vector outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape(); + auto outputCount = outputTensor.front().GetTensorTypeAndShapeInfo().GetElementCount(); + float *floatArray = outputTensor.front().GetTensorMutableData(); + std::vector outputData(floatArray, floatArray + outputCount); + return {outputData, outputShape}; + } + CommonOnnxModel(const std::wstring &path, const std::array &_meanValues, const std::array &_normValues, int numOfThread = 4) : meanValues(_meanValues), normValues(_normValues) + { + setNumThread(numOfThread); + session = std::make_unique(env, path.c_str(), sessionOptions); + getinputoutputNames(GetInputCount, inputNamesPtr, GetInputNameAllocated); + getinputoutputNames(GetOutputCount, outputNamesPtr, GetOutputNameAllocated); + } +}; + +class CrnnNet:public CommonOnnxModel{ +public: + CrnnNet(const std::wstring &pathStr, const std::wstring &keysPath, int numOfThread); + std::vector getTextLines(std::vector &partImg); + +private: + const float meanValues[3] = {127.5, 127.5, 127.5}; + const float normValues[3] = {1.0 / 127.5, 1.0 / 127.5, 1.0 / 127.5}; + const int dstHeight = 48; + + std::vector keys; + + TextLine scoreToTextLine(const std::vector &outputData, size_t h, size_t w); + + TextLine getTextLine(const cv::Mat &src); +}; + +class DbNet:public CommonOnnxModel{ +public: + DbNet(const std::wstring &pathStr, int numOfThread); + std::vector getTextBoxes(cv::Mat &src, ScaleParam &s, float boxScoreThresh, + float boxThresh, float unClipRatio); + +private: + const float meanValues[3] = {0.485 * 255, 0.456 * 255, 0.406 * 255}; + const float normValues[3] = {1.0 / 0.229 / 255.0, 1.0 / 0.224 / 255.0, 1.0 / 0.225 / 255.0}; +}; + +//onnxruntime init windows +ScaleParam getScaleParam(cv::Mat &src, const float scale) { + int srcWidth = src.cols; + int srcHeight = src.rows; + int dstWidth = int((float) srcWidth * scale); + int dstHeight = int((float) srcHeight * scale); + if (dstWidth % 32 != 0) { + dstWidth = (dstWidth / 32 - 1) * 32; + dstWidth = (std::max)(dstWidth, 32); + } + if (dstHeight % 32 != 0) { + dstHeight = (dstHeight / 32 - 1) * 32; + dstHeight = (std::max)(dstHeight, 32); + } + float scaleWidth = (float) dstWidth / (float) srcWidth; + float scaleHeight = (float) dstHeight / (float) srcHeight; + return {srcWidth, srcHeight, dstWidth, dstHeight, scaleWidth, scaleHeight}; +} + +ScaleParam getScaleParam(cv::Mat &src, const int targetSize) { + int srcWidth, srcHeight, dstWidth, dstHeight; + srcWidth = dstWidth = src.cols; + srcHeight = dstHeight = src.rows; + + float ratio = 1.f; + if (srcWidth > srcHeight) { + ratio = float(targetSize) / float(srcWidth); + } else { + ratio = float(targetSize) / float(srcHeight); + } + dstWidth = int(float(srcWidth) * ratio); + dstHeight = int(float(srcHeight) * ratio); + if (dstWidth % 32 != 0) { + dstWidth = (dstWidth / 32) * 32; + dstWidth = (std::max)(dstWidth, 32); + } + if (dstHeight % 32 != 0) { + dstHeight = (dstHeight / 32) * 32; + dstHeight = (std::max)(dstHeight, 32); + } + float ratioWidth = (float) dstWidth / (float) srcWidth; + float ratioHeight = (float) dstHeight / (float) srcHeight; + return {srcWidth, srcHeight, dstWidth, dstHeight, ratioWidth, ratioHeight}; +} + +std::vector getBox(const cv::RotatedRect &rect) { + cv::Point2f vertices[4]; + rect.points(vertices); + //std::vector ret(4); + std::vector ret2(vertices, vertices + sizeof(vertices) / sizeof(vertices[0])); + //memcpy(vertices, &ret[0], ret.size() * sizeof(ret[0])); + return ret2; +} + +cv::Mat getRotateCropImage(const cv::Mat &src, std::vector box) { + cv::Mat image; + src.copyTo(image); + std::vector points = box; + + int collectX[4] = {box[0].x, box[1].x, box[2].x, box[3].x}; + int collectY[4] = {box[0].y, box[1].y, box[2].y, box[3].y}; + int left = int(*std::min_element(collectX, collectX + 4)); + int right = int(*std::max_element(collectX, collectX + 4)); + int top = int(*std::min_element(collectY, collectY + 4)); + int bottom = int(*std::max_element(collectY, collectY + 4)); + + cv::Mat imgCrop; + image(cv::Rect(left, top, right - left, bottom - top)).copyTo(imgCrop); + + for (auto &point: points) { + point.x -= left; + point.y -= top; + } + + int imgCropWidth = int(sqrt(pow(points[0].x - points[1].x, 2) + + pow(points[0].y - points[1].y, 2))); + int imgCropHeight = int(sqrt(pow(points[0].x - points[3].x, 2) + + pow(points[0].y - points[3].y, 2))); + + cv::Point2f ptsDst[4]; + ptsDst[0] = cv::Point2f(0., 0.); + ptsDst[1] = cv::Point2f(imgCropWidth, 0.); + ptsDst[2] = cv::Point2f(imgCropWidth, imgCropHeight); + ptsDst[3] = cv::Point2f(0.f, imgCropHeight); + + cv::Point2f ptsSrc[4]; + ptsSrc[0] = cv::Point2f(points[0].x, points[0].y); + ptsSrc[1] = cv::Point2f(points[1].x, points[1].y); + ptsSrc[2] = cv::Point2f(points[2].x, points[2].y); + ptsSrc[3] = cv::Point2f(points[3].x, points[3].y); + + cv::Mat M = cv::getPerspectiveTransform(ptsSrc, ptsDst); + + cv::Mat partImg; + cv::warpPerspective(imgCrop, partImg, M, + cv::Size(imgCropWidth, imgCropHeight), + cv::BORDER_REPLICATE); + + // if (float(partImg.rows) >= float(partImg.cols) * 1.5) { + // cv::Mat srcCopy = cv::Mat(partImg.rows, partImg.cols, partImg.depth()); + // cv::transpose(partImg, srcCopy); + // cv::flip(srcCopy, srcCopy, 0); + // return srcCopy; + // } else { + // return partImg; + // } + + return partImg; +} + +bool cvPointCompare(const cv::Point &a, const cv::Point &b) { + return a.x < b.x; +} + +std::vector getMinBoxes(const cv::RotatedRect &boxRect, float &maxSideLen) { + maxSideLen = std::max(boxRect.size.width, boxRect.size.height); + std::vector boxPoint = getBox(boxRect); + std::sort(boxPoint.begin(), boxPoint.end(), cvPointCompare); + int index1, index2, index3, index4; + if (boxPoint[1].y > boxPoint[0].y) { + index1 = 0; + index4 = 1; + } else { + index1 = 1; + index4 = 0; + } + if (boxPoint[3].y > boxPoint[2].y) { + index2 = 2; + index3 = 3; + } else { + index2 = 3; + index3 = 2; + } + std::vector minBox(4); + minBox[0] = boxPoint[index1]; + minBox[1] = boxPoint[index2]; + minBox[2] = boxPoint[index3]; + minBox[3] = boxPoint[index4]; + return minBox; +} + +template +inline T clamp(T x, T min, T max) +{ + if (x > max) + return max; + if (x < min) + return min; + return x; +} +float boxScoreFast(const std::vector &boxes, const cv::Mat &pred) { + int width = pred.cols; + int height = pred.rows; + + float arrayX[4] = {boxes[0].x, boxes[1].x, boxes[2].x, boxes[3].x}; + float arrayY[4] = {boxes[0].y, boxes[1].y, boxes[2].y, boxes[3].y}; + + int minX = clamp(int(std::floor(*(std::min_element(arrayX, arrayX + 4)))), 0, width - 1); + int maxX = clamp(int(std::ceil(*(std::max_element(arrayX, arrayX + 4)))), 0, width - 1); + int minY = clamp(int(std::floor(*(std::min_element(arrayY, arrayY + 4)))), 0, height - 1); + int maxY = clamp(int(std::ceil(*(std::max_element(arrayY, arrayY + 4)))), 0, height - 1); + + cv::Mat mask = cv::Mat::zeros(maxY - minY + 1, maxX - minX + 1, CV_8UC1); + + cv::Point box[4]; + box[0] = cv::Point(int(boxes[0].x) - minX, int(boxes[0].y) - minY); + box[1] = cv::Point(int(boxes[1].x) - minX, int(boxes[1].y) - minY); + box[2] = cv::Point(int(boxes[2].x) - minX, int(boxes[2].y) - minY); + box[3] = cv::Point(int(boxes[3].x) - minX, int(boxes[3].y) - minY); + const cv::Point *pts[1] = {box}; + int npts[] = {4}; + cv::fillPoly(mask, pts, npts, 1, cv::Scalar(1)); + + cv::Mat croppedImg; + pred(cv::Rect(minX, minY, maxX - minX + 1, maxY - minY + 1)) + .copyTo(croppedImg); + + auto score = (float) cv::mean(croppedImg, mask)[0]; + return score; +} + +float getContourArea(const std::vector &box, float unClipRatio) { + size_t size = box.size(); + float area = 0.0f; + float dist = 0.0f; + for (size_t i = 0; i < size; i++) { + area += box[i].x * box[(i + 1) % size].y - + box[i].y * box[(i + 1) % size].x; + dist += sqrtf((box[i].x - box[(i + 1) % size].x) * + (box[i].x - box[(i + 1) % size].x) + + (box[i].y - box[(i + 1) % size].y) * + (box[i].y - box[(i + 1) % size].y)); + } + area = fabs(float(area / 2.0)); + + return area * unClipRatio / dist; +} + +cv::RotatedRect unClip(std::vector box, float unClipRatio) { + float distance = getContourArea(box, unClipRatio); + + Clipper2Lib::ClipperOffset offset; + Clipper2Lib::Path64 p; + p.push_back(Clipper2Lib::Point64(int(box[0].x), int(box[0].y))); + p.push_back(Clipper2Lib::Point64(int(box[1].x), int(box[1].y))); + p.push_back(Clipper2Lib::Point64(int(box[2].x), int(box[2].y))); + p.push_back(Clipper2Lib::Point64(int(box[3].x), int(box[3].y))); + offset.AddPath(p, Clipper2Lib::JoinType::Round, Clipper2Lib::EndType::Polygon); + Clipper2Lib::Paths64 soln; + offset.Execute(distance, soln); + std::vector points; + + for (size_t j = 0; j < soln.size(); j++) { + for (size_t i = 0; i < soln[soln.size() - 1].size(); i++) { + points.emplace_back(cv::Point2f{float(soln[j][i].x), float(soln[j][i].y)}); + } + } + cv::RotatedRect res; + if (points.empty()) { + res = cv::RotatedRect(cv::Point2f(0, 0), cv::Size2f(1, 1), 0); + } else { + res = cv::minAreaRect(points); + } + return res; +} +CrnnNet::CrnnNet(const std::wstring &pathStr, const std::wstring &keysPath, int numOfThread) : CommonOnnxModel(pathStr, {127.5, 127.5, 127.5}, {1.0 / 127.5, 1.0 / 127.5, 1.0 / 127.5}, numOfThread) +{ + // load keys + std::ifstream in(keysPath.c_str()); + std::string line; + if (in) + { + while (getline(in, line)) + { // line中不包括每行的换行符 + keys.push_back(line); + } + } + else + { + return; + } + keys.insert(keys.begin(), "#"); + keys.emplace_back(" "); +} + +template +inline static size_t argmax(ForwardIterator first, ForwardIterator last) +{ + return std::distance(first, std::max_element(first, last)); +} + +TextLine CrnnNet::scoreToTextLine(const std::vector &outputData, size_t h, size_t w) +{ + auto keySize = keys.size(); + auto dataSize = outputData.size(); + std::string strRes; + std::vector scores; + size_t lastIndex = 0; + size_t maxIndex; + float maxValue; + + for (size_t i = 0; i < h; i++) + { + size_t start = i * w; + size_t stop = (i + 1) * w; + if (stop > dataSize - 1) + { + stop = (i + 1) * w - 1; + } + maxIndex = int(argmax(&outputData[start], &outputData[stop])); + maxValue = float(*std::max_element(&outputData[start], &outputData[stop])); + + if (maxIndex > 0 && maxIndex < keySize && (!(i > 0 && maxIndex == lastIndex))) + { + scores.emplace_back(maxValue); + strRes.append(keys[maxIndex]); + } + lastIndex = maxIndex; + } + return strRes; +} + +TextLine CrnnNet::getTextLine(const cv::Mat &src) +{ + float scale = (float)dstHeight / (float)src.rows; + int dstWidth = int((float)src.cols * scale); + cv::Mat srcResize; + resize(src, srcResize, cv::Size(dstWidth, dstHeight)); + auto &&[outputData, outputShape] = RunSession(srcResize); + return scoreToTextLine(outputData, outputShape[1], outputShape[2]); +} + +std::vector CrnnNet::getTextLines(std::vector &partImg) +{ + int size = partImg.size(); + std::vector textLines(size); + for (int i = 0; i < size; ++i) + { + TextLine textLine = getTextLine(partImg[i]); + textLines[i] = textLine; + } + return textLines; +} + +DbNet::DbNet(const std::wstring &pathStr, int numOfThread) : CommonOnnxModel(pathStr, {0.485 * 255, 0.456 * 255, 0.406 * 255}, {1.0 / 0.229 / 255.0, 1.0 / 0.224 / 255.0, 1.0 / 0.225 / 255.0}, numOfThread) +{ +} + +std::vector findRsBoxes(const cv::Mat &predMat, const cv::Mat &dilateMat, ScaleParam &s, + const float boxScoreThresh, const float unClipRatio) +{ + const int longSideThresh = 3; // minBox 长边门限 + const int maxCandidates = 1000; + + std::vector> contours; + std::vector hierarchy; + + cv::findContours(dilateMat, contours, hierarchy, cv::RETR_LIST, + cv::CHAIN_APPROX_SIMPLE); + + size_t numContours = contours.size() >= maxCandidates ? maxCandidates : contours.size(); + + std::vector rsBoxes; + + for (size_t i = 0; i < numContours; i++) + { + if (contours[i].size() <= 2) + { + continue; + } + cv::RotatedRect minAreaRect = cv::minAreaRect(contours[i]); + + float longSide; + std::vector minBoxes = getMinBoxes(minAreaRect, longSide); + + if (longSide < longSideThresh) + { + continue; + } + + float boxScore = boxScoreFast(minBoxes, predMat); + if (boxScore < boxScoreThresh) + continue; + + //-----unClip----- + cv::RotatedRect clipRect = unClip(minBoxes, unClipRatio); + if (clipRect.size.height < 1.001 && clipRect.size.width < 1.001) + { + continue; + } + //-----unClip----- + + std::vector clipMinBoxes = getMinBoxes(clipRect, longSide); + if (longSide < longSideThresh + 2) + continue; + + std::vector intClipMinBoxes; + + for (auto &clipMinBox : clipMinBoxes) + { + float x = clipMinBox.x / s.ratioWidth; + float y = clipMinBox.y / s.ratioHeight; + int ptX = (std::min)((std::max)(int(x), 0), s.srcWidth - 1); + int ptY = (std::min)((std::max)(int(y), 0), s.srcHeight - 1); + cv::Point point{ptX, ptY}; + intClipMinBoxes.push_back(point); + } + rsBoxes.push_back(intClipMinBoxes); + } + reverse(rsBoxes.begin(), rsBoxes.end()); + return rsBoxes; +} + +std::vector DbNet::getTextBoxes(cv::Mat &src, ScaleParam &s, float boxScoreThresh, float boxThresh, float unClipRatio) +{ + cv::Mat srcResize; + resize(src, srcResize, cv::Size(s.dstWidth, s.dstHeight)); + auto &&[outputData, outputShape] = RunSession(srcResize); + + //-----Data preparation----- + int outHeight = (int)outputShape[2]; + int outWidth = (int)outputShape[3]; + size_t area = outHeight * outWidth; + + std::vector predData(area, 0.0); + std::vector cbufData(area, ' '); + + for (int i = 0; i < area; i++) + { + predData[i] = float(outputData[i]); + cbufData[i] = (unsigned char)((outputData[i]) * 255); + } + + cv::Mat predMat(outHeight, outWidth, CV_32F, (float *)predData.data()); + cv::Mat cBufMat(outHeight, outWidth, CV_8UC1, (unsigned char *)cbufData.data()); + + //-----boxThresh----- + const double maxValue = 255; + const double threshold = boxThresh * 255; + cv::Mat thresholdMat; + cv::threshold(cBufMat, thresholdMat, threshold, maxValue, cv::THRESH_BINARY); + + //-----dilate----- + cv::Mat dilateMat; + cv::Mat dilateElement = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(2, 2)); + cv::dilate(thresholdMat, dilateMat, dilateElement); + + return findRsBoxes(predMat, dilateMat, s, boxScoreThresh, unClipRatio); +} + +class OcrLite +{ +public: + OcrLite(const std::wstring &detPath, + const std::wstring &recPath, const std::wstring &keysPath, int numOfThread) : crnnNet(recPath, keysPath, numOfThread), dbNet(detPath, numOfThread) + { + } + + std::vector detect(const void *binptr, size_t size, + int padding, int maxSideLen, + float boxScoreThresh, float boxThresh, float unClipRatio, bool rotate); + +private: + DbNet dbNet; + CrnnNet crnnNet; + + std::vector getPartImages(cv::Mat &src, std::vector &textBoxes); + + std::vector detect_internal(cv::Mat &src, cv::Rect &originRect, ScaleParam &scale, + float boxScoreThresh = 0.6f, float boxThresh = 0.3f, + float unClipRatio = 2.0f, bool rotate = true); +}; + +cv::Mat makePadding(cv::Mat &src, const int padding) +{ + if (padding <= 0) + return src; + cv::Scalar paddingScalar = {255, 255, 255}; + cv::Mat paddingSrc; + cv::copyMakeBorder(src, paddingSrc, padding, padding, padding, padding, cv::BORDER_ISOLATED, paddingScalar); + return paddingSrc; +} + +std::vector OcrLite::detect(const void *binptr, size_t size, + const int padding, const int maxSideLen, + float boxScoreThresh, float boxThresh, float unClipRatio, bool rotate) +{ + std::vector bytes{(uchar *)binptr, (uchar *)binptr + size}; + cv::Mat originSrc = imdecode(bytes, cv::IMREAD_COLOR); // default : BGR + int originMaxSide = (std::max)(originSrc.cols, originSrc.rows); + int resize; + if (maxSideLen <= 0 || maxSideLen > originMaxSide) + { + resize = originMaxSide; + } + else + { + resize = maxSideLen; + } + resize += 2 * padding; + cv::Rect paddingRect(padding, padding, originSrc.cols, originSrc.rows); + cv::Mat paddingSrc = makePadding(originSrc, padding); + ScaleParam scale = getScaleParam(paddingSrc, resize); + return detect_internal(paddingSrc, paddingRect, scale, + boxScoreThresh, boxThresh, unClipRatio, rotate); +} + +std::vector OcrLite::getPartImages(cv::Mat &src, std::vector &textBoxes) +{ + std::vector partImages; + for (size_t i = 0; i < textBoxes.size(); ++i) + { + cv::Mat partImg = getRotateCropImage(src, textBoxes[i]); + partImages.emplace_back(partImg); + } + return partImages; +} + +cv::Mat matRotateClockWise180(cv::Mat src) { + flip(src, src, 0); + flip(src, src, 1); + return src; +} + +cv::Mat matRotateClockWise90(cv::Mat src) { + transpose(src, src); + flip(src, src, 1); + return src; +} + +std::vector OcrLite::detect_internal(cv::Mat &src, cv::Rect &originRect, ScaleParam &scale, + float boxScoreThresh, float boxThresh, float unClipRatio, bool rotate) +{ + + std::vector textBoxes = dbNet.getTextBoxes(src, scale, boxScoreThresh, boxThresh, unClipRatio); + std::vector partImages = getPartImages(src, textBoxes); + for (size_t i = 0; i < partImages.size(); ++i) + { + if (rotate) + { + partImages.at(i) = matRotateClockWise180(partImages[i]); + partImages.at(i) = matRotateClockWise90(partImages[i]); + } + } + + std::vector textLines = crnnNet.getTextLines(partImages); + + std::vector textBlocks; + for (size_t i = 0; i < textLines.size(); ++i) + { + std::vector boxPoint = std::vector(4); + int padding = originRect.x; // padding conversion + boxPoint[0] = cv::Point(textBoxes[i][0].x - padding, textBoxes[i][0].y - padding); + boxPoint[1] = cv::Point(textBoxes[i][1].x - padding, textBoxes[i][1].y - padding); + boxPoint[2] = cv::Point(textBoxes[i][2].x - padding, textBoxes[i][2].y - padding); + boxPoint[3] = cv::Point(textBoxes[i][3].x - padding, textBoxes[i][3].y - padding); + TextBlock textBlock{boxPoint, textLines[i]}; + textBlocks.emplace_back(textBlock); + } + + return textBlocks; +} + +#define DECLARE_API extern "C" __declspec(dllexport) + +struct ocrpoints +{ + int x1, y1, x2, y2, x3, y3, x4, y4; +}; +DECLARE_API OcrLite *OcrInit(const wchar_t *szDetModel, const wchar_t *szRecModel, const wchar_t *szKeyPath, int nThreads) +{ + OcrLite *pOcrObj = nullptr; + try + { + pOcrObj = new OcrLite(szDetModel, szRecModel, szKeyPath, nThreads); + } + catch (...) + { + } + if (pOcrObj) + { + return pOcrObj; + } + else + { + return nullptr; + } +} + +DECLARE_API void OcrDetect(OcrLite *pOcrObj, const void *binptr, size_t size, bool rotate, void (*cb)(ocrpoints, const char *)) +{ + if (!pOcrObj) + return; + + try + { + auto result = pOcrObj->detect(binptr, size, 50, 1024, 0.1, 0.1, 2.0, rotate); + + for (auto item : result) + { + cb({item.first[0].x, item.first[0].y, + item.first[1].x, item.first[1].y, + item.first[2].x, item.first[2].y, + item.first[3].x, item.first[3].y}, + item.second.c_str()); + } + } + catch (...) + { + } +} + +DECLARE_API void OcrDestroy(OcrLite *pOcrObj) +{ + if (pOcrObj) + delete pOcrObj; +} \ No newline at end of file diff --git a/src/plugins/applicationloopbackaudio/CMakeLists.txt b/src/plugins/applicationloopbackaudio/CMakeLists.txt index f26077fa..4c70445c 100644 --- a/src/plugins/applicationloopbackaudio/CMakeLists.txt +++ b/src/plugins/applicationloopbackaudio/CMakeLists.txt @@ -14,4 +14,4 @@ generate_product_version( add_library(loopbackaudio MODULE runer.cpp LoopbackCapture.cpp ${versioninfo}) target_precompile_headers(loopbackaudio REUSE_FROM pch) - target_link_libraries(loopbackaudio Mfplat mfuuid ) \ No newline at end of file + target_link_libraries(loopbackaudio Mfplat mfuuid wil) \ No newline at end of file diff --git a/src/plugins/applicationloopbackaudio/runer.cpp b/src/plugins/applicationloopbackaudio/runer.cpp index e0daa989..e73b0d12 100644 --- a/src/plugins/applicationloopbackaudio/runer.cpp +++ b/src/plugins/applicationloopbackaudio/runer.cpp @@ -1,8 +1,8 @@ #include "LoopbackCapture.h" -#define DECLARE extern "C" __declspec(dllexport) +#define DECLARE_API extern "C" __declspec(dllexport) -DECLARE void StartCaptureAsync(void (*datacb)(void *ptr, size_t size), void (*handlecb)(HANDLE)) +DECLARE_API void StartCaptureAsync(void (*datacb)(void *ptr, size_t size), void (*handlecb)(HANDLE)) { auto mutex = CreateSemaphoreW(NULL, 0, 1, NULL); handlecb(mutex); @@ -14,7 +14,7 @@ DECLARE void StartCaptureAsync(void (*datacb)(void *ptr, size_t size), void (*ha datacb(loopbackCapture.buffer.data(), loopbackCapture.buffer.size()); } -DECLARE void StopCaptureAsync(HANDLE m) +DECLARE_API void StopCaptureAsync(HANDLE m) { ReleaseSemaphore(m, 1, NULL); } \ No newline at end of file diff --git a/src/plugins/libs/Clipper2 b/src/plugins/libs/Clipper2 new file mode 160000 index 00000000..ed98928c --- /dev/null +++ b/src/plugins/libs/Clipper2 @@ -0,0 +1 @@ +Subproject commit ed98928c66707988d4eb2c49a31c41380a08931c diff --git a/src/plugins/libs/libs.cmake b/src/plugins/libs/libs.cmake index f596058d..3cfe7694 100644 --- a/src/plugins/libs/libs.cmake +++ b/src/plugins/libs/libs.cmake @@ -1,16 +1,26 @@  -set(Detours ${CMAKE_CURRENT_LIST_DIR}/Detours/src/creatwth.cpp ${CMAKE_CURRENT_LIST_DIR}/Detours/src/detours.cpp ${CMAKE_CURRENT_LIST_DIR}/Detours/src/modules.cpp ${CMAKE_CURRENT_LIST_DIR}/Detours/src/disasm.cpp) -include_directories(${CMAKE_CURRENT_LIST_DIR}) -include_directories(${CMAKE_CURRENT_LIST_DIR}/Detours/src) +add_library(Detours ${CMAKE_CURRENT_LIST_DIR}/Detours/src/creatwth.cpp ${CMAKE_CURRENT_LIST_DIR}/Detours/src/detours.cpp ${CMAKE_CURRENT_LIST_DIR}/Detours/src/modules.cpp ${CMAKE_CURRENT_LIST_DIR}/Detours/src/disasm.cpp) +target_include_directories(Detours PUBLIC ${CMAKE_CURRENT_LIST_DIR}/Detours/src) -include_directories(${CMAKE_CURRENT_LIST_DIR}/wil/include) -include_directories(${CMAKE_CURRENT_LIST_DIR}/miniaudio) -include_directories(${CMAKE_CURRENT_LIST_DIR}/tinymp3) -include_directories(${CMAKE_CURRENT_LIST_DIR}/rapidfuzz-cpp) +add_library(nlohmann INTERFACE) +target_include_directories(nlohmann INTERFACE ${CMAKE_CURRENT_LIST_DIR}) -include_directories(${CMAKE_CURRENT_LIST_DIR}/webview2/Microsoft.Web.WebView2.1.0.2535.41/build/native/include) +add_library(wil INTERFACE) +target_include_directories(wil INTERFACE ${CMAKE_CURRENT_LIST_DIR}/wil/include) + +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/tinymp3) +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/rapidfuzz-cpp) + + +add_library(webview2 INTERFACE) +target_include_directories(webview2 INTERFACE ${CMAKE_CURRENT_LIST_DIR}/webview2/Microsoft.Web.WebView2.1.0.2535.41/build/native/include) if(${CMAKE_SIZEOF_VOID_P} EQUAL 4) set(LTLPlatform "Win32") endif() -include("${CMAKE_CURRENT_LIST_DIR}/VC-LTL helper for cmake.cmake") \ No newline at end of file +include("${CMAKE_CURRENT_LIST_DIR}/VC-LTL helper for cmake.cmake") + + +file(GLOB Clipper2LibSrc ${CMAKE_CURRENT_LIST_DIR}/Clipper2/CPP/Clipper2Lib/src/*.cpp) +add_library(Clipper2Lib ${Clipper2LibSrc}) +target_include_directories(Clipper2Lib PUBLIC ${CMAKE_CURRENT_LIST_DIR}/Clipper2/CPP/Clipper2Lib/include) \ No newline at end of file diff --git a/src/plugins/libs/onnxruntime-static/OnnxRuntimeWrapper.cmake b/src/plugins/libs/onnxruntime-static/OnnxRuntimeWrapper.cmake new file mode 100644 index 00000000..33a3a2e8 --- /dev/null +++ b/src/plugins/libs/onnxruntime-static/OnnxRuntimeWrapper.cmake @@ -0,0 +1,16 @@ +if (APPLE) + message("配置macOS OnnxRuntime 路径: ${CMAKE_CURRENT_LIST_DIR}/macos") + set(OnnxRuntime_DIR "${CMAKE_CURRENT_LIST_DIR}/macos") +elseif (WIN32) + if (CMAKE_CL_64) + message("配置WINDOWS OnnxRuntime x64 路径: ${CMAKE_CURRENT_LIST_DIR}/windows-x64") + set(OnnxRuntime_DIR "${CMAKE_CURRENT_LIST_DIR}/windows-x64") + else () + message("配置WINDOWS OnnxRuntime x86 路径: ${CMAKE_CURRENT_LIST_DIR}/windows-x86") + set(OnnxRuntime_DIR "${CMAKE_CURRENT_LIST_DIR}/windows-x86") + endif () +elseif (UNIX) + message("配置Linux OnnxRuntime 路径: ${CMAKE_CURRENT_LIST_DIR}/linux") + set(OnnxRuntime_DIR "${CMAKE_CURRENT_LIST_DIR}/linux") +endif () + diff --git a/src/plugins/libs/opencv-static/OpenCVWrapperConfig.cmake b/src/plugins/libs/opencv-static/OpenCVWrapperConfig.cmake new file mode 100644 index 00000000..3855e276 --- /dev/null +++ b/src/plugins/libs/opencv-static/OpenCVWrapperConfig.cmake @@ -0,0 +1,15 @@ +if (WIN32) + if (CMAKE_CL_64) + message("配置WINDOWS OpenCV x64 路径: ${CMAKE_CURRENT_LIST_DIR}/windows-x64") + set(OpenCV_DIR "${CMAKE_CURRENT_LIST_DIR}/windows-x64") + else () + message("配置WINDOWS OpenCV x86 路径: ${CMAKE_CURRENT_LIST_DIR}/windows-x86") + set(OpenCV_DIR "${CMAKE_CURRENT_LIST_DIR}/windows-x86") + endif () +elseif (APPLE) + message("配置macOS OpenCV 路径: ${CMAKE_CURRENT_LIST_DIR}/macos/lib/cmake/opencv4") + set(OpenCV_DIR "${CMAKE_CURRENT_LIST_DIR}/macos/lib/cmake/opencv4") +elseif (UNIX) + message("配置Linux OpenCV 路径: ${CMAKE_CURRENT_LIST_DIR}/linux/lib/cmake/opencv4") + set(OpenCV_DIR "${CMAKE_CURRENT_LIST_DIR}/linux/lib/cmake/opencv4") +endif () diff --git a/src/plugins/libs/tinymp3 b/src/plugins/libs/tinymp3 index 9781d63b..57972618 160000 --- a/src/plugins/libs/tinymp3 +++ b/src/plugins/libs/tinymp3 @@ -1 +1 @@ -Subproject commit 9781d63bf7d057457b25b5e77492bcf2cfa109c5 +Subproject commit 579726185979f877f2672b956b23538ff00c08ba diff --git a/src/plugins/pch.h b/src/plugins/pch.h index cf1b3e23..01fffbfd 100644 --- a/src/plugins/pch.h +++ b/src/plugins/pch.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -69,7 +70,6 @@ #include #include -#include #pragma comment(lib, "shlwapi.lib") diff --git a/src/plugins/scripts/copytarget.py b/src/plugins/scripts/copytarget.py index b2f54ac5..a91bb7e9 100644 --- a/src/plugins/scripts/copytarget.py +++ b/src/plugins/scripts/copytarget.py @@ -6,6 +6,7 @@ if x86: shutil.copy('../builds/_x86/winrtutils32.dll','../../files/plugins/DLL32') shutil.copy('../builds/_x86/winsharedutils32.dll','../../files/plugins/DLL32') shutil.copy('../builds/_x86/wcocr.dll','../../files/plugins/DLL32') + shutil.copy('../builds/_x86/LunaOCR32.dll','../../files/plugins/DLL32') else: shutil.copy('../builds/_x64/shareddllproxy64.exe','../../files/plugins') shutil.copy('../builds/_x64/loopbackaudio.dll','../../files/plugins/DLL64') @@ -13,3 +14,4 @@ else: shutil.copy('../builds/_x64/winrtutils64.dll','../../files/plugins/DLL64') shutil.copy('../builds/_x64/winsharedutils64.dll','../../files/plugins/DLL64') shutil.copy('../builds/_x64/wcocr.dll','../../files/plugins/DLL64') + shutil.copy('../builds/_x64/LunaOCR64.dll','../../files/plugins/DLL64') diff --git a/src/plugins/scripts/fetchwebview2.py b/src/plugins/scripts/fetchwebview2.py index caf1b6c8..a4587d88 100644 --- a/src/plugins/scripts/fetchwebview2.py +++ b/src/plugins/scripts/fetchwebview2.py @@ -8,7 +8,7 @@ nuget_exe = os.path.join(target, "nuget.exe") print(nuget_exe) if os.path.exists(nuget_exe) == False: os.system( - rf'curl -sSLo "{nuget_exe}" https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' + rf'curl -SLo "{nuget_exe}" https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' ) mswebview2_dir = os.path.join(target, f"Microsoft.Web.WebView2.{mswebview2_version}") @@ -20,3 +20,34 @@ if os.path.exists(mswebview2_dir) == False: subprocess.run( rf'"{nuget_exe}" install Microsoft.Web.Webview2 -Verbosity quiet -Version "{mswebview2_version}" -OutputDirectory {target}' ) + + +onnx = os.path.normpath( + os.path.join( + os.path.dirname(__file__), "../libs/onnxruntime-static/windows-x64/lib/onnx.lib" + ) +) +opencv = os.path.normpath( + os.path.join( + os.path.dirname(__file__), + r"..\libs\opencv-static\windows-x64\x64\vc16\staticlib\opencv_core470.lib", + ) +) +onnx_1 = os.path.normpath( + os.path.join(os.path.dirname(__file__), "../libs/onnxruntime-static.zip") +) +opencv_1 = os.path.normpath( + os.path.join(os.path.dirname(__file__), r"..\libs\opencv-static.zip") +) + + +if os.path.exists(onnx) == False: + os.system( + rf'curl -SLo "{onnx_1}" https://github.com/HIllya51/RESOURCES/releases/download/common/onnxruntime-static.zip' + ) + os.system(rf'7z x -y "{onnx_1}" -o{os.path.dirname(opencv_1)}') +if os.path.exists(opencv) == False: + os.system( + rf'curl -SLo "{opencv_1}" https://github.com/HIllya51/RESOURCES/releases/download/common/opencv-static.zip' + ) + os.system(rf'7z x -y "{opencv_1}" -o{os.path.dirname(opencv_1)}') diff --git a/src/plugins/shareddllproxy/CMakeLists.txt b/src/plugins/shareddllproxy/CMakeLists.txt index 0cb7b48c..cddc745a 100644 --- a/src/plugins/shareddllproxy/CMakeLists.txt +++ b/src/plugins/shareddllproxy/CMakeLists.txt @@ -11,18 +11,23 @@ generate_product_version( VERSION_PATCH ${VERSION_PATCH} ) +set(shareddllproxy_common shareddllproxy.cpp dllinject.cpp ntleas.cpp aspatch.cpp update.cpp ${versioninfo}) -add_executable(shareddllproxy shareddllproxy.cpp dllinject.cpp ntleas.cpp aspatch.cpp update.cpp ${versioninfo}) -target_precompile_headers(shareddllproxy REUSE_FROM pch) if(${CMAKE_SIZEOF_VOID_P} EQUAL 8) - - target_link_libraries(shareddllproxy Mfplat mfuuid) + add_executable(shareddllproxy ${shareddllproxy_common}) + +else() + add_executable(shareddllproxy ${shareddllproxy_common} Atlas.cpp eztrans.cpp dreye.cpp jbj7.cpp kingsoft.cpp le.cpp neospeech.cpp ../implsapi.cpp LR.cpp) + add_subdirectory(voiceroid2) +endif() + +target_precompile_headers(shareddllproxy REUSE_FROM pch) + +if(${CMAKE_SIZEOF_VOID_P} EQUAL 8) + target_link_libraries(shareddllproxy nlohmann Mfplat mfuuid) set_target_properties(shareddllproxy PROPERTIES OUTPUT_NAME "shareddllproxy64") else() - add_subdirectory(voiceroid2) - add_library(x86lib Atlas.cpp eztrans.cpp dreye.cpp jbj7.cpp kingsoft.cpp le.cpp neospeech.cpp ../implsapi.cpp LR.cpp ${Detours}) target_precompile_headers(voiceroid2 REUSE_FROM pch) - target_precompile_headers(x86lib REUSE_FROM pch) - target_link_libraries(shareddllproxy Mfplat mfuuid x86lib voiceroid2) + target_link_libraries(shareddllproxy nlohmann Mfplat mfuuid voiceroid2 Detours) set_target_properties(shareddllproxy PROPERTIES OUTPUT_NAME "shareddllproxy32") -endif() +endif() \ No newline at end of file diff --git a/src/plugins/shareddllproxy/aspatch.cpp b/src/plugins/shareddllproxy/aspatch.cpp index a7ef98d7..52ad1512 100644 --- a/src/plugins/shareddllproxy/aspatch.cpp +++ b/src/plugins/shareddllproxy/aspatch.cpp @@ -1,4 +1,6 @@ +#include + HANDLE runexe(const std::wstring &exe, const std::optional &startup_argument) { STARTUPINFOW si; diff --git a/src/plugins/wcocr/wcocr.cpp b/src/plugins/wcocr/wcocr.cpp index b3ae02ea..05f74bc4 100644 --- a/src/plugins/wcocr/wcocr.cpp +++ b/src/plugins/wcocr/wcocr.cpp @@ -1,8 +1,8 @@ #include #include -#define DECLARE extern "C" __declspec(dllexport) +#define DECLARE_API extern "C" __declspec(dllexport) -DECLARE void *wcocr_init(const wchar_t *wexe, const wchar_t *wwcdir) +DECLARE_API void *wcocr_init(const wchar_t *wexe, const wchar_t *wwcdir) { auto obj = new CWeChatOCR(wexe, wwcdir); if (obj->wait_connection(5000)) @@ -16,14 +16,14 @@ DECLARE void *wcocr_init(const wchar_t *wexe, const wchar_t *wwcdir) } } -DECLARE void wcocr_destroy(void *pobj) +DECLARE_API void wcocr_destroy(void *pobj) { if (!pobj) return; auto obj = reinterpret_cast(pobj); delete obj; } -DECLARE bool wcocr_ocr(void *pobj, const char *u8path, void (*cb)(int, int, int, int, LPCSTR)) +DECLARE_API bool wcocr_ocr(void *pobj, const char *u8path, void (*cb)(int, int, int, int, LPCSTR)) { if (!pobj) return false; diff --git a/src/plugins/winsharedutils/AreoAcrylic.cpp b/src/plugins/winsharedutils/AreoAcrylic.cpp index 356c7e8a..5aa3310c 100644 --- a/src/plugins/winsharedutils/AreoAcrylic.cpp +++ b/src/plugins/winsharedutils/AreoAcrylic.cpp @@ -84,7 +84,7 @@ typedef BOOL(WINAPI *pfnSetWindowCompositionAttribute)(HWND, WINDOWCOMPOSITIONAT if (!setWindowCompositionAttribute) \ return false; -DECLARE bool setAcrylicEffect(HWND hwnd, bool isEnableShadow) +DECLARE_API bool setAcrylicEffect(HWND hwnd, bool isEnableShadow) { // win7全都用areo DWM_BLURBEHIND bb = {0}; @@ -102,7 +102,7 @@ DECLARE bool setAcrylicEffect(HWND hwnd, bool isEnableShadow) accentPolicy.AccentFlags = accentFlags; return setWindowCompositionAttribute(hwnd, &winCompAttrData); } -DECLARE bool setAeroEffect(HWND hwnd, bool isEnableShadow) +DECLARE_API bool setAeroEffect(HWND hwnd, bool isEnableShadow) { DWM_BLURBEHIND bb = {0}; bb.dwFlags = DWM_BB_ENABLE; @@ -117,7 +117,7 @@ DECLARE bool setAeroEffect(HWND hwnd, bool isEnableShadow) accentPolicy.AccentFlags = accentFlags; return setWindowCompositionAttribute(hwnd, &winCompAttrData); } -DECLARE bool clearEffect(HWND hwnd) +DECLARE_API bool clearEffect(HWND hwnd) { DWM_BLURBEHIND bb = {0}; bb.dwFlags = DWM_BB_ENABLE; diff --git a/src/plugins/winsharedutils/CMakeLists.txt b/src/plugins/winsharedutils/CMakeLists.txt index af235263..b6a2ed3e 100644 --- a/src/plugins/winsharedutils/CMakeLists.txt +++ b/src/plugins/winsharedutils/CMakeLists.txt @@ -10,10 +10,9 @@ generate_product_version( VERSION_MINOR ${VERSION_MINOR} VERSION_PATCH ${VERSION_PATCH} ) -add_library(tinymp3 ../libs/tinymp3/shine_mp3.c) add_library(winsharedutils MODULE mp3enc.cpp webview2_extra.cpp AreoAcrylic.cpp screenshot.cpp ../implsapi.cpp hwnd.cpp globalmessagelistener.cpp theme.cpp version.cpp otsu.cpp clipboard.cpp lnk.cpp dllmain.cpp levenshtein.cpp muteprocess.cpp sapi_dll.cpp simplemecab.cpp SimpleBrowser.cpp MWebBrowser.cpp icon.cpp ${versioninfo}) target_precompile_headers(winsharedutils REUSE_FROM pch) -target_link_libraries(winsharedutils tinymp3 Shcore) +target_link_libraries(winsharedutils tinymp3 Shcore rapidfuzz wil webview2) if(${CMAKE_SIZEOF_VOID_P} EQUAL 8) set_target_properties(winsharedutils PROPERTIES OUTPUT_NAME "winsharedutils64") else() diff --git a/src/plugins/winsharedutils/clipboard.cpp b/src/plugins/winsharedutils/clipboard.cpp index 84d602f6..5ed0208d 100644 --- a/src/plugins/winsharedutils/clipboard.cpp +++ b/src/plugins/winsharedutils/clipboard.cpp @@ -123,7 +123,7 @@ static void clipboard_callback_1(void (*callback)(const wchar_t *, bool), HANDLE } } -DECLARE HWND clipboard_callback(void (*callback)(const wchar_t *, bool)) +DECLARE_API HWND clipboard_callback(void (*callback)(const wchar_t *, bool)) { HANDLE hsema = CreateSemaphoreW(0, 0, 10, 0); HWND hwnd; @@ -137,7 +137,7 @@ DECLARE HWND clipboard_callback(void (*callback)(const wchar_t *, bool)) else return NULL; } -DECLARE void clipboard_callback_stop(HWND hwnd) +DECLARE_API void clipboard_callback_stop(HWND hwnd) { if (!hwnd) return; @@ -145,7 +145,7 @@ DECLARE void clipboard_callback_stop(HWND hwnd) DestroyWindow(hwnd); } -DECLARE bool clipboard_set_image(HWND hwnd, void *ptr, size_t size) +DECLARE_API bool clipboard_set_image(HWND hwnd, void *ptr, size_t size) { size -= sizeof(BITMAPFILEHEADER); HGLOBAL hDib = GlobalAlloc(GMEM_MOVEABLE, size); diff --git a/src/plugins/winsharedutils/define.h b/src/plugins/winsharedutils/define.h index 293c407f..49b98e33 100644 --- a/src/plugins/winsharedutils/define.h +++ b/src/plugins/winsharedutils/define.h @@ -1,5 +1,5 @@ #pragma once -#define DECLARE extern "C" __declspec(dllexport) +#define DECLARE_API extern "C" __declspec(dllexport) extern "C" { diff --git a/src/plugins/winsharedutils/globalmessagelistener.cpp b/src/plugins/winsharedutils/globalmessagelistener.cpp index 989d6e7a..82eaa217 100644 --- a/src/plugins/winsharedutils/globalmessagelistener.cpp +++ b/src/plugins/winsharedutils/globalmessagelistener.cpp @@ -52,12 +52,12 @@ void globalmessagelistener_1(void *callback) DispatchMessage(&msg); } } -DECLARE void globalmessagelistener(void *callback) +DECLARE_API void globalmessagelistener(void *callback) { std::thread(std::bind(globalmessagelistener_1, callback)).detach(); } -DECLARE void dispatchcloseevent() +DECLARE_API void dispatchcloseevent() { PostMessage(HWND_BROADCAST, LUNA_UPDATE_PREPARED_OK, 0, 0); } \ No newline at end of file diff --git a/src/plugins/winsharedutils/hwnd.cpp b/src/plugins/winsharedutils/hwnd.cpp index c6cd8652..cc7fe660 100644 --- a/src/plugins/winsharedutils/hwnd.cpp +++ b/src/plugins/winsharedutils/hwnd.cpp @@ -1,6 +1,6 @@ #include "define.h" #include -DECLARE void showintab(HWND hwnd, bool show, bool tool) +DECLARE_API void showintab(HWND hwnd, bool show, bool tool) { // WS_EX_TOOLWINDOW可以立即生效，WS_EX_APPWINDOW必须切换焦点才生效。但是WS_EX_TOOLWINDOW会改变窗口样式，因此只对无边框窗口使用。 LONG style = GetWindowLong(hwnd, GWL_STYLE); @@ -22,7 +22,7 @@ DECLARE void showintab(HWND hwnd, bool show, bool tool) SetWindowLong(hwnd, GWL_EXSTYLE, style_ex); } -DECLARE bool pid_running(DWORD pid) +DECLARE_API bool pid_running(DWORD pid) { DWORD code; GetExitCodeProcess(AutoHandle(OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid)), &code); @@ -54,14 +54,14 @@ BOOL CALLBACK EnumWindowsProc(HWND hwnd, LPARAM lParam) } return TRUE; } -DECLARE HWND getpidhwndfirst(DWORD pid) +DECLARE_API HWND getpidhwndfirst(DWORD pid) { __EnumWindowsProc info = {pid, 0}; EnumWindows(EnumWindowsProc, (LPARAM)&info); return info.hwnd; } -DECLARE bool Is64bit(DWORD pid) +DECLARE_API bool Is64bit(DWORD pid) { SYSTEM_INFO sysinfo; GetNativeSystemInfo(&sysinfo); @@ -78,7 +78,7 @@ DECLARE bool Is64bit(DWORD pid) return false; } -DECLARE void getprocesses(void (*cb)(DWORD, const wchar_t *)) +DECLARE_API void getprocesses(void (*cb)(DWORD, const wchar_t *)) { std::unordered_map> exe_pid; AutoHandle hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); @@ -96,7 +96,7 @@ DECLARE void getprocesses(void (*cb)(DWORD, const wchar_t *)) } while (Process32Next(hSnapshot, &pe32)); } } -DECLARE UINT GetMonitorDpiScaling(HWND hwnd) +DECLARE_API UINT GetMonitorDpiScaling(HWND hwnd) { HMONITOR hMonitor = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONEAREST); if (!hMonitor) diff --git a/src/plugins/winsharedutils/mp3enc.cpp b/src/plugins/winsharedutils/mp3enc.cpp index ea6cf385..d1e72f6e 100644 --- a/src/plugins/winsharedutils/mp3enc.cpp +++ b/src/plugins/winsharedutils/mp3enc.cpp @@ -12,7 +12,7 @@ int stereo = STEREO; -DECLARE void encodemp3(void *ptr, size_t size, void (*cb)(void *ptr, size_t size)) +DECLARE_API void encodemp3(void *ptr, size_t size, void (*cb)(void *ptr, size_t size)) { shine_config_t config; shine_t s; diff --git a/src/plugins/winsharedutils/sapi_dll.cpp b/src/plugins/winsharedutils/sapi_dll.cpp index 6c217c19..a0e4c7e5 100644 --- a/src/plugins/winsharedutils/sapi_dll.cpp +++ b/src/plugins/winsharedutils/sapi_dll.cpp @@ -42,7 +42,7 @@ namespace SAPI } }; -DECLARE bool SAPI_Speak(const wchar_t *Content, int version, int voiceid, int rate, int volume, void (*cb)(byte *, size_t)) +DECLARE_API bool SAPI_Speak(const wchar_t *Content, int version, int voiceid, int rate, int volume, void (*cb)(byte *, size_t)) { auto _c = std::wstring(Content); if (auto _ = std::move(SAPI::Speak(_c, version, voiceid, rate, volume))) diff --git a/src/plugins/winsharedutils/screenshot.cpp b/src/plugins/winsharedutils/screenshot.cpp index ab3812a4..c71bacce 100644 --- a/src/plugins/winsharedutils/screenshot.cpp +++ b/src/plugins/winsharedutils/screenshot.cpp @@ -157,7 +157,7 @@ std::vector SaveBitmapToBuffer(HBITMAP hBitmap) return data; } -DECLARE void gdi_screenshot(HWND hwnd, RECT rect, void (*cb)(byte *, size_t)) +DECLARE_API void gdi_screenshot(HWND hwnd, RECT rect, void (*cb)(byte *, size_t)) { if (rect.bottom == rect.top || rect.left == rect.right) return; @@ -176,7 +176,7 @@ DECLARE void gdi_screenshot(HWND hwnd, RECT rect, void (*cb)(byte *, size_t)) ReleaseDC(hwnd, hdc); } -DECLARE void maximum_window(HWND hwnd) +DECLARE_API void maximum_window(HWND hwnd) { RECT rect; GetVirtualDesktopRect(rect); diff --git a/src/plugins/winsharedutils/theme.cpp b/src/plugins/winsharedutils/theme.cpp index 2c06a3a6..f9e2cdf3 100644 --- a/src/plugins/winsharedutils/theme.cpp +++ b/src/plugins/winsharedutils/theme.cpp @@ -150,13 +150,13 @@ static void SetWindowTheme(HWND hWnd, bool darkBorder, bool darkMenu) noexcept RefreshImmersiveColorPolicyState(); FlushMenuThemes(); } -DECLARE void setdwmextendframe(HWND hwnd) +DECLARE_API void setdwmextendframe(HWND hwnd) { MARGINS mar{-1, -1, -1, -1}; DwmExtendFrameIntoClientArea(hwnd, &mar); } -DECLARE void _SetTheme( +DECLARE_API void _SetTheme( HWND _hWnd, bool dark, int backdrop) @@ -191,7 +191,7 @@ DECLARE void _SetTheme( DwmSetWindowAttribute(_hWnd, DWMWA_SYSTEMBACKDROP_TYPE, &value, sizeof(value)); } -DECLARE bool isDark() +DECLARE_API bool isDark() { HKEY hKey; const char *subKey = "Software\\Microsoft\\Windows\\CurrentVersion\\Themes\\Personalize"; diff --git a/src/plugins/winsharedutils/webview2_extra.cpp b/src/plugins/winsharedutils/webview2_extra.cpp index 2b67d0a6..809f8fab 100644 --- a/src/plugins/winsharedutils/webview2_extra.cpp +++ b/src/plugins/winsharedutils/webview2_extra.cpp @@ -11,7 +11,7 @@ using namespace Microsoft::WRL; if (FAILED((x))) \ return x; -DECLARE void set_transparent_background(void* m_host){ +DECLARE_API void set_transparent_background(void* m_host){ COREWEBVIEW2_COLOR color; ZeroMemory(&color,sizeof(color)); wil::com_ptr m_controller(reinterpret_cast(m_host)); @@ -22,7 +22,7 @@ DECLARE void set_transparent_background(void* m_host){ } } -DECLARE HRESULT put_PreferredColorScheme(void *m_host, COREWEBVIEW2_PREFERRED_COLOR_SCHEME scheme) +DECLARE_API HRESULT put_PreferredColorScheme(void *m_host, COREWEBVIEW2_PREFERRED_COLOR_SCHEME scheme) { wil::com_ptr m_controller(reinterpret_cast(m_host)); @@ -37,7 +37,7 @@ DECLARE HRESULT put_PreferredColorScheme(void *m_host, COREWEBVIEW2_PREFERRED_CO } return S_FALSE; } -DECLARE void *add_ZoomFactorChanged(void *m_host, void (*signal)(double)) +DECLARE_API void *add_ZoomFactorChanged(void *m_host, void (*signal)(double)) { EventRegistrationToken *m_zoomFactorChangedToken = new EventRegistrationToken; // Register a handler for the ZoomFactorChanged event. @@ -58,19 +58,19 @@ DECLARE void *add_ZoomFactorChanged(void *m_host, void (*signal)(double)) m_zoomFactorChangedToken); return m_zoomFactorChangedToken; } -DECLARE void remove_ZoomFactorChanged(void *m_host, void *m_zoomFactorChangedToken) +DECLARE_API void remove_ZoomFactorChanged(void *m_host, void *m_zoomFactorChangedToken) { reinterpret_cast(m_host)->remove_ZoomFactorChanged(*reinterpret_cast(m_zoomFactorChangedToken)); delete m_zoomFactorChangedToken; } -DECLARE double get_ZoomFactor(void *m_host) +DECLARE_API double get_ZoomFactor(void *m_host) { double zoomFactor; reinterpret_cast(m_host)->get_ZoomFactor(&zoomFactor); return zoomFactor; } -DECLARE void put_ZoomFactor(void *m_host, double zoomFactor) +DECLARE_API void put_ZoomFactor(void *m_host, double zoomFactor) { reinterpret_cast(m_host)->put_ZoomFactor(zoomFactor); } \ No newline at end of file