From c990426f639fd20d3db461516f95aedbcd9b6c3a Mon Sep 17 00:00:00 2001 From: HIllya51 <1871535768@qq.com> Date: Wed, 10 Jan 2024 20:32:14 +0800 Subject: [PATCH] fix --- .../LunaTranslator/network/libcurl/libcurl.py | 1 + .../network/libcurl/requests.py | 9 +-- .../LunaTranslator/network/requests_common.py | 57 ++++++++++--------- .../network/winhttp/brotli_dec.py | 19 +++++++ .../network/winhttp/requests.py | 23 +++++++- LunaTranslator/pack.py | 8 ++- 6 files changed, 81 insertions(+), 36 deletions(-) create mode 100644 LunaTranslator/LunaTranslator/network/winhttp/brotli_dec.py diff --git a/LunaTranslator/LunaTranslator/network/libcurl/libcurl.py b/LunaTranslator/LunaTranslator/network/libcurl/libcurl.py index 6a4041e6..d2c597de 100644 --- a/LunaTranslator/LunaTranslator/network/libcurl/libcurl.py +++ b/LunaTranslator/LunaTranslator/network/libcurl/libcurl.py @@ -138,6 +138,7 @@ class CURLoption(c_int): CURLOPT_COOKIEJAR=CURLOPTTYPE_STRINGPOINT+82 CURLOPT_COOKIESESSION=CURLOPTTYPE_LONG+96 CURLOPT_SHARE=CURLOPTTYPE_OBJECTPOINT+100 + CURLOPT_ACCEPT_ENCODING=CURLOPTTYPE_STRINGPOINT+102 CURLOPT_CONNECT_ONLY=CURLOPTTYPE_LONG+141 class CURLINFO(c_int): CURLINFO_STRING =0x100000 diff --git a/LunaTranslator/LunaTranslator/network/libcurl/requests.py b/LunaTranslator/LunaTranslator/network/libcurl/requests.py index a3cf71dd..12b10922 100644 --- a/LunaTranslator/LunaTranslator/network/libcurl/requests.py +++ b/LunaTranslator/LunaTranslator/network/libcurl/requests.py @@ -50,8 +50,7 @@ class Session(Sessionbase): return cast(mem.memory,POINTER(c_char))[:mem.size] def request_impl(self, method,scheme,server,port,param,url,headers,cookies,dataptr,datalen,proxy,stream,verify): - - headers=self._parseheader(headers,None)#curl对于headers中有cookie且session中也有cookie的情况下，不会自动合并而是会有两个cookie键，所以不能在header里放cookie + if self._status==0: curl=self.curl __=autostatus(self) @@ -65,6 +64,8 @@ class Session(Sessionbase): if cookies: cookie=self._parsecookie(cookies) curl_easy_setopt(curl, CURLoption.CURLOPT_COOKIE, cookie.encode('utf8')); + curl_easy_setopt(curl,CURLoption.CURLOPT_ACCEPT_ENCODING, headers['Accept-Encoding'].encode('utf8')) + curl_easy_setopt(curl,CURLoption.CURLOPT_CUSTOMREQUEST,method.upper().encode('utf8')) self.last_error=curl_easy_setopt(curl,CURLoption.CURLOPT_URL,url.encode('utf8')) @@ -72,7 +73,7 @@ class Session(Sessionbase): curl_easy_setopt(curl, CURLoption.CURLOPT_PORT, port ) lheaders=Autoslist() - for _ in headers: + for _ in self._parseheader(headers,None): lheaders = curl_slist_append(cast(lheaders,POINTER(curl_slist)), _.encode('utf8')); self.last_error=curl_easy_setopt(curl, CURLoption.CURLOPT_HTTPHEADER, lheaders); self.raise_for_status() @@ -91,7 +92,7 @@ class Session(Sessionbase): curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryCallback) self._perform(curl) - self.content=self._getmembyte(_content) + self.rawdata=self._getmembyte(_content) self._update_header_cookie(self._getmembyte(_headers).decode('utf8')) diff --git a/LunaTranslator/LunaTranslator/network/requests_common.py b/LunaTranslator/LunaTranslator/network/requests_common.py index 55a02476..3b1e1c4b 100644 --- a/LunaTranslator/LunaTranslator/network/requests_common.py +++ b/LunaTranslator/LunaTranslator/network/requests_common.py @@ -1,8 +1,8 @@ -import json,gzip,base64 +import json,base64,re from collections.abc import Callable, Mapping, MutableMapping from collections import OrderedDict from urllib.parse import urlencode,urlsplit - +from functools import partial,partialmethod class CaseInsensitiveDict(MutableMapping): def __init__(self, data=None, **kwargs): @@ -52,11 +52,11 @@ class Sessionbase: self.UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' self.last_error=0 self.status_code=0 - self.content=b'{}' + self.rawdata=b'{}' self.cookies={} self.dfheaders=CaseInsensitiveDict({ "User-Agent": self.UA, - "Accept-Encoding": 'gzip, deflate',#br + "Accept-Encoding": 'gzip, deflate, br', "Accept": "*/*", "Connection": "keep-alive", }) @@ -166,33 +166,34 @@ class Sessionbase: else: header[line[:idx]]=line[idx+2:] return CaseInsensitiveDict(header),cookie - + def decompress_impl(self,data,encode): + return data @property - def text(self): - encode=self.headers.get('Content-Encoding',None) - try: - if encode =='gzip': - self.content=gzip.decompress(self.content) - # elif encode=='br': - # self.content=brotli.decompress(self.content) - return self.content.decode('utf8') + def content(self): + return self.decompress_impl(self.rawdata,self.headers.get('Content-Encoding',None)) + @property + def text(self): + try: + return self.content.decode(self.charset) except: - raise Exception('unenable to decode {}'.format(encode)) + raise Exception('unenable to decode with {}'.format(self.charset)) + @property + def charset(self): + content_type=self.headers.get('Content-Type','') + m = re.search(r"charset=([\w-]+)", content_type) + charset = m.group(1) if m else "utf-8" + return charset def json(self): return json.loads(self.text) - def get(self, url, **kwargs): - return self.request("GET", url, **kwargs) - def post(self, url, **kwargs): - return self.request("POST", url, **kwargs) - def options(self, url, **kwargs): - return self.request("OPTIONS", url, **kwargs) + def request_impl(self,*args): pass def request(self, method, url, params=None, data=None, headers=None,proxies=None, json=None,cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, hooks=None, stream=None, verify=False, cert=None, ): - headers=CaseInsensitiveDict(headers) if headers else self.dfheaders + headers=CaseInsensitiveDict(headers if headers else {}) + headers.update(self.dfheaders) if auth and isinstance(auth,tuple) and len(auth)==2: headers['Authorization']="Basic " + ( base64.b64encode(b":".join((auth[0].encode("latin1"), auth[1].encode("latin1")))).strip() ).decode() @@ -204,16 +205,16 @@ class Sessionbase: _= self.request_impl(method,scheme,server,port,param,url,headers,cookies,dataptr,datalen,proxy,stream,verify) return _ + get=partialmethod(request,"GET") + post=partialmethod(request,"POST") + options=partialmethod(request,"OPTIONS") Sessionimpl=[Sessionbase] def request(method, url, **kwargs): with Sessionimpl[0]() as session: return session.request(method=method, url=url, **kwargs) -def get(url, params=None, **kwargs): - return request("GET", url, params=params, **kwargs) -def post(url, params=None, **kwargs): - return request("POST", url, params=params, **kwargs) -def options(url, params=None, **kwargs): - return request("OPTIONS", url, params=params, **kwargs) def session(): with Sessionimpl[0]() as session: - return session \ No newline at end of file + return session +get=partial(request,"GET") +post=partial(request,"POST") +options=partial(request,"OPTIONS") \ No newline at end of file diff --git a/LunaTranslator/LunaTranslator/network/winhttp/brotli_dec.py b/LunaTranslator/LunaTranslator/network/winhttp/brotli_dec.py new file mode 100644 index 00000000..59817272 --- /dev/null +++ b/LunaTranslator/LunaTranslator/network/winhttp/brotli_dec.py @@ -0,0 +1,19 @@ + +from ctypes import CDLL,c_size_t,c_void_p,POINTER,pointer,create_string_buffer +import os,platform + +dllpath=os.path.abspath(os.path.join('./files/plugins/brotli',['./x64/brotlicommon.dll','./x86/brotlicommon.dll'][platform.architecture()[0]=='32bit'])) +_brotli=CDLL(dllpath) +dllpath=os.path.abspath(os.path.join('./files/plugins/brotli',['./x64/brotlidec.dll','./x86/brotlidec.dll'][platform.architecture()[0]=='32bit'])) +_brotli=CDLL(dllpath) +BrotliDecoderDecompress=_brotli.BrotliDecoderDecompress +BrotliDecoderDecompress.argtypes=c_size_t,c_void_p,POINTER(c_size_t),c_void_p +def decompress(data): + size=c_size_t(1024) + while 1: + buff=create_string_buffer(size.value) + succ=BrotliDecoderDecompress(len(data),data,pointer(size),buff) + if succ==0: + size=c_size_t(size.value*2) + else:break + return buff.raw[:size.value] \ No newline at end of file diff --git a/LunaTranslator/LunaTranslator/network/winhttp/requests.py b/LunaTranslator/LunaTranslator/network/winhttp/requests.py index c28033af..b0a17419 100644 --- a/LunaTranslator/LunaTranslator/network/winhttp/requests.py +++ b/LunaTranslator/LunaTranslator/network/winhttp/requests.py @@ -1,6 +1,12 @@ from winhttp import * from network.requests_common import * +import gzip,zlib +from ctypes import pointer,create_string_buffer +try: + from brotli_dec import decompress +except: + pass class Session(Sessionbase): def __init__(self) -> None: super().__init__() @@ -83,7 +89,7 @@ class Session(Sessionbase): succ=WinHttpReadData(hRequest,buff,availableSize,pointer(downloadedSize)) if succ==0:raise WinhttpException(GetLastError()) downloadeddata+=buff[:downloadedSize.value] - self.content=downloadeddata + self.rawdata=downloadeddata #print(self.text) return self @@ -99,7 +105,20 @@ class Session(Sessionbase): del self.hconn break yield buff[:downloadedSize.value] - + def decompress_impl(self,data,encode): + #WINHTTP_OPTION_DECOMPRESSION + #支持gzip和deflate，WINHTTP_DECOMPRESSION_FLAG_GZIP|WINHTTP_DECOMPRESSION_FLAG_DEFLATE + #但只支持win8.1+,不支持br + try: + if encode =='gzip': + data=gzip.decompress(data) + elif encode =='deflate': + data=zlib.decompress(data, -zlib.MAX_WBITS) + elif encode=='br': + data=decompress(data) + return data + except: + raise Exception('unenable to decompress {}'.format(encode)) Sessionimpl[0]=Session if __name__=='__main__': pass diff --git a/LunaTranslator/pack.py b/LunaTranslator/pack.py index 61928957..76ad45b7 100644 --- a/LunaTranslator/pack.py +++ b/LunaTranslator/pack.py @@ -9,7 +9,9 @@ if x86: badcdlls= [ 'libcurl-x64.dll','libmecab64.dll', 'ocr64.dll', - 'winsharedutils64.dll','winrtutils64.dll' + 'winsharedutils64.dll','winrtutils64.dll', + r'brotli\x64\brotlicommon.dll', + r'brotli\x64\brotlidec.dll', ] downlevel=f'C:\Windows\SysWOW64\downlevel' target='LunaTranslator_x86.zip' @@ -21,7 +23,9 @@ else: badcdlls= [ 'libcurl.dll','libmecab32.dll', 'ocr32.dll', - 'winsharedutils32.dll','winrtutils32.dll' + 'winsharedutils32.dll','winrtutils32.dll', + r'brotli\x86\brotlicommon.dll', + r'brotli\x86\brotlidec.dll', ] downlevel=f'C:\Windows\system32\downlevel' targetdir_in=rf'{targetdir}\LunaTranslator'