diff --git a/LunaTranslator/LunaTranslator/network/libcurl/requests.py b/LunaTranslator/LunaTranslator/network/libcurl/requests.py index f30f507b..c531fa6c 100644 --- a/LunaTranslator/LunaTranslator/network/libcurl/requests.py +++ b/LunaTranslator/LunaTranslator/network/libcurl/requests.py @@ -1,8 +1,9 @@ from libcurl import * -import winsharedutils - +import winsharedutils,windows +import threading,queue from network.requests_common import * +from traceback import print_exc class autostatus: def __init__(self,ref) -> None: self.ref=ref @@ -10,12 +11,37 @@ class autostatus: def __del__(self): self.ref._status=0 -class Response(ResponseBase): +class Response(ResponseBase): def __init__(self): super().__init__() self.last_error=0 - def iter_content(self,chunk_size=1024): - yield self.content + def iter_content_impl(self,chunk_size=1): + + downloadeddata=b'' + getnum=0 + canend=False + allbs=0 + while not(getnum==self._contentd.size and canend): + buff=self.queue.get() + + if buff is None: + canend=True + continue + allbs+=len(buff) + if chunk_size: + downloadeddata+=buff + while len(downloadeddata)>chunk_size: + yield downloadeddata[:chunk_size] + downloadeddata=downloadeddata[chunk_size:] + else: + yield buff + getnum+=1 + while len(downloadeddata): + yield downloadeddata[:chunk_size] + downloadeddata=downloadeddata[chunk_size:] + del self.hreadd + del self.hwrited + def raise_for_status(self): if self.last_error: raise CURLException(self.last_error) @@ -107,22 +133,73 @@ class Session(Sessionbase): if datalen: curl_easy_setopt(curl,CURLoption.CURLOPT_POSTFIELDS,dataptr) curl_easy_setopt(curl,CURLoption.CURLOPT_POSTFIELDSIZE,datalen) - - _content=winsharedutils.MemoryStruct() - curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEDATA,pointer(_content)) - curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEFUNCTION,winsharedutils.WriteMemoryCallback) - _headers=winsharedutils.MemoryStruct() - curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERDATA,pointer(_headers)) - curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryCallback) - - self._perform(curl) + resp=Response() - resp.content=self._getmembyte(_content) - resp.status_code=self._getStatusCode(curl) + + if stream: + resp.queue=queue.Queue() + hreadd,hwrited=windows.CreatePipe(None,1024*1024*4) + resp.hreadd=hreadd + resp.hwrited=hwrited + _contentd=winsharedutils.Pipeinfo() + _contentd.memory=hwrited + resp._contentd=_contentd + curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEDATA,pointer(resp._contentd)) + curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEFUNCTION,winsharedutils.WriteMemoryToPipe) + + hreadh,hwriteh=windows.CreatePipe(None,1024*1024*4) + _contenth=winsharedutils.Pipeinfo() + _contenth.memory=hwriteh + curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERDATA,pointer(_contenth)) + curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryToPipe) + headerqueue=queue.Queue() + headerok=threading.Lock() + headerok.acquire() + def ___perform(): + try: + self._perform(curl) + except: + print_exc() + headerqueue.put(None) + headerok.acquire() + curl_easy_reset(curl) + resp.queue.put(None) + def ___read(q,h): + while True: + size=windows.ReadFile(h,4,None) + if len(size)==0:break + data=windows.ReadFile(h,c_uint.from_buffer_copy(size).value,None) + q.put(data) + threading.Thread(target=___read,args=(resp.queue,hreadd),daemon=True).start() + threading.Thread(target=___read,args=(headerqueue,hreadh),daemon=True).start() + threading.Thread(target=___perform,daemon=True).start() + + headerb=b'' + while True: + _headerb=headerqueue.get() + if _headerb is None: + break + headerb+=_headerb + if _headerb==b'\r\n': + break + resp.headers=self._update_header_cookie(headerb.decode('utf8')) + headerok.release() + else: + _content=winsharedutils.MemoryStruct() + curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEDATA,pointer(_content)) + curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEFUNCTION,winsharedutils.WriteMemoryCallback) + _headers=winsharedutils.MemoryStruct() + curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERDATA,pointer(_headers)) + curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryCallback) + #curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,cast(WRITEFUNCTION(WRITEFUNCTIONXX),c_void_p)) + self._perform(curl) + resp.content=self._getmembyte(_content) + resp.headers=self._update_header_cookie(self._getmembyte(_headers).decode('utf8')) + resp.status_code=self._getStatusCode(curl) resp.last_error=self.last_error - resp.headers=self._update_header_cookie(self._getmembyte(_headers).decode('utf8')) resp.cookies=self.cookies - curl_easy_reset(curl) + if stream==False: + curl_easy_reset(curl) return resp Sessionimpl[0]=Session diff --git a/LunaTranslator/LunaTranslator/network/requests_common.py b/LunaTranslator/LunaTranslator/network/requests_common.py index 200ccd1f..8c559b72 100644 --- a/LunaTranslator/LunaTranslator/network/requests_common.py +++ b/LunaTranslator/LunaTranslator/network/requests_common.py @@ -69,6 +69,41 @@ class ResponseBase: return charset def json(self): return json.loads(self.text) + def iter_content(self,chunk_size=1,decode_unicode=False): + for chunk in self.iter_content_impl(chunk_size): + if decode_unicode: + yield chunk.decode('utf8') + else: + yield chunk + def iter_content_impl(self,chunk_size=1): + pass + def iter_lines( + self, chunk_size=512, decode_unicode=False, delimiter=None + ): + pending = None + size=0 + for chunk in self.iter_content( + chunk_size=chunk_size, decode_unicode=decode_unicode + ): + size+=len(chunk) + if pending is not None: + chunk = pending + chunk + + if delimiter: + lines = chunk.split(delimiter) + else: + lines = chunk.splitlines() + + if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: + pending = lines.pop() + else: + pending = None + + yield from lines + + if pending is not None: + yield pending + print(size) class Sessionbase: def __init__(self) -> None: self.UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' diff --git a/LunaTranslator/LunaTranslator/network/winhttp/requests.py b/LunaTranslator/LunaTranslator/network/winhttp/requests.py index 341b63ae..a3f5a811 100644 --- a/LunaTranslator/LunaTranslator/network/winhttp/requests.py +++ b/LunaTranslator/LunaTranslator/network/winhttp/requests.py @@ -8,11 +8,7 @@ try: except: pass class Response(ResponseBase): - def __del__(self): - del self.hreq - del self.hconn - del self.keepref - def iter_content(self,chunk_size=None): + def iter_content_impl(self,chunk_size=1): availableSize=DWORD() downloadedSize=DWORD() downloadeddata=b'' diff --git a/LunaTranslator/LunaTranslator/windows.py b/LunaTranslator/LunaTranslator/windows.py index 52e7eeec..56763c1c 100644 --- a/LunaTranslator/LunaTranslator/windows.py +++ b/LunaTranslator/LunaTranslator/windows.py @@ -521,13 +521,6 @@ def WaitNamedPipe(pipename,timeout): # def TerminateProcess(phandle,code): # return _TerminateProcess(phandle,code) -# _CreatePipe=_kernel32.CreatePipe -# _CreatePipe.argtypes=c_void_p,c_void_p,c_void_p,c_uint -# def CreatePipe(lpsecu,sz): -# hread=c_void_p() -# hwrite=c_void_p() -# _CreatePipe(pointer(hread),pointer(hwrite),lpsecu,sz) -# return hread.value,hwrite.value # _GetCurrentProcess=_kernel32.GetCurrentProcess # _DuplicateHandle=_kernel32.DuplicateHandle @@ -764,4 +757,13 @@ MAPVK_VK_TO_VSC=0 MAPVK_VSC_TO_VK=1 MAPVK_VK_TO_CHAR=2 def MapVirtualKey(char,uMapType=MAPVK_VK_TO_CHAR): - return _MapVirtualKey(ord(char),uMapType) \ No newline at end of file + return _MapVirtualKey(ord(char),uMapType) + + +_CreatePipe=_kernel32.CreatePipe +_CreatePipe.argtypes=c_void_p,c_void_p,c_void_p,c_uint +def CreatePipe(lpsecu=None,sz=0): + hread=HANDLE() + hwrite=HANDLE() + _CreatePipe(pointer(hread),pointer(hwrite),lpsecu,sz) + return AutoHandle(hread.value),AutoHandle(hwrite.value) \ No newline at end of file diff --git a/LunaTranslator/LunaTranslator/winsharedutils.py b/LunaTranslator/LunaTranslator/winsharedutils.py index 56f8b82a..69850e64 100644 --- a/LunaTranslator/LunaTranslator/winsharedutils.py +++ b/LunaTranslator/LunaTranslator/winsharedutils.py @@ -164,6 +164,7 @@ def extracticon2data(fname): return None WriteMemoryCallback=utilsdll.WriteMemoryCallback +WriteMemoryToPipe=utilsdll.WriteMemoryToPipe c_free=utilsdll.c_free c_free.argtypes=c_void_p, class MemoryStruct(Structure): @@ -177,4 +178,15 @@ class MemoryStruct(Structure): self.size=0 def __del__(self): if self.memory: - c_free(self.memory) \ No newline at end of file + c_free(self.memory) + + +class Pipeinfo(Structure): + _fields_=[ + ('memory',c_void_p), + ('size',c_size_t) + ] + def __init__(self ) : + super().__init__( ) + self.memory=0 + self.size=0 \ No newline at end of file diff --git a/plugins/winsharedutils/cinterface.cpp b/plugins/winsharedutils/cinterface.cpp index 29fbc2c9..3211b897 100644 --- a/plugins/winsharedutils/cinterface.cpp +++ b/plugins/winsharedutils/cinterface.cpp @@ -79,4 +79,15 @@ size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *user } void c_free(void* ptr){ free(ptr); -} \ No newline at end of file +} + +size_t WriteMemoryToPipe(void *contents, size_t size, size_t nmemb, void *userp){ + size_t realsize = size * nmemb; + auto mem=(MemoryStruct*)userp; + auto hWrite=(HANDLE)mem->memory; + mem->size+=1; + DWORD _; + WriteFile(hWrite,&realsize,4,&_,NULL); + WriteFile(hWrite,contents,realsize,&_,NULL); + return realsize; +} diff --git a/plugins/winsharedutils/define.h b/plugins/winsharedutils/define.h index a6a5e21a..b17da529 100644 --- a/plugins/winsharedutils/define.h +++ b/plugins/winsharedutils/define.h @@ -32,5 +32,6 @@ extern "C" { __declspec(dllexport) size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp); + __declspec(dllexport) size_t WriteMemoryToPipe(void *contents, size_t size, size_t nmemb, void *userp); __declspec(dllexport) void c_free(void*); } \ No newline at end of file