Update requests.py

LunaTranslator/LunaTranslator/network/libcurl/requests.py

@@ -1,8 +1,9 @@
  
 from libcurl import *
-import winsharedutils 
+import winsharedutils,windows
-
+import threading,queue
 from network.requests_common import *
+from traceback import print_exc
 class autostatus:
     def __init__(self,ref) -> None:
         self.ref=ref
@@ -10,12 +11,37 @@ class autostatus:
         
     def __del__(self):
         self.ref._status=0
-class Response(ResponseBase):  
+class Response(ResponseBase):
     def __init__(self):
         super().__init__()
         self.last_error=0
-    def iter_content(self,chunk_size=1024):
+    def iter_content_impl(self,chunk_size=1):
-        yield self.content
+        
+        downloadeddata=b''
+        getnum=0
+        canend=False
+        allbs=0
+        while not(getnum==self._contentd.size and canend):
+            buff=self.queue.get()
+            
+            if buff is None:
+                canend=True
+                continue
+            allbs+=len(buff)
+            if chunk_size:
+                downloadeddata+=buff
+                while len(downloadeddata)>chunk_size:
+                    yield downloadeddata[:chunk_size]
+                    downloadeddata=downloadeddata[chunk_size:]
+            else:
+                yield buff
+            getnum+=1
+        while len(downloadeddata):
+            yield downloadeddata[:chunk_size]
+            downloadeddata=downloadeddata[chunk_size:]
+        del self.hreadd
+        del self.hwrited
+        
     def raise_for_status(self): 
         if self.last_error:
             raise CURLException(self.last_error) 
@@ -107,22 +133,73 @@ class Session(Sessionbase):
         if datalen:
             curl_easy_setopt(curl,CURLoption.CURLOPT_POSTFIELDS,dataptr)
             curl_easy_setopt(curl,CURLoption.CURLOPT_POSTFIELDSIZE,datalen)
-
+        
-        _content=winsharedutils.MemoryStruct() 
-        curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEDATA,pointer(_content))
-        curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEFUNCTION,winsharedutils.WriteMemoryCallback)
-        _headers=winsharedutils.MemoryStruct() 
-        curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERDATA,pointer(_headers))
-        curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryCallback)  
-
-        self._perform(curl)
         resp=Response()
-        resp.content=self._getmembyte(_content)
+
-        resp.status_code=self._getStatusCode(curl) 
+        if stream:
+            resp.queue=queue.Queue()
+            hreadd,hwrited=windows.CreatePipe(None,1024*1024*4)
+            resp.hreadd=hreadd
+            resp.hwrited=hwrited
+            _contentd=winsharedutils.Pipeinfo()
+            _contentd.memory=hwrited
+            resp._contentd=_contentd
+            curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEDATA,pointer(resp._contentd))
+            curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEFUNCTION,winsharedutils.WriteMemoryToPipe)
+
+            hreadh,hwriteh=windows.CreatePipe(None,1024*1024*4)
+            _contenth=winsharedutils.Pipeinfo() 
+            _contenth.memory=hwriteh
+            curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERDATA,pointer(_contenth))
+            curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryToPipe)
+            headerqueue=queue.Queue()
+            headerok=threading.Lock()
+            headerok.acquire()
+            def ___perform():
+                try:
+                    self._perform(curl)
+                except:
+                    print_exc()
+                    headerqueue.put(None)
+                headerok.acquire()
+                curl_easy_reset(curl)
+                resp.queue.put(None)
+            def ___read(q,h):
+                while True:
+                    size=windows.ReadFile(h,4,None)
+                    if len(size)==0:break
+                    data=windows.ReadFile(h,c_uint.from_buffer_copy(size).value,None)
+                    q.put(data)
+            threading.Thread(target=___read,args=(resp.queue,hreadd),daemon=True).start()
+            threading.Thread(target=___read,args=(headerqueue,hreadh),daemon=True).start()
+            threading.Thread(target=___perform,daemon=True).start()
+            
+            headerb=b''
+            while True:
+                _headerb=headerqueue.get()
+                if _headerb is None:
+                    break
+                headerb+=_headerb
+                if _headerb==b'\r\n':
+                    break
+            resp.headers=self._update_header_cookie(headerb.decode('utf8'))
+            headerok.release()
+        else:
+            _content=winsharedutils.MemoryStruct() 
+            curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEDATA,pointer(_content))
+            curl_easy_setopt(curl,CURLoption.CURLOPT_WRITEFUNCTION,winsharedutils.WriteMemoryCallback)
+            _headers=winsharedutils.MemoryStruct() 
+            curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERDATA,pointer(_headers))
+            curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,winsharedutils.WriteMemoryCallback)
+            #curl_easy_setopt(curl,CURLoption.CURLOPT_HEADERFUNCTION,cast(WRITEFUNCTION(WRITEFUNCTIONXX),c_void_p))
+            self._perform(curl)
+            resp.content=self._getmembyte(_content)
+            resp.headers=self._update_header_cookie(self._getmembyte(_headers).decode('utf8'))
+        resp.status_code=self._getStatusCode(curl)
         resp.last_error=self.last_error
-        resp.headers=self._update_header_cookie(self._getmembyte(_headers).decode('utf8'))
         resp.cookies=self.cookies
-        curl_easy_reset(curl)
+        if stream==False:
+            curl_easy_reset(curl)
         return resp
     
 Sessionimpl[0]=Session

LunaTranslator/LunaTranslator/network/requests_common.py

@@ -69,6 +69,41 @@ class ResponseBase:
         return charset
     def json(self):
         return json.loads(self.text)
+    def iter_content(self,chunk_size=1,decode_unicode=False):
+        for chunk in self.iter_content_impl(chunk_size):
+            if decode_unicode:
+                yield chunk.decode('utf8')
+            else:
+                yield chunk
+    def iter_content_impl(self,chunk_size=1):
+        pass
+    def iter_lines(
+        self, chunk_size=512, decode_unicode=False, delimiter=None
+    ):
+        pending = None
+        size=0
+        for chunk in self.iter_content(
+            chunk_size=chunk_size, decode_unicode=decode_unicode
+        ):
+            size+=len(chunk)
+            if pending is not None:
+                chunk = pending + chunk
+
+            if delimiter:
+                lines = chunk.split(delimiter)
+            else:
+                lines = chunk.splitlines()
+
+            if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
+                pending = lines.pop()
+            else:
+                pending = None
+
+            yield from lines
+
+        if pending is not None:
+            yield pending
+        print(size)
 class Sessionbase:
     def __init__(self) -> None:
         self.UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'

LunaTranslator/LunaTranslator/network/winhttp/requests.py

@@ -8,11 +8,7 @@ try:
 except:
     pass
 class Response(ResponseBase):
-    def __del__(self):
+    def iter_content_impl(self,chunk_size=1):
-        del self.hreq
-        del self.hconn
-        del self.keepref
-    def iter_content(self,chunk_size=None):
         availableSize=DWORD()
         downloadedSize=DWORD()
         downloadeddata=b''

LunaTranslator/LunaTranslator/windows.py

@@ -521,13 +521,6 @@ def WaitNamedPipe(pipename,timeout):
 # def TerminateProcess(phandle,code):
 #     return _TerminateProcess(phandle,code)
  
-# _CreatePipe=_kernel32.CreatePipe 
-# _CreatePipe.argtypes=c_void_p,c_void_p,c_void_p,c_uint
-# def CreatePipe(lpsecu,sz):
-#     hread=c_void_p()
-#     hwrite=c_void_p()
-#     _CreatePipe(pointer(hread),pointer(hwrite),lpsecu,sz)
-#     return hread.value,hwrite.value
 
 # _GetCurrentProcess=_kernel32.GetCurrentProcess
 # _DuplicateHandle=_kernel32.DuplicateHandle
@@ -764,4 +757,13 @@ MAPVK_VK_TO_VSC=0
 MAPVK_VSC_TO_VK=1
 MAPVK_VK_TO_CHAR=2
 def MapVirtualKey(char,uMapType=MAPVK_VK_TO_CHAR):
-    return _MapVirtualKey(ord(char),uMapType)
+    return _MapVirtualKey(ord(char),uMapType)
+
+
+_CreatePipe=_kernel32.CreatePipe 
+_CreatePipe.argtypes=c_void_p,c_void_p,c_void_p,c_uint
+def CreatePipe(lpsecu=None,sz=0):
+    hread=HANDLE()
+    hwrite=HANDLE()
+    _CreatePipe(pointer(hread),pointer(hwrite),lpsecu,sz)
+    return AutoHandle(hread.value),AutoHandle(hwrite.value)

LunaTranslator/LunaTranslator/winsharedutils.py

@@ -164,6 +164,7 @@ def extracticon2data(fname):
         return None
     
 WriteMemoryCallback=utilsdll.WriteMemoryCallback
+WriteMemoryToPipe=utilsdll.WriteMemoryToPipe
 c_free=utilsdll.c_free
 c_free.argtypes=c_void_p,
 class MemoryStruct(Structure):
@@ -177,4 +178,15 @@ class MemoryStruct(Structure):
         self.size=0
     def __del__(self):
         if self.memory:
-            c_free(self.memory)
+            c_free(self.memory)
+
+
+class Pipeinfo(Structure):
+    _fields_=[
+        ('memory',c_void_p),
+        ('size',c_size_t)
+    ]
+    def __init__(self )  :
+        super().__init__( )
+        self.memory=0
+        self.size=0

plugins/winsharedutils/cinterface.cpp

@@ -79,4 +79,15 @@ size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *user
 }
 void c_free(void* ptr){
     free(ptr);
-}
+}
+
+size_t WriteMemoryToPipe(void *contents, size_t size, size_t nmemb, void *userp){
+  size_t realsize = size * nmemb;
+  auto mem=(MemoryStruct*)userp;
+  auto hWrite=(HANDLE)mem->memory;
+  mem->size+=1;
+  DWORD _;
+  WriteFile(hWrite,&realsize,4,&_,NULL);
+  WriteFile(hWrite,contents,realsize,&_,NULL);
+  return realsize;
+}

plugins/winsharedutils/define.h

@@ -32,5 +32,6 @@ extern "C" {
 
     
     __declspec(dllexport) size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp);
+    __declspec(dllexport) size_t WriteMemoryToPipe(void *contents, size_t size, size_t nmemb, void *userp);
     __declspec(dllexport) void c_free(void*);
 }