fix

LunaTranslator/LunaTranslator/metadata/abstract.py

@@ -4,6 +4,7 @@ from threading import Thread
 from myutils.commonbase import proxysession
 from myutils.config import globalconfig, savehook_new_data
 from traceback import print_exc
+from network.requests_common import NetWorkException
 
 
 class common:
@@ -60,8 +61,22 @@ class common:
         while True:
             pair = self.__tasks_searchfordata.get()
             gameuid, vid = pair
-            self.__do_searchfordata_1(gameuid, vid)
+            remove = True
-            self.__safe_remove_task("searchfordatatasks", pair)
+            try:
+                self.__do_searchfordata(gameuid, vid)
+                vis = f"{self.config_all['name']}: {vid} data loaded success"
+            except NetWorkException:
+                remove = False
+                vis = f"{self.config_all['name']}: {vid} network error, retry later"
+            except:
+                print_exc()
+                vis = f"{self.config_all['name']}: {vid} load failed"
+            if remove:
+
+                self.__safe_remove_task("searchfordatatasks", pair)
+            else:
+                self.__tasks_searchfordata.put((gameuid, vid))
+            gobject.baseobject.translation_ui.displayglobaltooltip.emit(vis)
 
     def __tasks_downloadimg_thread(self):
         while True:
@@ -69,16 +84,19 @@ class common:
             url, save = pair
             if os.path.exists(save):
                 self.__safe_remove_task("downloadtasks", pair)
-
                 continue
-            if self.__do_download_img(url, save):
+            try:
+                self.__do_download_img(url, save)
+            except NetWorkException:
+                remove = False
+            else:
+                print_exc()
+            if remove:
                 self.__safe_remove_task("downloadtasks", pair)
             else:
                 self.__tasks_downloadimg.put(pair)
 
     def __do_download_img(self, url, save):
-        if os.path.exists(save):
-            return True
         print(url, save)
         headers = {
             "sec-ch-ua": '"Microsoft Edge";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
@@ -87,14 +105,11 @@ class common:
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.42",
             "sec-ch-ua-platform": '"Windows"',
         }
-        try:
+
-            _content = self.proxysession.get(url, headers=headers).content
+        _content = self.proxysession.get(url, headers=headers).content
-            os.makedirs(os.path.dirname(save), exist_ok=True)
+        os.makedirs(os.path.dirname(save), exist_ok=True)
-            with open(save, "wb") as ff:
+        with open(save, "wb") as ff:
-                ff.write(_content)
+            ff.write(_content)
-            return True
-        except:
-            return False
 
     def dispatchdownloadtask(self, url):
         __routine = f"cache/metadata/{self.typename}"
@@ -111,20 +126,12 @@ class common:
         self.__tasks_downloadimg.put((url, savepath))
         return savepath
 
-    def __b64string(self, a):
+    def __b64string(self, a: str):
         return hashlib.md5(a.encode("utf8")).hexdigest()
 
-    def __safe_searchfordata(self, vid):
-        try:
-            return self.searchfordata(vid)
-        except:
-            print_exc()
-            return None
-
     def __do_searchfordata(self, gameuid, vid):
-        data = self.__safe_searchfordata(vid)
+
-        if not data:
+        data = self.searchfordata(vid)
-            return None
         title = data.get("title", None)
         namemap = data.get("namemap", None)
         developers = data.get("developers", [])
@@ -152,15 +159,8 @@ class common:
             savehook_new_data[gameuid]["developers"] = developers
         return True
 
-    def __do_searchfordata_1(self, gameuid, vid):
-
-        succ = self.__do_searchfordata(gameuid, vid)
-        if succ:
-            vis = f"{self.config_all['name']}: {vid} data loaded"
-        else:
-            vis = f"{self.config_all['name']}: {vid} load failed"
-        gobject.baseobject.translation_ui.displayglobaltooltip.emit(vis)
-
     def dispatchsearchfordata(self, gameuid, vid):
-        globalconfig["metadata"][self.typename]["searchfordatatasks"].append((gameuid, vid))
+        globalconfig["metadata"][self.typename]["searchfordatatasks"].append(
+            (gameuid, vid)
+        )
         self.__tasks_searchfordata.put((gameuid, vid))

LunaTranslator/LunaTranslator/metadata/vndb.py

@@ -1,14 +1,8 @@
 import requests, re
-from myutils.config import (
+from myutils.config import savegametaged, _TR, savehook_new_data
-    tryreadconfig,
-    safesave,
-    savegametaged,
-    _TR,
-    savehook_new_data,
-)
 from myutils.utils import initanewitem, gamdidchangedtask
-import gzip, json, functools
+import functools
-import shutil, gobject, time
+import time
 from qtsymbols import *
 from gui.inputdialog import autoinitdialog
 from metadata.abstract import common
@@ -19,18 +13,13 @@ from myutils.wrapper import Singleton_close
 
 def saferequestvndb(proxy, method, url, json=None, headers=None):
     print(method, url, json)
-    try:
+    resp = requests.request(
-        resp = requests.request(
+        method,
-            method,
+        "https://api.vndb.org/kana/" + url,
-            "https://api.vndb.org/kana/" + url,
+        headers=headers,
-            headers=headers,
+        json=json,
-            json=json,
+        proxies=proxy,
-            proxies=proxy,
+    )
-        )
-    except:
-        time.sleep(3)
-        print("retry network error")
-        return saferequestvndb(proxy, method, url, json, headers)
     if resp.status_code == 429:
         time.sleep(3)
         print("retry 429")
@@ -153,77 +142,6 @@ def getcharnamemapbyid(proxy, vid):
     return namemap
 
 
-def decompress_gzip_file(gzip_file, output_file):
-    with gzip.open(gzip_file, "rb") as f_in:
-        with open(output_file, "wb") as f_out:
-            shutil.copyfileobj(f_in, f_out)
-
-
-def safedownload(proxy):
-    try:
-        resp = requests.get(
-            "https://dl.vndb.org/dump/vndb-tags-latest.json.gz",
-            proxies=proxy,
-        )
-        jsongz = gobject.gettempdir("vndb-tags-latest.json.gz")
-        jsonfile = gobject.gettempdir("vndb-tags-latest.json")
-        with open(jsongz, "wb") as ff:
-            ff.write(resp.content)
-        decompress_gzip_file(jsongz, jsonfile)
-        with open(jsonfile, "r", encoding="utf8") as ff:
-            js = json.load(ff)
-        newjs = {}
-        for item in js:
-            gid = "g" + str(item["id"])
-            name = item["name"]
-            newjs[gid] = name
-        return newjs
-    except:
-        from traceback import print_exc
-
-        print_exc()
-        return None
-
-
-def getvntagsbyid(proxy, vid):
-
-    js = safegetvndbjson(
-        "vn",
-        {
-            "filters": [
-                "id",
-                "=",
-                vid,
-            ],
-            "fields": "tags.rating",
-        },
-    )
-    if not js:
-        return
-    res = js["results"][0]["tags"]
-    if not res:
-        return
-    tags = []
-    vndbtagdata = tryreadconfig("vndbtagdata.json")
-    changed = False
-    try:
-        for r in res:
-            tag = r["id"]
-            if tag not in vndbtagdata and not changed:
-                js = safedownload(proxy)
-                if js:
-                    vndbtagdata.update(js)
-                changed = True
-            if tag not in vndbtagdata:
-                continue
-            tags.append(vndbtagdata[r["id"]])
-    except:
-        pass
-    if changed:
-        safesave("./userconfig/vndbtagdata.json", vndbtagdata)
-    return tags
-
-
 @Singleton_close
 class vndbsettings(QDialog):
 
@@ -407,10 +325,9 @@ class searcher(common):
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.42",
             "sec-ch-ua-platform": '"Windows"',
         }
-        try:
+
-            html = self.proxysession.get(self.refmainpage(_vid), headers=headers).text
+        html = self.proxysession.get(self.refmainpage(_vid), headers=headers).text
-        except:
+
-            return []
         find = re.search('<div id="vntags">([\\s\\S]*?)</div>', html)
         if find:
             html = find.groups()[0]
@@ -424,10 +341,7 @@ class searcher(common):
 
         title = infos["title"]
         namemap = getcharnamemapbyid(self.proxy, vid)
-        vndbtags = []  # getvntagsbyid(self.proxy, vid) #这个东西谜之慢
+        vndbtags = self.gettagfromhtml(_vid)
-        if len(vndbtags) == 0:
-            # 没代理时下不动那个tag的json
-            vndbtags = self.gettagfromhtml(_vid)
         developers = infos["dev"]
 
         img = [self.dispatchdownloadtask(_) for _ in infos["imgs"]]