From c9f88401e0f37af66fb117cc6726623d1ca8a6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=81=8D=E5=85=AE=E6=83=9A=E5=85=AE?= <101191390+HIllya51@users.noreply.github.com> Date: Thu, 27 Jun 2024 22:11:30 +0800 Subject: [PATCH] fix --- .../LunaTranslator/metadata/abstract.py | 70 +++++------ .../LunaTranslator/metadata/vndb.py | 114 +++--------------- 2 files changed, 49 insertions(+), 135 deletions(-) diff --git a/LunaTranslator/LunaTranslator/metadata/abstract.py b/LunaTranslator/LunaTranslator/metadata/abstract.py index 2bfa736f..a79fde11 100644 --- a/LunaTranslator/LunaTranslator/metadata/abstract.py +++ b/LunaTranslator/LunaTranslator/metadata/abstract.py @@ -4,6 +4,7 @@ from threading import Thread from myutils.commonbase import proxysession from myutils.config import globalconfig, savehook_new_data from traceback import print_exc +from network.requests_common import NetWorkException class common: @@ -60,8 +61,22 @@ class common: while True: pair = self.__tasks_searchfordata.get() gameuid, vid = pair - self.__do_searchfordata_1(gameuid, vid) - self.__safe_remove_task("searchfordatatasks", pair) + remove = True + try: + self.__do_searchfordata(gameuid, vid) + vis = f"{self.config_all['name']}: {vid} data loaded success" + except NetWorkException: + remove = False + vis = f"{self.config_all['name']}: {vid} network error, retry later" + except: + print_exc() + vis = f"{self.config_all['name']}: {vid} load failed" + if remove: + + self.__safe_remove_task("searchfordatatasks", pair) + else: + self.__tasks_searchfordata.put((gameuid, vid)) + gobject.baseobject.translation_ui.displayglobaltooltip.emit(vis) def __tasks_downloadimg_thread(self): while True: @@ -69,16 +84,19 @@ class common: url, save = pair if os.path.exists(save): self.__safe_remove_task("downloadtasks", pair) - continue - if self.__do_download_img(url, save): + try: + self.__do_download_img(url, save) + except NetWorkException: + remove = False + else: + print_exc() + if remove: self.__safe_remove_task("downloadtasks", pair) else: self.__tasks_downloadimg.put(pair) def __do_download_img(self, url, save): - if os.path.exists(save): - return True print(url, save) headers = { "sec-ch-ua": '"Microsoft Edge";v="113", "Chromium";v="113", "Not-A.Brand";v="24"', @@ -87,14 +105,11 @@ class common: "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.42", "sec-ch-ua-platform": '"Windows"', } - try: - _content = self.proxysession.get(url, headers=headers).content - os.makedirs(os.path.dirname(save), exist_ok=True) - with open(save, "wb") as ff: - ff.write(_content) - return True - except: - return False + + _content = self.proxysession.get(url, headers=headers).content + os.makedirs(os.path.dirname(save), exist_ok=True) + with open(save, "wb") as ff: + ff.write(_content) def dispatchdownloadtask(self, url): __routine = f"cache/metadata/{self.typename}" @@ -111,20 +126,12 @@ class common: self.__tasks_downloadimg.put((url, savepath)) return savepath - def __b64string(self, a): + def __b64string(self, a: str): return hashlib.md5(a.encode("utf8")).hexdigest() - def __safe_searchfordata(self, vid): - try: - return self.searchfordata(vid) - except: - print_exc() - return None - def __do_searchfordata(self, gameuid, vid): - data = self.__safe_searchfordata(vid) - if not data: - return None + + data = self.searchfordata(vid) title = data.get("title", None) namemap = data.get("namemap", None) developers = data.get("developers", []) @@ -152,15 +159,8 @@ class common: savehook_new_data[gameuid]["developers"] = developers return True - def __do_searchfordata_1(self, gameuid, vid): - - succ = self.__do_searchfordata(gameuid, vid) - if succ: - vis = f"{self.config_all['name']}: {vid} data loaded" - else: - vis = f"{self.config_all['name']}: {vid} load failed" - gobject.baseobject.translation_ui.displayglobaltooltip.emit(vis) - def dispatchsearchfordata(self, gameuid, vid): - globalconfig["metadata"][self.typename]["searchfordatatasks"].append((gameuid, vid)) + globalconfig["metadata"][self.typename]["searchfordatatasks"].append( + (gameuid, vid) + ) self.__tasks_searchfordata.put((gameuid, vid)) diff --git a/LunaTranslator/LunaTranslator/metadata/vndb.py b/LunaTranslator/LunaTranslator/metadata/vndb.py index 53516e86..18371bf7 100644 --- a/LunaTranslator/LunaTranslator/metadata/vndb.py +++ b/LunaTranslator/LunaTranslator/metadata/vndb.py @@ -1,14 +1,8 @@ import requests, re -from myutils.config import ( - tryreadconfig, - safesave, - savegametaged, - _TR, - savehook_new_data, -) +from myutils.config import savegametaged, _TR, savehook_new_data from myutils.utils import initanewitem, gamdidchangedtask -import gzip, json, functools -import shutil, gobject, time +import functools +import time from qtsymbols import * from gui.inputdialog import autoinitdialog from metadata.abstract import common @@ -19,18 +13,13 @@ from myutils.wrapper import Singleton_close def saferequestvndb(proxy, method, url, json=None, headers=None): print(method, url, json) - try: - resp = requests.request( - method, - "https://api.vndb.org/kana/" + url, - headers=headers, - json=json, - proxies=proxy, - ) - except: - time.sleep(3) - print("retry network error") - return saferequestvndb(proxy, method, url, json, headers) + resp = requests.request( + method, + "https://api.vndb.org/kana/" + url, + headers=headers, + json=json, + proxies=proxy, + ) if resp.status_code == 429: time.sleep(3) print("retry 429") @@ -153,77 +142,6 @@ def getcharnamemapbyid(proxy, vid): return namemap -def decompress_gzip_file(gzip_file, output_file): - with gzip.open(gzip_file, "rb") as f_in: - with open(output_file, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) - - -def safedownload(proxy): - try: - resp = requests.get( - "https://dl.vndb.org/dump/vndb-tags-latest.json.gz", - proxies=proxy, - ) - jsongz = gobject.gettempdir("vndb-tags-latest.json.gz") - jsonfile = gobject.gettempdir("vndb-tags-latest.json") - with open(jsongz, "wb") as ff: - ff.write(resp.content) - decompress_gzip_file(jsongz, jsonfile) - with open(jsonfile, "r", encoding="utf8") as ff: - js = json.load(ff) - newjs = {} - for item in js: - gid = "g" + str(item["id"]) - name = item["name"] - newjs[gid] = name - return newjs - except: - from traceback import print_exc - - print_exc() - return None - - -def getvntagsbyid(proxy, vid): - - js = safegetvndbjson( - "vn", - { - "filters": [ - "id", - "=", - vid, - ], - "fields": "tags.rating", - }, - ) - if not js: - return - res = js["results"][0]["tags"] - if not res: - return - tags = [] - vndbtagdata = tryreadconfig("vndbtagdata.json") - changed = False - try: - for r in res: - tag = r["id"] - if tag not in vndbtagdata and not changed: - js = safedownload(proxy) - if js: - vndbtagdata.update(js) - changed = True - if tag not in vndbtagdata: - continue - tags.append(vndbtagdata[r["id"]]) - except: - pass - if changed: - safesave("./userconfig/vndbtagdata.json", vndbtagdata) - return tags - - @Singleton_close class vndbsettings(QDialog): @@ -407,10 +325,9 @@ class searcher(common): "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.42", "sec-ch-ua-platform": '"Windows"', } - try: - html = self.proxysession.get(self.refmainpage(_vid), headers=headers).text - except: - return [] + + html = self.proxysession.get(self.refmainpage(_vid), headers=headers).text + find = re.search('
([\\s\\S]*?)
', html) if find: html = find.groups()[0] @@ -424,10 +341,7 @@ class searcher(common): title = infos["title"] namemap = getcharnamemapbyid(self.proxy, vid) - vndbtags = [] # getvntagsbyid(self.proxy, vid) #这个东西谜之慢 - if len(vndbtags) == 0: - # 没代理时下不动那个tag的json - vndbtags = self.gettagfromhtml(_vid) + vndbtags = self.gettagfromhtml(_vid) developers = infos["dev"] img = [self.dispatchdownloadtask(_) for _ in infos["imgs"]]