From 767ac206c44d8b75b4357746ba17e258143d9983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=81=8D=E5=85=AE=E6=83=9A=E5=85=AE?= <101191390+HIllya51@users.noreply.github.com> Date: Sat, 1 Jun 2024 19:38:29 +0800 Subject: [PATCH] lru --- LunaTranslator/LunaTranslator/myutils/post.py | 14 ++++++- .../LunaTranslator/myutils/utils.py | 42 ++++++++++++++++++- .../defaultconfig/postprocessconfig.json | 16 +++++++ LunaTranslator/files/lang/ar.json | 4 +- LunaTranslator/files/lang/cht.json | 4 +- LunaTranslator/files/lang/en.json | 4 +- LunaTranslator/files/lang/es.json | 4 +- LunaTranslator/files/lang/fr.json | 4 +- LunaTranslator/files/lang/it.json | 4 +- LunaTranslator/files/lang/ja.json | 4 +- LunaTranslator/files/lang/ko.json | 4 +- LunaTranslator/files/lang/pl.json | 4 +- LunaTranslator/files/lang/ru.json | 4 +- LunaTranslator/files/lang/th.json | 4 +- LunaTranslator/files/lang/tr.json | 4 +- LunaTranslator/files/lang/uk.json | 4 +- LunaTranslator/files/lang/vi.json | 4 +- LunaTranslator/files/lang/zh.json | 4 +- 18 files changed, 115 insertions(+), 17 deletions(-) diff --git a/LunaTranslator/LunaTranslator/myutils/post.py b/LunaTranslator/LunaTranslator/myutils/post.py index 7c32d55b..f6dc78ae 100644 --- a/LunaTranslator/LunaTranslator/myutils/post.py +++ b/LunaTranslator/LunaTranslator/myutils/post.py @@ -2,7 +2,7 @@ import re, codecs from traceback import print_exc from collections import Counter import importlib, gobject -from myutils.utils import getfilemd5 +from myutils.utils import getfilemd5, LRUCache from myutils.config import ( postprocessconfig, globalconfig, @@ -10,6 +10,17 @@ from myutils.config import ( getlangsrc, ) +lrucache = LRUCache(0) + + +def dedump(line, args): + size = args["cachesize"] + lrucache.setcap(size) + if lrucache.test(line): + return "" + else: + return line + def _2_f(line, args): keepnodump = args["保持非重复字符"] @@ -337,6 +348,7 @@ def POSTSOLVE(line): "_remove_control": _remove_control, "_remove_chaos": _remove_chaos, "_remove_not_in_ja_bracket": _remove_not_in_ja_bracket, + "dedump": dedump, } useranklist = globalconfig["postprocess_rank"] usedpostprocessconfig = postprocessconfig diff --git a/LunaTranslator/LunaTranslator/myutils/utils.py b/LunaTranslator/LunaTranslator/myutils/utils.py index 2442aa7d..3c0178ea 100644 --- a/LunaTranslator/LunaTranslator/myutils/utils.py +++ b/LunaTranslator/LunaTranslator/myutils/utils.py @@ -8,6 +8,7 @@ import ctypes, importlib import time import ctypes.wintypes import time +from collections import OrderedDict from qtsymbols import * from traceback import print_exc from myutils.config import ( @@ -633,10 +634,49 @@ def str2rgba(string, alpha100): alpha100 / 100, ) + def get_time_stamp(): ct = time.time() local_time = time.localtime(ct) data_head = time.strftime("%Y-%m-%d %H:%M:%S", local_time) data_secs = (ct - int(ct)) * 1000 time_stamp = "%s.%03d" % (data_head, data_secs) - return time_stamp \ No newline at end of file + return time_stamp + + +class LRUCache: + def __init__(self, capacity: int): + self.cache = {} + self.capacity = capacity + self.order = [] + + def setcap(self, cap): + if cap == -1: + cap = 9999999999 + self.capacity = cap + while len(self.cache) > self.capacity: + self.cache.popitem(last=False) + + def get(self, key: int) -> bool: + if key in self.cache: + self.order.remove(key) + self.order.append(key) + return True + return False + + def put(self, key: int) -> None: + if not self.capacity: + return + if key in self.cache: + self.order.remove(key) + elif len(self.order) == self.capacity: + old_key = self.order.pop(0) + del self.cache[old_key] + self.cache[key] = None + self.order.append(key) + + def test(self, key): + _ = self.get(key) + if not _: + self.put(key) + return _ diff --git a/LunaTranslator/files/defaultconfig/postprocessconfig.json b/LunaTranslator/files/defaultconfig/postprocessconfig.json index 52ff6342..021a6a8a 100644 --- a/LunaTranslator/files/defaultconfig/postprocessconfig.json +++ b/LunaTranslator/files/defaultconfig/postprocessconfig.json @@ -46,6 +46,22 @@ } } }, + "dedump": { + "use": true, + "name": "过滤历史重复_LRU", + "args": { + "cachesize": 3 + }, + "argstype": { + "cachesize": { + "type": "intspin", + "name": "缓存条数", + "min": -1, + "max": 10000, + "step": 1 + } + } + }, "_3": { "use": false, "name": "去除重复行_ABCDABCDABCD->ABCD", diff --git a/LunaTranslator/files/lang/ar.json b/LunaTranslator/files/lang/ar.json index 28dabdbd..8c2460c6 100644 --- a/LunaTranslator/files/lang/ar.json +++ b/LunaTranslator/files/lang/ar.json @@ -807,5 +807,7 @@ "网络请求": "طلب شبكة", "网页显示": "عرض الصفحة", "显示时间": "عرض الوقت", - "不显示时间": "لا وقت العرض" + "不显示时间": "لا وقت العرض", + "过滤历史重复": "تصفية التاريخ تكرار", + "缓存条数": "عدد شرائط التخزين المؤقت" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/cht.json b/LunaTranslator/files/lang/cht.json index 3ed9af91..f916a65b 100644 --- a/LunaTranslator/files/lang/cht.json +++ b/LunaTranslator/files/lang/cht.json @@ -807,5 +807,7 @@ "网络请求": "網絡請求", "网页显示": "網頁顯示", "显示时间": "顯示時間", - "不显示时间": "不顯示時間" + "不显示时间": "不顯示時間", + "过滤历史重复": "過濾歷史重複", + "缓存条数": "緩存條數" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/en.json b/LunaTranslator/files/lang/en.json index 35098b08..7cf1d272 100644 --- a/LunaTranslator/files/lang/en.json +++ b/LunaTranslator/files/lang/en.json @@ -807,5 +807,7 @@ "网络请求": "Network request", "网页显示": "Web page display", "显示时间": "Display time", - "不显示时间": "Do not display time" + "不显示时间": "Do not display time", + "过滤历史重复": "Filter historical duplicates", + "缓存条数": "Number of cache entries" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/es.json b/LunaTranslator/files/lang/es.json index c46c4602..c64813ce 100644 --- a/LunaTranslator/files/lang/es.json +++ b/LunaTranslator/files/lang/es.json @@ -807,5 +807,7 @@ "网络请求": "Solicitud de red", "网页显示": "Pantalla de la página web", "显示时间": "Tiempo de visualización", - "不显示时间": "No muestra el tiempo" + "不显示时间": "No muestra el tiempo", + "过滤历史重复": "Filtrar repetición histórica", + "缓存条数": "Número de barras de caché" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/fr.json b/LunaTranslator/files/lang/fr.json index 7bf30533..0c0cdc1e 100644 --- a/LunaTranslator/files/lang/fr.json +++ b/LunaTranslator/files/lang/fr.json @@ -807,5 +807,7 @@ "网络请求": "Demande de réseau", "网页显示": "Affichage de la page Web", "显示时间": "Afficher l'heure", - "不显示时间": "Ne pas afficher l'heure" + "不显示时间": "Ne pas afficher l'heure", + "过滤历史重复": "Filtrer historique répétition", + "缓存条数": "Nombre de barres de cache" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/it.json b/LunaTranslator/files/lang/it.json index 3c86bcc1..81e18e9d 100644 --- a/LunaTranslator/files/lang/it.json +++ b/LunaTranslator/files/lang/it.json @@ -807,5 +807,7 @@ "网络请求": "Richiesta di rete", "网页显示": "Visualizzazione pagina web", "显示时间": "Tempo di visualizzazione", - "不显示时间": "Non visualizzare l'ora" + "不显示时间": "Non visualizzare l'ora", + "过滤历史重复": "Filtra duplicati storici", + "缓存条数": "Numero di voci della cache" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/ja.json b/LunaTranslator/files/lang/ja.json index bc070242..e767b8c8 100644 --- a/LunaTranslator/files/lang/ja.json +++ b/LunaTranslator/files/lang/ja.json @@ -807,5 +807,7 @@ "网络请求": "ネットワーク要求", "网页显示": "Webページ表示", "显示时间": "表示時間", - "不显示时间": "時間を表示しない" + "不显示时间": "時間を表示しない", + "过滤历史重复": "フィルタ履歴の繰り返し", + "缓存条数": "キャッシュ・エントリ数" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/ko.json b/LunaTranslator/files/lang/ko.json index 3b37215b..a8b43ab1 100644 --- a/LunaTranslator/files/lang/ko.json +++ b/LunaTranslator/files/lang/ko.json @@ -807,5 +807,7 @@ "网络请求": "네트워크 요청", "网页显示": "웹 페이지 표시", "显示时间": "표시 시간", - "不显示时间": "시간 표시 안 함" + "不显示时间": "시간 표시 안 함", + "过滤历史重复": "반복된 기록 필터링", + "缓存条数": "캐시 바 수" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/pl.json b/LunaTranslator/files/lang/pl.json index 2c65e769..3a890403 100644 --- a/LunaTranslator/files/lang/pl.json +++ b/LunaTranslator/files/lang/pl.json @@ -807,5 +807,7 @@ "网络请求": "Żądanie sieci", "网页显示": "Wyświetlanie strony internetowej", "显示时间": "Czas wyświetlania", - "不显示时间": "Nie wyświetlaj czasu" + "不显示时间": "Nie wyświetlaj czasu", + "过滤历史重复": "Filtruj duplikaty historyczne", + "缓存条数": "Liczba wpisów pamięci podręcznej" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/ru.json b/LunaTranslator/files/lang/ru.json index cba04542..b47ad99c 100644 --- a/LunaTranslator/files/lang/ru.json +++ b/LunaTranslator/files/lang/ru.json @@ -807,5 +807,7 @@ "网络请求": "Сетевой запрос", "网页显示": "Показать страницу", "显示时间": "Показать время", - "不显示时间": "Не показывать время" + "不显示时间": "Не показывать время", + "过滤历史重复": "Фильтровать повторение истории", + "缓存条数": "Количество кэшированных записей" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/th.json b/LunaTranslator/files/lang/th.json index 8a486dd8..bb314959 100644 --- a/LunaTranslator/files/lang/th.json +++ b/LunaTranslator/files/lang/th.json @@ -807,5 +807,7 @@ "网络请求": "คำขอเครือข่าย", "网页显示": "แสดงหน้าเว็บ", "显示时间": "แสดงเวลา", - "不显示时间": "ไม่แสดงเวลา" + "不显示时间": "ไม่แสดงเวลา", + "过滤历史重复": "ประวัติการกรอง ทำซ้ำ", + "缓存条数": "จำนวนแถบแคช" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/tr.json b/LunaTranslator/files/lang/tr.json index b9d4797b..b360bcd3 100644 --- a/LunaTranslator/files/lang/tr.json +++ b/LunaTranslator/files/lang/tr.json @@ -807,5 +807,7 @@ "网络请求": "Network request", "网页显示": "Web sayfası gösterisi", "显示时间": "Display time", - "不显示时间": "Zamanı gösterme" + "不显示时间": "Zamanı gösterme", + "过滤历史重复": "Tarihi çizgileri sil", + "缓存条数": "Cache girişlerinin sayısı" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/uk.json b/LunaTranslator/files/lang/uk.json index 6ee58f56..6a711cd2 100644 --- a/LunaTranslator/files/lang/uk.json +++ b/LunaTranslator/files/lang/uk.json @@ -807,5 +807,7 @@ "网络请求": "Запит на мережу", "网页显示": "Показ веб- сторінок", "显示时间": "Час показу", - "不显示时间": "Не показувати час" + "不显示时间": "Не показувати час", + "过滤历史重复": "Фільтрувати історичні дублікати", + "缓存条数": "Кількість записів кешу" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/vi.json b/LunaTranslator/files/lang/vi.json index 9038c865..694f29cc 100644 --- a/LunaTranslator/files/lang/vi.json +++ b/LunaTranslator/files/lang/vi.json @@ -807,5 +807,7 @@ "网络请求": "Yêu cầu mạng", "网页显示": "Hiển thị trang web", "显示时间": "Thời gian hiển thị", - "不显示时间": "Không hiển thị thời gian" + "不显示时间": "Không hiển thị thời gian", + "过滤历史重复": "Lọc lịch sử lặp lại", + "缓存条数": "Số thanh bộ nhớ cache" } \ No newline at end of file diff --git a/LunaTranslator/files/lang/zh.json b/LunaTranslator/files/lang/zh.json index 417f24ee..22da53cd 100644 --- a/LunaTranslator/files/lang/zh.json +++ b/LunaTranslator/files/lang/zh.json @@ -807,5 +807,7 @@ "网络请求": "", "网页显示": "", "显示时间": "", - "不显示时间": "" + "不显示时间": "", + "过滤历史重复": "", + "缓存条数": "" } \ No newline at end of file