This commit is contained in:
恍兮惚兮 2024-06-01 19:38:29 +08:00
parent 4b37f8194f
commit 767ac206c4
18 changed files with 115 additions and 17 deletions

View File

@ -2,7 +2,7 @@ import re, codecs
from traceback import print_exc
from collections import Counter
import importlib, gobject
from myutils.utils import getfilemd5
from myutils.utils import getfilemd5, LRUCache
from myutils.config import (
postprocessconfig,
globalconfig,
@ -10,6 +10,17 @@ from myutils.config import (
getlangsrc,
)
lrucache = LRUCache(0)
def dedump(line, args):
size = args["cachesize"]
lrucache.setcap(size)
if lrucache.test(line):
return ""
else:
return line
def _2_f(line, args):
keepnodump = args["保持非重复字符"]
@ -337,6 +348,7 @@ def POSTSOLVE(line):
"_remove_control": _remove_control,
"_remove_chaos": _remove_chaos,
"_remove_not_in_ja_bracket": _remove_not_in_ja_bracket,
"dedump": dedump,
}
useranklist = globalconfig["postprocess_rank"]
usedpostprocessconfig = postprocessconfig

View File

@ -8,6 +8,7 @@ import ctypes, importlib
import time
import ctypes.wintypes
import time
from collections import OrderedDict
from qtsymbols import *
from traceback import print_exc
from myutils.config import (
@ -633,10 +634,49 @@ def str2rgba(string, alpha100):
alpha100 / 100,
)
def get_time_stamp():
ct = time.time()
local_time = time.localtime(ct)
data_head = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
data_secs = (ct - int(ct)) * 1000
time_stamp = "%s.%03d" % (data_head, data_secs)
return time_stamp
return time_stamp
class LRUCache:
def __init__(self, capacity: int):
self.cache = {}
self.capacity = capacity
self.order = []
def setcap(self, cap):
if cap == -1:
cap = 9999999999
self.capacity = cap
while len(self.cache) > self.capacity:
self.cache.popitem(last=False)
def get(self, key: int) -> bool:
if key in self.cache:
self.order.remove(key)
self.order.append(key)
return True
return False
def put(self, key: int) -> None:
if not self.capacity:
return
if key in self.cache:
self.order.remove(key)
elif len(self.order) == self.capacity:
old_key = self.order.pop(0)
del self.cache[old_key]
self.cache[key] = None
self.order.append(key)
def test(self, key):
_ = self.get(key)
if not _:
self.put(key)
return _

View File

@ -46,6 +46,22 @@
}
}
},
"dedump": {
"use": true,
"name": "过滤历史重复_LRU",
"args": {
"cachesize": 3
},
"argstype": {
"cachesize": {
"type": "intspin",
"name": "缓存条数",
"min": -1,
"max": 10000,
"step": 1
}
}
},
"_3": {
"use": false,
"name": "去除重复行_ABCDABCDABCD->ABCD",

View File

@ -807,5 +807,7 @@
"网络请求": "طلب شبكة",
"网页显示": "عرض الصفحة",
"显示时间": "عرض الوقت",
"不显示时间": "لا وقت العرض"
"不显示时间": "لا وقت العرض",
"过滤历史重复": "تصفية التاريخ تكرار",
"缓存条数": "عدد شرائط التخزين المؤقت"
}

View File

@ -807,5 +807,7 @@
"网络请求": "網絡請求",
"网页显示": "網頁顯示",
"显示时间": "顯示時間",
"不显示时间": "不顯示時間"
"不显示时间": "不顯示時間",
"过滤历史重复": "過濾歷史重複",
"缓存条数": "緩存條數"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Network request",
"网页显示": "Web page display",
"显示时间": "Display time",
"不显示时间": "Do not display time"
"不显示时间": "Do not display time",
"过滤历史重复": "Filter historical duplicates",
"缓存条数": "Number of cache entries"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Solicitud de red",
"网页显示": "Pantalla de la página web",
"显示时间": "Tiempo de visualización",
"不显示时间": "No muestra el tiempo"
"不显示时间": "No muestra el tiempo",
"过滤历史重复": "Filtrar repetición histórica",
"缓存条数": "Número de barras de caché"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Demande de réseau",
"网页显示": "Affichage de la page Web",
"显示时间": "Afficher l'heure",
"不显示时间": "Ne pas afficher l'heure"
"不显示时间": "Ne pas afficher l'heure",
"过滤历史重复": "Filtrer historique répétition",
"缓存条数": "Nombre de barres de cache"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Richiesta di rete",
"网页显示": "Visualizzazione pagina web",
"显示时间": "Tempo di visualizzazione",
"不显示时间": "Non visualizzare l'ora"
"不显示时间": "Non visualizzare l'ora",
"过滤历史重复": "Filtra duplicati storici",
"缓存条数": "Numero di voci della cache"
}

View File

@ -807,5 +807,7 @@
"网络请求": "ネットワーク要求",
"网页显示": "Webページ表示",
"显示时间": "表示時間",
"不显示时间": "時間を表示しない"
"不显示时间": "時間を表示しない",
"过滤历史重复": "フィルタ履歴の繰り返し",
"缓存条数": "キャッシュ・エントリ数"
}

View File

@ -807,5 +807,7 @@
"网络请求": "네트워크 요청",
"网页显示": "웹 페이지 표시",
"显示时间": "표시 시간",
"不显示时间": "시간 표시 안 함"
"不显示时间": "시간 표시 안 함",
"过滤历史重复": "반복된 기록 필터링",
"缓存条数": "캐시 바 수"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Żądanie sieci",
"网页显示": "Wyświetlanie strony internetowej",
"显示时间": "Czas wyświetlania",
"不显示时间": "Nie wyświetlaj czasu"
"不显示时间": "Nie wyświetlaj czasu",
"过滤历史重复": "Filtruj duplikaty historyczne",
"缓存条数": "Liczba wpisów pamięci podręcznej"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Сетевой запрос",
"网页显示": "Показать страницу",
"显示时间": "Показать время",
"不显示时间": "Не показывать время"
"不显示时间": "Не показывать время",
"过滤历史重复": "Фильтровать повторение истории",
"缓存条数": "Количество кэшированных записей"
}

View File

@ -807,5 +807,7 @@
"网络请求": "คำขอเครือข่าย",
"网页显示": "แสดงหน้าเว็บ",
"显示时间": "แสดงเวลา",
"不显示时间": "ไม่แสดงเวลา"
"不显示时间": "ไม่แสดงเวลา",
"过滤历史重复": "ประวัติการกรอง ทำซ้ำ",
"缓存条数": "จำนวนแถบแคช"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Network request",
"网页显示": "Web sayfası gösterisi",
"显示时间": "Display time",
"不显示时间": "Zamanı gösterme"
"不显示时间": "Zamanı gösterme",
"过滤历史重复": "Tarihi çizgileri sil",
"缓存条数": "Cache girişlerinin sayısı"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Запит на мережу",
"网页显示": "Показ веб- сторінок",
"显示时间": "Час показу",
"不显示时间": "Не показувати час"
"不显示时间": "Не показувати час",
"过滤历史重复": "Фільтрувати історичні дублікати",
"缓存条数": "Кількість записів кешу"
}

View File

@ -807,5 +807,7 @@
"网络请求": "Yêu cầu mạng",
"网页显示": "Hiển thị trang web",
"显示时间": "Thời gian hiển thị",
"不显示时间": "Không hiển thị thời gian"
"不显示时间": "Không hiển thị thời gian",
"过滤历史重复": "Lọc lịch sử lặp lại",
"缓存条数": "Số thanh bộ nhớ cache"
}

View File

@ -807,5 +807,7 @@
"网络请求": "",
"网页显示": "",
"显示时间": "",
"不显示时间": ""
"不显示时间": "",
"过滤历史重复": "",
"缓存条数": ""
}