mirror of
https://github.com/Loyalsoldier/v2ray-rules-dat.git
synced 2024-11-30 08:55:38 +08:00
Add findRedundantDomain.py Python script
This commit is contained in:
parent
335b4d5b39
commit
de9bd9a473
67
findRedundantDomain.py
Normal file
67
findRedundantDomain.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import sys
|
||||||
|
print(sys.argv[1], sys.argv[2])
|
||||||
|
|
||||||
|
''' Find redundant items in domain lists.
|
||||||
|
e.g. 'bar.foo.com' is redundant for 'foo.com'.
|
||||||
|
'''
|
||||||
|
|
||||||
|
def load(list):
|
||||||
|
''' Parse conf file & Prepare data structure
|
||||||
|
Returns: [ ['abc', 'com'],
|
||||||
|
['bar', 'foo', 'com'],
|
||||||
|
... ]
|
||||||
|
'''
|
||||||
|
|
||||||
|
results = []
|
||||||
|
with open(list, 'r') as f:
|
||||||
|
for line in f.readlines():
|
||||||
|
line = line.strip()
|
||||||
|
if line == '' or line.startswith('#'):
|
||||||
|
continue
|
||||||
|
# A domain name is case-insensitive and
|
||||||
|
# consists of several labels, separated by a full stop
|
||||||
|
domain_labels = line.lower().split('.')
|
||||||
|
results.append(domain_labels)
|
||||||
|
|
||||||
|
# Sort results by domain labels' length
|
||||||
|
results.sort(key=len)
|
||||||
|
return results
|
||||||
|
|
||||||
|
def find(labelses, removedDomainFile):
|
||||||
|
''' Find redundant items by a tree of top-level domain label to sub-level.
|
||||||
|
`tree` is like { 'com': { 'foo: { 'bar': LEAF },
|
||||||
|
'abc': LEAF },
|
||||||
|
'org': ... }
|
||||||
|
'''
|
||||||
|
|
||||||
|
tree = {}
|
||||||
|
LEAF = 1
|
||||||
|
for labels in labelses:
|
||||||
|
domain = '.'.join(labels)
|
||||||
|
# Init root node as current node
|
||||||
|
node = tree
|
||||||
|
while len(labels) > 0:
|
||||||
|
label = labels.pop()
|
||||||
|
if label in node:
|
||||||
|
# If child node is a LEAF node,
|
||||||
|
# current domain must be an existed domain or a subdomain of an existed.
|
||||||
|
if node[label] == LEAF:
|
||||||
|
print(f"Redundant found: {domain} at {'.'.join(labels)}")
|
||||||
|
with open(removedDomainFile, "a") as f:
|
||||||
|
f.write(domain)
|
||||||
|
f.write("\n")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
# Create a leaf node if current label is last one
|
||||||
|
if len(labels) == 0:
|
||||||
|
node[label] = LEAF
|
||||||
|
# Create a branch node
|
||||||
|
else:
|
||||||
|
node[label] = {}
|
||||||
|
# Iterate to child node
|
||||||
|
node = node[label]
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
find(load(sys.argv[1]), sys.argv[2])
|
Loading…
Reference in New Issue
Block a user