Public-Mirror/meta-rules-dat
1
0
Fork 0
mirror of https://github.com/MetaCubeX/meta-rules-dat.git synced 2024-11-30 10:25:44 +08:00
Code Issues Projects Releases Wiki Activity

Refine regex scripts using Perl

Browse Source
This commit is contained in:
loyalsoldier 2020-01-27 22:29:56 +08:00
parent bc7c8f7a4c
commit f8094e3e00
1 changed files with 14 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.github/workflows/build.yml vendored
View File

@ -71,15 +71,15 @@ jobs:
- name: Get and add direct domains into temp-direct.txt file
run: |
curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' > temp-direct.txt
curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt
- name: Get and add proxy domains into temp-proxy.txt file
run: |
curl -sSL $GFWLIST_DOMAINS_URL > temp-proxy.txt
curl -sSL $GREATFIRE_DOMAINS_URL >> temp-proxy.txt
curl -sSL $GOOGLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
curl -sSL $APPLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
curl -sSL $CDN_DOMAINS_URL | awk '{print $0}' >> temp-proxy.txt
curl -sSL $GFWLIST_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt
curl -sSL $GREATFIRE_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt
curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
curl -sSL $CDN_DOMAINS_URL| perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt
curl -sSL $PROFILES_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+PROXY/ {print $2}' >> temp-proxy.txt
curl -sSL $GEQ1AN_RULES_APPLE_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Apple/ {print $2}' >> temp-proxy.txt
curl -sSL $GEQ1AN_RULES_MICROSOFT_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Microsoft/ {print $2}' >> temp-proxy.txt
@ -97,20 +97,14 @@ jobs:
cat proxy.txt >> temp-proxy.txt
cat direct.txt >> temp-direct.txt
- name: Remove repeated domains, sort domains and write domains to appropriate list
- name: Sort and generate lists
run: |
cat temp-proxy.txt | sort --ignore-case -u > proxy-sort.txt
cat temp-direct.txt | sort --ignore-case -u > direct-sort.txt
cat temp-reject.txt | sort --ignore-case -u > reject-sort.txt
- name: Filter valid domains
run: |
cat proxy-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/proxylist
cat proxy-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > proxy-filter.txt
cat direct-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/directlist
cat direct-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > direct-filter.txt
cat reject-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist
cat reject-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > reject-filter.txt
cat temp-proxy.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/proxylist
cat temp-proxy.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > proxy-excluse-list.txt
cat temp-direct.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/directlist
cat temp-direct.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > direct-excluse-list.txt
cat temp-reject.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist
cat temp-reject.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > reject-excluse-list.txt
- name: Add list into appropriate category file
run: |
@ -123,7 +117,7 @@ jobs:
run: |
domain-list-community
mv ./dlc.dat ./publish/geosite.dat
mv ./*-filter.txt ./publish/
mv ./*-excluse-list.txt ./publish/
cp -f $GOPATH/src/$GEOSITE_REPO/data/{proxy,direct,reject}list ./publish/
- name: Release dat files