From f8094e3e00dc84972a0b790c4692483977d96ef9 Mon Sep 17 00:00:00 2001
From: loyalsoldier <loyalsoldier@outlook.com>
Date: Mon, 27 Jan 2020 22:29:56 +0800
Subject: [PATCH] Refine regex scripts using Perl

---
 .github/workflows/build.yml | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fecc664850..fc58306f67 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -71,15 +71,15 @@ jobs:
 
       - name: Get and add direct domains into temp-direct.txt file
         run: |
-          curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' > temp-direct.txt
+          curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt
 
       - name: Get and add proxy domains into temp-proxy.txt file
         run: |
-          curl -sSL $GFWLIST_DOMAINS_URL > temp-proxy.txt
-          curl -sSL $GREATFIRE_DOMAINS_URL >> temp-proxy.txt
-          curl -sSL $GOOGLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
-          curl -sSL $APPLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
-          curl -sSL $CDN_DOMAINS_URL | awk '{print $0}' >> temp-proxy.txt
+          curl -sSL $GFWLIST_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt
+          curl -sSL $GREATFIRE_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt
+          curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
+          curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
+          curl -sSL $CDN_DOMAINS_URL| perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt
           curl -sSL $PROFILES_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+PROXY/ {print $2}' >> temp-proxy.txt
           curl -sSL $GEQ1AN_RULES_APPLE_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Apple/ {print $2}' >> temp-proxy.txt
           curl -sSL $GEQ1AN_RULES_MICROSOFT_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Microsoft/ {print $2}' >> temp-proxy.txt
@@ -97,20 +97,14 @@ jobs:
           cat proxy.txt >> temp-proxy.txt
           cat direct.txt >> temp-direct.txt
 
-      - name: Remove repeated domains, sort domains and write domains to appropriate list
+      - name: Sort and generate lists
         run: |
-          cat temp-proxy.txt | sort --ignore-case -u > proxy-sort.txt
-          cat temp-direct.txt | sort --ignore-case -u > direct-sort.txt
-          cat temp-reject.txt | sort --ignore-case -u > reject-sort.txt
-
-      - name: Filter valid domains
-        run: |
-          cat proxy-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/proxylist
-          cat proxy-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > proxy-filter.txt
-          cat direct-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/directlist
-          cat direct-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > direct-filter.txt
-          cat reject-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist
-          cat reject-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > reject-filter.txt
+          cat temp-proxy.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/proxylist
+          cat temp-proxy.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > proxy-excluse-list.txt
+          cat temp-direct.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/directlist
+          cat temp-direct.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > direct-excluse-list.txt
+          cat temp-reject.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist
+          cat temp-reject.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > reject-excluse-list.txt
 
       - name: Add list into appropriate category file
         run: |
@@ -123,7 +117,7 @@ jobs:
         run: |
           domain-list-community
           mv ./dlc.dat ./publish/geosite.dat
-          mv ./*-filter.txt ./publish/
+          mv ./*-excluse-list.txt ./publish/
           cp -f $GOPATH/src/$GEOSITE_REPO/data/{proxy,direct,reject}list ./publish/
 
       - name: Release dat files