diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fecc664..fc58306 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,15 +71,15 @@ jobs: - name: Get and add direct domains into temp-direct.txt file run: | - curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' > temp-direct.txt + curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt - name: Get and add proxy domains into temp-proxy.txt file run: | - curl -sSL $GFWLIST_DOMAINS_URL > temp-proxy.txt - curl -sSL $GREATFIRE_DOMAINS_URL >> temp-proxy.txt - curl -sSL $GOOGLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt - curl -sSL $APPLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt - curl -sSL $CDN_DOMAINS_URL | awk '{print $0}' >> temp-proxy.txt + curl -sSL $GFWLIST_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt + curl -sSL $GREATFIRE_DOMAINS_URL | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt + curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt + curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt + curl -sSL $CDN_DOMAINS_URL| perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> temp-proxy.txt curl -sSL $PROFILES_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+PROXY/ {print $2}' >> temp-proxy.txt curl -sSL $GEQ1AN_RULES_APPLE_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Apple/ {print $2}' >> temp-proxy.txt curl -sSL $GEQ1AN_RULES_MICROSOFT_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Microsoft/ {print $2}' >> temp-proxy.txt @@ -97,20 +97,14 @@ jobs: cat proxy.txt >> temp-proxy.txt cat direct.txt >> temp-direct.txt - - name: Remove repeated domains, sort domains and write domains to appropriate list + - name: Sort and generate lists run: | - cat temp-proxy.txt | sort --ignore-case -u > proxy-sort.txt - cat temp-direct.txt | sort --ignore-case -u > direct-sort.txt - cat temp-reject.txt | sort --ignore-case -u > reject-sort.txt - - - name: Filter valid domains - run: | - cat proxy-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/proxylist - cat proxy-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > proxy-filter.txt - cat direct-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/directlist - cat direct-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > direct-filter.txt - cat reject-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist - cat reject-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > reject-filter.txt + cat temp-proxy.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/proxylist + cat temp-proxy.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > proxy-excluse-list.txt + cat temp-direct.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/directlist + cat temp-direct.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > direct-excluse-list.txt + cat temp-reject.txt | sort --ignore-case -u | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/ && print "$1\n"'' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist + cat temp-reject.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+)/' > reject-excluse-list.txt - name: Add list into appropriate category file run: | @@ -123,7 +117,7 @@ jobs: run: | domain-list-community mv ./dlc.dat ./publish/geosite.dat - mv ./*-filter.txt ./publish/ + mv ./*-excluse-list.txt ./publish/ cp -f $GOPATH/src/$GEOSITE_REPO/data/{proxy,direct,reject}list ./publish/ - name: Release dat files