Remove repeat domains

This commit is contained in:
Loyalsoldier 2019-12-15 18:14:44 +08:00 committed by GitHub
parent 8888445e2e
commit 9ad5f9fd0d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -4,7 +4,7 @@ on:
- cron: "0 19 * * *" - cron: "0 19 * * *"
push: push:
branches: branches:
- master - master
jobs: jobs:
build: build:
@ -25,6 +25,8 @@ jobs:
echo "::set-env name=NC::\033[0m" echo "::set-env name=NC::\033[0m"
echo "::set-env name=GEOIP_REPO::github.com/v2ray/geoip" echo "::set-env name=GEOIP_REPO::github.com/v2ray/geoip"
echo "::set-env name=GEOSITE_REPO::github.com/v2ray/domain-list-community" echo "::set-env name=GEOSITE_REPO::github.com/v2ray/domain-list-community"
echo "::set-env name=GOOGLE_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf"
echo "::set-env name=APPLE_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf"
echo "::set-env name=GFWLIST_URL::https://cokebar.github.io/gfwlist2dnsmasq/gfwlist_domain.txt" echo "::set-env name=GFWLIST_URL::https://cokebar.github.io/gfwlist2dnsmasq/gfwlist_domain.txt"
echo "::set-env name=Profiles_URL::https://raw.githubusercontent.com/ConnersHua/Profiles/master/Shadow/Pro.conf" echo "::set-env name=Profiles_URL::https://raw.githubusercontent.com/ConnersHua/Profiles/master/Shadow/Pro.conf"
echo "::set-env name=Blocked_DOMAINS_URL::https://raw.githubusercontent.com/wongsyrone/domain-block-list/master/domains.txt" echo "::set-env name=Blocked_DOMAINS_URL::https://raw.githubusercontent.com/wongsyrone/domain-block-list/master/domains.txt"
@ -50,35 +52,40 @@ jobs:
run: | run: |
go get -u -v -insecure $GEOSITE_REPO go get -u -v -insecure $GEOSITE_REPO
- name: Get and add gfwlist into geolocation-!cn - name: Get and add gfwlist into temp-proxy.txt file
run: | run: |
cd $GOPATH/src/$GEOSITE_REPO/data cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $GFWLIST_URL > gfwlist curl -sSL $GFWLIST_URL > temp-proxy.txt
echo "include:gfwlist" >> geolocation-\!cn
- name: Get and add google domains into temp-proxy.txt file
run: |
cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $GOOGLE_URL >> temp-proxy.txt
- name: Get and add proxy domains from @ConnersHua/Profiles into geolocation-!cn - name: Get and add apple domains into temp-proxy.txt file
run: | run: |
cd $GOPATH/src/$GEOSITE_REPO/data cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $Profiles_URL | awk '/^DOMAIN.+PROXY/' | cut -d ',' -f 2 > profileproxy curl -sSL $APPLE_URL >> temp-proxy.txt
echo "include:profileproxy" >> geolocation-\!cn
- name: Get and add proxy domains from @ConnersHua/Profiles into temp-proxy.txt file
run: |
cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $Profiles_URL | awk '/^DOMAIN.+PROXY/' | cut -d ',' -f 2 >> temp-proxy.txt
- name: Get and add blocked domains from @wongsyrone/domain-block-list into geolocation-!cn - name: Get and add blocked domains from @wongsyrone/domain-block-list into temp-proxy.txt file
run: | run: |
cd $GOPATH/src/$GEOSITE_REPO/data cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $Blocked_DOMAINS_URL > blockedlist curl -sSL $Blocked_DOMAINS_URL >> temp-proxy.txt
echo "include:blockedlist" >> geolocation-\!cn
- name: Get and add direct domains from @ConnersHua/Profiles into cn - name: Get and add direct domains from @ConnersHua/Profiles into temp-direct.txt file
run: | run: |
cd $GOPATH/src/$GEOSITE_REPO/data cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $Profiles_URL | awk '/^DOMAIN.+DIRECT/' | cut -d ',' -f 2 > profiledirect curl -sSL $Profiles_URL | awk '/^DOMAIN.+DIRECT/' | cut -d ',' -f 2 > temp-direct.txt
echo "include:profiledirect" >> cn
- name: Get and add chinalist into cn - name: Get and add chinalist into temp-direct.txt file
run: | run: |
cd $GOPATH/src/$GEOSITE_REPO/data cd $GOPATH/src/$GEOSITE_REPO
curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' > chinalist curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-direct.txt
echo "include:chinalist" >> cn
- name: Get and add reject domains from @ConnersHua/Profiles into category-ads-all - name: Get and add reject domains from @ConnersHua/Profiles into category-ads-all
run: | run: |
@ -86,6 +93,17 @@ jobs:
curl -sSL $Profiles_URL | awk '/^DOMAIN.+REJECT/' | cut -d ',' -f 2 > profilereject curl -sSL $Profiles_URL | awk '/^DOMAIN.+REJECT/' | cut -d ',' -f 2 > profilereject
echo "include:profilereject" >> category-ads-all echo "include:profilereject" >> category-ads-all
- name: Remove repeated domains and write domains to new lists
run: |
cd $GOPATH/src/$GEOSITE_REPO
cat temp-proxy.txt | sort --ignore-case -u > ./data/proxylist
cat temp-direct.txt | sort --ignore-case -u > ./data/directlist
- name: Add lists into appropriate category
run: |
echo "include:proxylist" >> geolocation-\!cn
echo "include:directlist" >> cn
- name: Build geosite.dat file - name: Build geosite.dat file
run: | run: |
domain-list-community domain-list-community
@ -93,32 +111,37 @@ jobs:
- name: List above process results - name: List above process results
run: | run: |
echo ">>>>>>>>>>>>>>>>>>>>>>>>" echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo "list files in geosite folder"
ls -lah $GOPATH/src/$GEOSITE_REPO
echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo "list files in data folder" echo "list files in data folder"
ls -lah $GOPATH/src/$GEOSITE_REPO/data ls -lah $GOPATH/src/$GEOSITE_REPO/data
echo ">>>>>>>>>>>>>>>>>>>>>>>>" echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo "list last 10 lines of cn file" echo "list last 10 lines of cn file"
tail -n 10 $GOPATH/src/$GEOSITE_REPO/data/cn tail -n 10 $GOPATH/src/$GEOSITE_REPO/data/cn
echo ">>>>>>>>>>>>>>>>>>>>>>>>" echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo "list last 10 lines of geolocation-!cn file" echo "list last 10 lines of geolocation-!cn file"
tail -n 10 $GOPATH/src/$GEOSITE_REPO/data/geolocation-\!cn tail -n 10 $GOPATH/src/$GEOSITE_REPO/data/geolocation-\!cn
echo ">>>>>>>>>>>>>>>>>>>>>>>>" echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo "list last 10 lines of category-ads-all file" echo "list last 10 lines of category-ads-all file"
tail -n 10 $GOPATH/src/$GEOSITE_REPO/data/category-ads-all tail -n 10 $GOPATH/src/$GEOSITE_REPO/data/category-ads-all
echo ">>>>>>>>>>>>>>>>>>>>>>>>" echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo "list files in publish folder" echo "list files in publish folder"
ls -lah ./publish ls -lah ./publish
echo ">>>>>>>>>>>>>>>>>>>>>>>>" echo "${GREEN}>>>>>>>>>>>>>>>>>>>>>>>>${NC}"
echo -e "${GREEN}完成啦!🌈${NC}" echo -e "${GREEN}完成啦!🌈${NC}"