diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 4f0d83a9..a87eea00 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -13,36 +13,12 @@ jobs: fetch-depth: 0 - name: Install WHOIS client - run: sudo apt install -y whois - - - name: Download IPs - run: | - set -x - bash google/downloader.sh - bash googlebot/downloader.sh - bash amazon/downloader.sh - bash microsoft/downloader.sh - bash oracle/downloader.sh - bash digitalocean/downloader.sh - bash bing/downloader.sh - bash github/downloader.sh - bash facebook/downloader.sh - bash twitter/downloader.sh - bash linode/downloader.sh - bash telegram/downloader.sh - bash openai/downloader.sh || echo 'GPTBot download failed' # TODO: fixme - bash cloudflare/downloader.sh - bash vultr/downloader.sh - - - name: Create All-In-One ranges - run: | - cat google/ipv4.txt googlebot/ipv4.txt amazon/ipv4.txt microsoft/ipv4.txt oracle/ipv4.txt digitalocean/ipv4.txt bing/ipv4.txt github/ipv4.txt facebook/ipv4.txt twitter/ipv4.txt linode/ipv4.txt telegram/ipv4.txt openai/ipv4.txt cloudflare/ipv4.txt vultr/ipv4.txt | sort -V | uniq > all/ipv4.txt - cat google/ipv6.txt googlebot/ipv6.txt amazon/ipv6.txt microsoft/ipv6.txt digitalocean/ipv6.txt github/ipv6.txt facebook/ipv6.txt twitter/ipv6.txt linode/ipv6.txt telegram/ipv6.txt cloudflare/ipv6.txt vultr/ipv6.txt | sort -V | uniq > all/ipv6.txt + run: sudo apt install -y whois parallel gawk - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: '3.13' - name: Cache pip uses: actions/cache@v4 @@ -57,44 +33,28 @@ jobs: run: | pip install -r utils/requirements.txt - - name: Merge Ranges + - name: Download IPs + run: | + set -euo pipefail + set -x + find . -name downloader.sh | sort -h | awk '{print "Executing "$1"...";system("bash "$1)}' + + - name: Create All-In-One ranges + run: | + cat $(find . -name ipv4.txt | sort -h) | sort -V | uniq > all/ipv4.txt + cat $(find . -name ipv6.txt | sort -h) | sort -V | uniq > all/ipv6.txt + + - name: Merge ipv4 Ranges + run: | + set -euo pipefail + set -x + find . -name ipv4.txt | sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -V '>' {.}_merged.txt + + - name: Merge ipv6 Ranges run: | set -euo pipefail set -x - # ipv4 - python utils/merge.py --source=google/ipv4.txt | sort -V > google/ipv4_merged.txt - python utils/merge.py --source=googlebot/ipv4.txt | sort -V > googlebot/ipv4_merged.txt - python utils/merge.py --source=amazon/ipv4.txt | sort -V > amazon/ipv4_merged.txt - python utils/merge.py --source=microsoft/ipv4.txt | sort -V > microsoft/ipv4_merged.txt - python utils/merge.py --source=oracle/ipv4.txt | sort -V > oracle/ipv4_merged.txt - python utils/merge.py --source=digitalocean/ipv4.txt | sort -V > digitalocean/ipv4_merged.txt - python utils/merge.py --source=bing/ipv4.txt | sort -V > bing/ipv4_merged.txt - python utils/merge.py --source=github/ipv4.txt | sort -V > github/ipv4_merged.txt - python utils/merge.py --source=facebook/ipv4.txt | sort -V > facebook/ipv4_merged.txt - python utils/merge.py --source=twitter/ipv4.txt | sort -V > twitter/ipv4_merged.txt - python utils/merge.py --source=linode/ipv4.txt | sort -V > linode/ipv4_merged.txt - python utils/merge.py --source=telegram/ipv4.txt | sort -V > telegram/ipv4_merged.txt - python utils/merge.py --source=openai/ipv4.txt | sort -V > openai/ipv4_merged.txt - python utils/merge.py --source=cloudflare/ipv4.txt | sort -V > cloudflare/ipv4_merged.txt - python utils/merge.py --source=vultr/ipv4.txt | sort -V > vultr/ipv4_merged.txt - python utils/merge.py --source=all/ipv4.txt | sort -V > all/ipv4_merged.txt - # ipv6 - python utils/merge.py --source=google/ipv6.txt | sort -V > google/ipv6_merged.txt - python utils/merge.py --source=googlebot/ipv6.txt | sort -V > googlebot/ipv6_merged.txt - python utils/merge.py --source=amazon/ipv6.txt | sort -V > amazon/ipv6_merged.txt - python utils/merge.py --source=microsoft/ipv6.txt | sort -V > microsoft/ipv6_merged.txt - # oracle not provide ipv6 - python utils/merge.py --source=digitalocean/ipv6.txt | sort -V > digitalocean/ipv6_merged.txt - # bing not provide ipv6 - python utils/merge.py --source=github/ipv6.txt | sort -V > github/ipv6_merged.txt - python utils/merge.py --source=facebook/ipv6.txt | sort -V > facebook/ipv6_merged.txt - python utils/merge.py --source=twitter/ipv6.txt | sort -V > twitter/ipv6_merged.txt - python utils/merge.py --source=linode/ipv6.txt | sort -V > linode/ipv6_merged.txt - python utils/merge.py --source=telegram/ipv6.txt | sort -V > telegram/ipv6_merged.txt - # openai not provide ipv6 - python utils/merge.py --source=cloudflare/ipv6.txt | sort -V > cloudflare/ipv6_merged.txt - python utils/merge.py --source=vultr/ipv6.txt | sort -V > vultr/ipv6_merged.txt - python utils/merge.py --source=all/ipv6.txt | sort -V > all/ipv6_merged.txt + find . -name ipv6.txt | sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -V '>' {.}_merged.txt - name: Commit files env: diff --git a/README.md b/README.md index 10f667ce..387c9ab1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![IP Ranges](logo.png) -List all IP ranges from: Google (Cloud & GoogleBot), Bing (Bingbot), Amazon (AWS), Microsoft, Oracle (Cloud), DigitalOcean, GitHub, Facebook (Meta), Twitter, Linode, Telegram, OpenAI (GPTBot) and CloudFlare with daily updates. +List all IP ranges from: Google (Cloud & GoogleBot), Bing (Bingbot), Amazon (AWS), Microsoft, Oracle (Cloud), DigitalOcean, GitHub, Facebook (Meta), Twitter, Linode, Telegram, OpenAI (GPTBot), CloudFlare, Vultr, Apple (Private Relay) and ProtonVPN with daily updates. All lists are obtained from public sources. @@ -117,6 +117,20 @@ To allow GoogleBot, block all Google IP addresses and then allow the GoogleBot a - IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/vultr/ipv6.txt - IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/vultr/ipv6_merged.txt +### Apple (Private Relay) + +- IPv4: https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv4.txt +- IPv4 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv4_merged.txt +- IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv6.txt +- IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv6_merged.txt + +### ProtonVPN (exit nodes) + +- IPv4: https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv4.txt +- IPv4 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv4_merged.txt +- IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv6.txt +- IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv6_merged.txt + ## All-In-One IPs A list of IP addresses from all sources combined into one file. diff --git a/apple-proxy/downloader.sh b/apple-proxy/downloader.sh new file mode 100755 index 00000000..d86eb869 --- /dev/null +++ b/apple-proxy/downloader.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# https://developer.apple.com/icloud/prepare-your-network-for-icloud-private-relay/ +curl -s https://mask-api.icloud.com/egress-ip-ranges.csv | cut -d',' -f1 > /tmp/apple-proxy.txt + +grep -v ':' /tmp/apple-proxy.txt > /tmp/apple-proxy-ipv4.txt +grep ':' /tmp/apple-proxy.txt > /tmp/apple-proxy-ipv6.txt + +sort -V /tmp/apple-proxy-ipv4.txt | uniq > apple-proxy/ipv4.txt +sort -V /tmp/apple-proxy-ipv6.txt | uniq > apple-proxy/ipv6.txt diff --git a/openai/downloader.sh b/openai/downloader.sh index 62b21f12..b0be6e56 100644 --- a/openai/downloader.sh +++ b/openai/downloader.sh @@ -8,12 +8,37 @@ set -x # get from public ranges -curl -s https://openai.com/gptbot-ranges.txt > /tmp/openai.txt - -# save ipv4 -grep -v ':' /tmp/openai.txt > /tmp/openai-ipv4.txt - -# ipv6 not provided +download_and_parse_json() { + curl --connect-timeout 60 --retry 3 --retry-delay 15 -s "${1}" \ + -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \ + -H 'accept-language: en' \ + -H 'cache-control: no-cache' \ + -H 'pragma: no-cache' \ + -H 'priority: u=0, i' \ + -H 'sec-ch-ua: "Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + -H 'sec-fetch-dest: document' \ + -H 'sec-fetch-mode: navigate' \ + -H 'sec-fetch-site: none' \ + -H 'sec-fetch-user: ?1' \ + -H 'upgrade-insecure-requests: 1' \ + -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36' \ + > /tmp/openai.json + + jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r /tmp/openai.json > /tmp/openai.txt + + # save ipv4 + grep -v ':' /tmp/openai.txt >> /tmp/openai-ipv4.txt + + # ipv6 not provided + + sleep 10 +} + +download_and_parse_json "https://openai.com/chatgpt-user.json" +download_and_parse_json "https://openai.com/searchbot.json" +download_and_parse_json "https://openai.com/gptbot.json" # sort & uniq diff --git a/openai/ipv4.txt b/openai/ipv4.txt index 4a46e069..2e9dbda6 100644 --- a/openai/ipv4.txt +++ b/openai/ipv4.txt @@ -1,2 +1,35 @@ +4.196.118.112/28 +4.197.22.112/28 +4.227.36.0/25 +13.65.240.240/28 +20.42.10.176/28 +20.61.70.160/28 +20.97.189.96/28 +20.125.66.80/28 +20.161.75.208/28 +20.171.206.0/24 +20.171.207.0/24 +20.215.188.192/28 +23.98.142.176/28 +23.98.179.16/28 +40.84.180.64/28 +40.84.180.224/28 +40.84.221.208/28 +40.84.221.224/28 +51.8.102.0/24 +51.8.155.48/28 +51.8.155.64/28 +51.8.155.112/28 +52.156.77.144/28 +52.159.249.96/28 +52.225.75.208/28 52.230.152.0/24 52.233.106.0/24 +52.236.94.144/28 +135.234.64.0/24 +135.237.131.208/28 +172.178.140.144/28 +172.178.141.128/28 +172.182.193.160/28 +172.183.222.128/28 +172.203.190.128/28 diff --git a/protonvpn/downloader.sh b/protonvpn/downloader.sh new file mode 100755 index 00000000..57afa829 --- /dev/null +++ b/protonvpn/downloader.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail +set -x + + +# get from public ranges +curl -s https://api.protonvpn.ch/vpn/logicals > /tmp/protonvpn.json + + +# get all prefixes without some keys +jq '.LogicalServers[].Servers[].EntryIP' -r /tmp/protonvpn.json | tr -d '"' > /tmp/protonvpn-all.txt + + +# save ipv4 +grep -v ':' /tmp/protonvpn-all.txt > /tmp/protonvpn-ipv4.txt + +# save ipv6 +grep ':' /tmp/protonvpn-all.txt > /tmp/protonvpn-ipv6.txt + + +# sort & uniq +sort -V /tmp/protonvpn-ipv4.txt | uniq > protonvpn/ipv4.txt +sort -V /tmp/protonvpn-ipv6.txt | uniq > protonvpn/ipv6.txt