Add new providers (#22)

* Add Apple proxy * Add ProtonVPN * Fix download OpenAI * Improve workflow
3 months ago · 69fe4035a7
6 changed files with 137 additions and 68 deletions
--- a/.github/workflows/update.yml
+++ b/.github/workflows/update.yml
@ -13,31 +13,7 @@ jobs:
          fetch-depth: 0

      - name: Install WHOIS client
-        run: sudo apt install -y whois
-
-      - name: Download IPs
-        run: |
-          set -x
-          bash google/downloader.sh
-          bash googlebot/downloader.sh
-          bash amazon/downloader.sh
-          bash microsoft/downloader.sh
-          bash oracle/downloader.sh
-          bash digitalocean/downloader.sh
-          bash bing/downloader.sh
-          bash github/downloader.sh
-          bash facebook/downloader.sh
-          bash twitter/downloader.sh
-          bash linode/downloader.sh
-          bash telegram/downloader.sh
-          bash openai/downloader.sh || echo 'GPTBot download failed'  # TODO: fixme
-          bash cloudflare/downloader.sh
-          bash vultr/downloader.sh
-
-      - name: Create All-In-One ranges
-        run: |
-          cat google/ipv4.txt googlebot/ipv4.txt amazon/ipv4.txt microsoft/ipv4.txt oracle/ipv4.txt digitalocean/ipv4.txt bing/ipv4.txt github/ipv4.txt facebook/ipv4.txt twitter/ipv4.txt linode/ipv4.txt telegram/ipv4.txt openai/ipv4.txt cloudflare/ipv4.txt vultr/ipv4.txt | sort -V | uniq > all/ipv4.txt
-          cat google/ipv6.txt googlebot/ipv6.txt amazon/ipv6.txt microsoft/ipv6.txt digitalocean/ipv6.txt github/ipv6.txt facebook/ipv6.txt twitter/ipv6.txt linode/ipv6.txt telegram/ipv6.txt cloudflare/ipv6.txt vultr/ipv6.txt | sort -V | uniq > all/ipv6.txt
+        run: sudo apt install -y whois parallel gawk

      - name: Set up Python
        uses: actions/setup-python@v5
@ -57,44 +33,28 @@ jobs:
        run: |
          pip install -r utils/requirements.txt

-      - name: Merge Ranges
+      - name: Download IPs
+        run: |
+          set -euo pipefail
+          set -x
+          find . -name downloader.sh | sort -h | awk '{print "Executing "$1"...";system("bash "$1)}'
+
+      - name: Create All-In-One ranges
+        run: |
+          cat $(find . -name ipv4.txt | sort -h) | sort -V | uniq > all/ipv4.txt
+          cat $(find . -name ipv6.txt | sort -h) | sort -V | uniq > all/ipv6.txt
+
+      - name: Merge ipv4 Ranges
+        run: |
+          set -euo pipefail
+          set -x
+          find . -name ipv4.txt | sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|'  sort -V '>' {.}_merged.txt
+
+      - name: Merge ipv6 Ranges
        run: |
          set -euo pipefail
          set -x
-          # ipv4
-          python utils/merge.py --source=google/ipv4.txt | sort -V > google/ipv4_merged.txt
-          python utils/merge.py --source=googlebot/ipv4.txt | sort -V > googlebot/ipv4_merged.txt
-          python utils/merge.py --source=amazon/ipv4.txt | sort -V > amazon/ipv4_merged.txt
-          python utils/merge.py --source=microsoft/ipv4.txt | sort -V > microsoft/ipv4_merged.txt
-          python utils/merge.py --source=oracle/ipv4.txt | sort -V > oracle/ipv4_merged.txt
-          python utils/merge.py --source=digitalocean/ipv4.txt | sort -V > digitalocean/ipv4_merged.txt
-          python utils/merge.py --source=bing/ipv4.txt | sort -V > bing/ipv4_merged.txt
-          python utils/merge.py --source=github/ipv4.txt | sort -V > github/ipv4_merged.txt
-          python utils/merge.py --source=facebook/ipv4.txt | sort -V > facebook/ipv4_merged.txt
-          python utils/merge.py --source=twitter/ipv4.txt | sort -V > twitter/ipv4_merged.txt
-          python utils/merge.py --source=linode/ipv4.txt | sort -V > linode/ipv4_merged.txt
-          python utils/merge.py --source=telegram/ipv4.txt | sort -V > telegram/ipv4_merged.txt
-          python utils/merge.py --source=openai/ipv4.txt | sort -V > openai/ipv4_merged.txt
-          python utils/merge.py --source=cloudflare/ipv4.txt | sort -V > cloudflare/ipv4_merged.txt
-          python utils/merge.py --source=vultr/ipv4.txt | sort -V > vultr/ipv4_merged.txt
-          python utils/merge.py --source=all/ipv4.txt | sort -V > all/ipv4_merged.txt
-          # ipv6
-          python utils/merge.py --source=google/ipv6.txt | sort -V > google/ipv6_merged.txt
-          python utils/merge.py --source=googlebot/ipv6.txt | sort -V > googlebot/ipv6_merged.txt
-          python utils/merge.py --source=amazon/ipv6.txt | sort -V > amazon/ipv6_merged.txt
-          python utils/merge.py --source=microsoft/ipv6.txt | sort -V > microsoft/ipv6_merged.txt
-          # oracle not provide ipv6
-          python utils/merge.py --source=digitalocean/ipv6.txt | sort -V > digitalocean/ipv6_merged.txt
-          # bing not provide ipv6
-          python utils/merge.py --source=github/ipv6.txt | sort -V > github/ipv6_merged.txt
-          python utils/merge.py --source=facebook/ipv6.txt | sort -V > facebook/ipv6_merged.txt
-          python utils/merge.py --source=twitter/ipv6.txt | sort -V > twitter/ipv6_merged.txt
-          python utils/merge.py --source=linode/ipv6.txt | sort -V > linode/ipv6_merged.txt
-          python utils/merge.py --source=telegram/ipv6.txt | sort -V > telegram/ipv6_merged.txt
-          # openai not provide ipv6
-          python utils/merge.py --source=cloudflare/ipv6.txt | sort -V > cloudflare/ipv6_merged.txt
-          python utils/merge.py --source=vultr/ipv6.txt | sort -V > vultr/ipv6_merged.txt
-          python utils/merge.py --source=all/ipv6.txt | sort -V > all/ipv6_merged.txt
+          find . -name ipv6.txt | sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -V '>' {.}_merged.txt

      - name: Commit files
        env:
--- a/README.md
+++ b/README.md
@ -2,7 +2,7 @@

 ![IP Ranges](logo.png)

-List all IP ranges from: Google (Cloud & GoogleBot), Bing (Bingbot), Amazon (AWS), Microsoft, Oracle (Cloud), DigitalOcean, GitHub, Facebook (Meta), Twitter, Linode, Telegram, OpenAI (GPTBot) and CloudFlare with daily updates.
+List all IP ranges from: Google (Cloud & GoogleBot), Bing (Bingbot), Amazon (AWS), Microsoft, Oracle (Cloud), DigitalOcean, GitHub, Facebook (Meta), Twitter, Linode, Telegram, OpenAI (GPTBot), CloudFlare, Vultr, Apple (Private Relay) and ProtonVPN with daily updates.

 All lists are obtained from public sources.

@ -117,6 +117,20 @@ To allow GoogleBot, block all Google IP addresses and then allow the GoogleBot a
 - IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/vultr/ipv6.txt
 - IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/vultr/ipv6_merged.txt

+### Apple (Private Relay)
+
+- IPv4: https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv4.txt
+- IPv4 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv4_merged.txt
+- IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv6.txt
+- IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/apple-proxy/ipv6_merged.txt
+
+### ProtonVPN (exit nodes)
+
+- IPv4: https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv4.txt
+- IPv4 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv4_merged.txt
+- IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv6.txt
+- IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/protonvpn/ipv6_merged.txt
+
 ## All-In-One IPs

 A list of IP addresses from all sources combined into one file.
--- a/apple-proxy/downloader.sh
+++ b/apple-proxy/downloader.sh
@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+set -x
+
+# https://developer.apple.com/icloud/prepare-your-network-for-icloud-private-relay/
+curl -s https://mask-api.icloud.com/egress-ip-ranges.csv | cut -d',' -f1  > /tmp/apple-proxy.txt
+
+grep -v ':' /tmp/apple-proxy.txt > /tmp/apple-proxy-ipv4.txt
+grep ':' /tmp/apple-proxy.txt > /tmp/apple-proxy-ipv6.txt
+
+sort -V /tmp/apple-proxy-ipv4.txt | uniq > apple-proxy/ipv4.txt
+sort -V /tmp/apple-proxy-ipv6.txt | uniq > apple-proxy/ipv6.txt
--- a/openai/downloader.sh
+++ b/openai/downloader.sh
@ -8,12 +8,37 @@ set -x


 # get from public ranges
-curl -s https://openai.com/gptbot-ranges.txt > /tmp/openai.txt
-
-# save ipv4
-grep -v ':' /tmp/openai.txt > /tmp/openai-ipv4.txt
-
-# ipv6 not provided
+download_and_parse_json() {
+    curl --connect-timeout 60 --retry 3 --retry-delay 15 -s "${1}" \
+    -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
+    -H 'accept-language: en' \
+    -H 'cache-control: no-cache' \
+    -H 'pragma: no-cache' \
+    -H 'priority: u=0, i' \
+    -H 'sec-ch-ua: "Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"' \
+    -H 'sec-ch-ua-mobile: ?0' \
+    -H 'sec-ch-ua-platform: "macOS"' \
+    -H 'sec-fetch-dest: document' \
+    -H 'sec-fetch-mode: navigate' \
+    -H 'sec-fetch-site: none' \
+    -H 'sec-fetch-user: ?1' \
+    -H 'upgrade-insecure-requests: 1' \
+    -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36' \
+    > /tmp/openai.json
+
+    jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r /tmp/openai.json > /tmp/openai.txt
+
+    # save ipv4
+    grep -v ':' /tmp/openai.txt >> /tmp/openai-ipv4.txt
+
+    # ipv6 not provided
+
+    sleep 10
+}
+
+download_and_parse_json "https://openai.com/chatgpt-user.json"
+download_and_parse_json "https://openai.com/searchbot.json"
+download_and_parse_json "https://openai.com/gptbot.json"


 # sort & uniq
--- a/openai/ipv4.txt
+++ b/openai/ipv4.txt
@ -1,2 +1,35 @@
+4.196.118.112/28
+4.197.22.112/28
+4.227.36.0/25
+13.65.240.240/28
+20.42.10.176/28
+20.61.70.160/28
+20.97.189.96/28
+20.125.66.80/28
+20.161.75.208/28
+20.171.206.0/24
+20.171.207.0/24
+20.215.188.192/28
+23.98.142.176/28
+23.98.179.16/28
+40.84.180.64/28
+40.84.180.224/28
+40.84.221.208/28
+40.84.221.224/28
+51.8.102.0/24
+51.8.155.48/28
+51.8.155.64/28
+51.8.155.112/28
+52.156.77.144/28
+52.159.249.96/28
+52.225.75.208/28
 52.230.152.0/24
 52.233.106.0/24
+52.236.94.144/28
+135.234.64.0/24
+135.237.131.208/28
+172.178.140.144/28
+172.178.141.128/28
+172.182.193.160/28
+172.183.222.128/28
+172.203.190.128/28
--- a/protonvpn/downloader.sh
+++ b/protonvpn/downloader.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+set -x
+
+
+# get from public ranges
+curl -s https://api.protonvpn.ch/vpn/logicals > /tmp/protonvpn.json
+
+
+# get all prefixes without some keys
+jq '.LogicalServers[].Servers[].EntryIP' -r /tmp/protonvpn.json | tr -d '"' > /tmp/protonvpn-all.txt
+
+
+# save ipv4
+grep -v ':' /tmp/protonvpn-all.txt > /tmp/protonvpn-ipv4.txt
+
+# save ipv6
+grep ':' /tmp/protonvpn-all.txt > /tmp/protonvpn-ipv6.txt
+
+
+# sort & uniq
+sort -V /tmp/protonvpn-ipv4.txt | uniq > protonvpn/ipv4.txt
+sort -V /tmp/protonvpn-ipv6.txt | uniq > protonvpn/ipv6.txt