From c6c93a44aa7911bd92f13f77e339c033c8f1209b Mon Sep 17 00:00:00 2001 From: Lord Alfred <2259979+lord-alfred@users.noreply.github.com> Date: Wed, 9 Nov 2022 00:05:59 +0300 Subject: [PATCH] Add Twitter --- .github/workflows/update.yml | 7 +++++-- README.md | 14 ++++++++++---- facebook/downloader.sh | 8 ++++---- twitter/downloader.sh | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 twitter/downloader.sh diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 7df89642..474b2e69 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -20,11 +20,12 @@ jobs: bash bing/downloader.sh bash github/downloader.sh bash facebook/downloader.sh + bash twitter/downloader.sh - name: Create All-In-One ranges run: | - cat google/ipv4.txt amazon/ipv4.txt microsoft/ipv4.txt oracle/ipv4.txt digitalocean/ipv4.txt bing/ipv4.txt github/ipv4.txt facebook/ipv4.txt | sort -h > all/ipv4.txt - cat google/ipv6.txt amazon/ipv6.txt microsoft/ipv6.txt digitalocean/ipv6.txt github/ipv6.txt facebook/ipv6.txt | sort -h > all/ipv6.txt + cat google/ipv4.txt amazon/ipv4.txt microsoft/ipv4.txt oracle/ipv4.txt digitalocean/ipv4.txt bing/ipv4.txt github/ipv4.txt facebook/ipv4.txt twitter/ipv4.txt | sort -h > all/ipv4.txt + cat google/ipv6.txt amazon/ipv6.txt microsoft/ipv6.txt digitalocean/ipv6.txt github/ipv6.txt facebook/ipv6.txt twitter/ipv6.txt | sort -h > all/ipv6.txt - name: Set up Python 3.7 uses: actions/setup-python@v2 @@ -56,6 +57,7 @@ jobs: python utils/merge.py --source=bing/ipv4.txt | sort -h > bing/ipv4_merged.txt python utils/merge.py --source=github/ipv4.txt | sort -h > github/ipv4_merged.txt python utils/merge.py --source=facebook/ipv4.txt | sort -h > facebook/ipv4_merged.txt + python utils/merge.py --source=twitter/ipv4.txt | sort -h > twitter/ipv4_merged.txt python utils/merge.py --source=all/ipv4.txt | sort -h > all/ipv4_merged.txt # ipv6 python utils/merge.py --source=google/ipv6.txt | sort -h > google/ipv6_merged.txt @@ -66,6 +68,7 @@ jobs: # bing not provide ipv6 python utils/merge.py --source=github/ipv6.txt | sort -h > github/ipv6_merged.txt python utils/merge.py --source=facebook/ipv6.txt | sort -h > facebook/ipv6_merged.txt + python utils/merge.py --source=twitter/ipv6.txt | sort -h > twitter/ipv6_merged.txt python utils/merge.py --source=all/ipv6.txt | sort -h > all/ipv6_merged.txt - name: Commit files diff --git a/README.md b/README.md index a1b4519c..2de8d884 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # IPRanges -List all IP ranges from: Google (Cloud & GoogleBot), Bing (Bingbot), Amazon (AWS), Microsoft (Azure), Oracle (Cloud), DigitalOcean, GitHub and Facebook (Meta) with daily updates. +List all IP ranges from: Google (Cloud & GoogleBot), Bing (Bingbot), Amazon (AWS), Microsoft (Azure), Oracle (Cloud), DigitalOcean, GitHub, Facebook (Meta) and Twitter with daily updates. All lists are obtained from public sources. ## List types -`ipv4.txt`/`ipv6.txt` – the list of addresses (IPv4 or IPv6), which is the result of parsing and one or more sources. +`ipv4.txt`/`ipv6.txt` – the list of addresses (IPv4 or IPv6), which is the result of parsing one or more sources. `ipv4_merged.txt`/`ipv6_merged.txt` – list of addresses, after combining them into the smallest possible list of CIDRs. @@ -66,6 +66,13 @@ All addresses are stored in `.txt` files with CIDRs, where each range is on a ne - IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/facebook/ipv6.txt - IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/facebook/ipv6_merged.txt +### Twitter + +- IPv4: https://raw.githubusercontent.com/lord-alfred/ipranges/main/twitter/ipv4.txt +- IPv4 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/twitter/ipv4_merged.txt +- IPv6: https://raw.githubusercontent.com/lord-alfred/ipranges/main/twitter/ipv6.txt +- IPv6 (merged): https://raw.githubusercontent.com/lord-alfred/ipranges/main/twitter/ipv6_merged.txt + ## All-In-One IPs A list of IP addresses from all sources combined into one file. @@ -79,10 +86,9 @@ A list of IP addresses from all sources combined into one file. - Akamai: https://github.com/SecOps-Institute/Akamai-ASN-and-IPs-List - LinkedIn: https://github.com/SecOps-Institute/LinkedInIPLists -- Twitter: https://github.com/SecOps-Institute/TwitterIPLists - TOR: https://github.com/SecOps-Institute/Tor-IP-Addresses - Spamhaus: https://github.com/SecOps-Institute/SpamhausIPLists -- Alibaba, Facebook, Linode, Telegram and others: https://github.com/im-sm/Mikrotik-IP-List +- Alibaba, Linode, Telegram and others: https://github.com/im-sm/Mikrotik-IP-List ## Author diff --git a/facebook/downloader.sh b/facebook/downloader.sh index 6b540720..2d2ba9e6 100644 --- a/facebook/downloader.sh +++ b/facebook/downloader.sh @@ -18,10 +18,10 @@ get_routes() { whois -h whois.bgp.net.br -- "-i origin $1" | grep '^route' | awk '{ print $2; }' } -get_routes 'AS32934' > /tmp/facebook.txt -get_routes 'AS54115' >> /tmp/facebook.txt -get_routes 'AS63293' >> /tmp/facebook.txt -get_routes 'AS149642' >> /tmp/facebook.txt +get_routes 'AS32934' > /tmp/facebook.txt || echo 'failed' +get_routes 'AS54115' >> /tmp/facebook.txt || echo 'failed' +get_routes 'AS63293' >> /tmp/facebook.txt || echo 'failed' +get_routes 'AS149642' >> /tmp/facebook.txt || echo 'failed' # save ipv4 diff --git a/twitter/downloader.sh b/twitter/downloader.sh new file mode 100644 index 00000000..7660d990 --- /dev/null +++ b/twitter/downloader.sh @@ -0,0 +1,36 @@ +#!/bin/bash + + +# https://www.irr.net/docs/list.html +# https://bgp.he.net/search?search%5Bsearch%5D=twitter&commit=Search +# https://github.com/SecOps-Institute/TwitterIPLists/blob/master/twitter_asn_list.lst + +set -euo pipefail +set -x + + +# get from Autonomous System +get_routes() { + whois -h riswhois.ripe.net -- "-i origin $1" | grep '^route' | awk '{ print $2; }' + whois -h whois.radb.net -- "-i origin $1" | grep '^route' | awk '{ print $2; }' + whois -h rr.ntt.net -- "-i origin $1" | grep '^route' | awk '{ print $2; }' + whois -h whois.rogerstelecom.net -- "-i origin $1" | grep '^route' | awk '{ print $2; }' + whois -h whois.bgp.net.br -- "-i origin $1" | grep '^route' | awk '{ print $2; }' +} + +get_routes 'AS13414' > /tmp/twitter.txt || echo 'failed' +get_routes 'AS35995' >> /tmp/twitter.txt || echo 'failed' +get_routes 'AS54888' >> /tmp/twitter.txt || echo 'failed' +get_routes 'AS63179' >> /tmp/twitter.txt || echo 'failed' + + +# save ipv4 +grep -v ':' /tmp/twitter.txt > /tmp/twitter-ipv4.txt + +# save ipv6 +grep ':' /tmp/twitter.txt > /tmp/twitter-ipv6.txt + + +# sort & uniq +sort -h /tmp/twitter-ipv4.txt | uniq > twitter/ipv4.txt +sort -h /tmp/twitter-ipv6.txt | uniq > twitter/ipv6.txt