diff --git a/google/downloader.sh b/google/downloader.sh index 17740044..8e64a871 100755 --- a/google/downloader.sh +++ b/google/downloader.sh @@ -15,6 +15,10 @@ set -x curl -s https://www.gstatic.com/ipranges/goog.txt > /tmp/goog.txt curl -s https://www.gstatic.com/ipranges/cloud.json > /tmp/cloud.json +# Public GoogleBot IP ranges +# From: https://developers.google.com/search/docs/advanced/crawling/verifying-googlebot +curl -s https://developers.google.com/search/apis/ipranges/googlebot.json > /tmp/googlebot.json + # get from netblocks txt="$(dig TXT _netblocks.google.com +short @8.8.8.8)" idx=2 @@ -44,11 +48,13 @@ get_dns_spf "_spf.google.com" >> /tmp/netblocks.txt # save ipv4 grep -v ':' /tmp/goog.txt > /tmp/google-ipv4.txt jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r /tmp/cloud.json >> /tmp/google-ipv4.txt +jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r /tmp/googlebot.json >> /tmp/google-ipv4.txt grep -v ':' /tmp/netblocks.txt >> /tmp/google-ipv4.txt # save ipv6 grep ':' /tmp/goog.txt > /tmp/google-ipv6.txt jq '.prefixes[] | [.ipv6Prefix][] | select(. != null)' -r /tmp/cloud.json >> /tmp/google-ipv6.txt +jq '.prefixes[] | [.ipv6Prefix][] | select(. != null)' -r /tmp/googlebot.json >> /tmp/google-ipv6.txt grep ':' /tmp/netblocks.txt >> /tmp/google-ipv6.txt