th3gundy · December 20, 2019 12:04 · Dec 20, 2019 · Dec 20, 2019 · Dec 17, 2019 · Dec 17, 2019
diff --git a/scanio.sh b/scanio.sh
@@ -61,7 +61,6 @@ die $pid "CNAME records grepped." $msg
 
 # List of CNAMEs we're going to grep for
 declare -a arr=(
-  "\.cloudfront.net"
   "\.s3-website"
   "\.s3.amazonaws.com"
   "w.amazonaws.com"

diff --git a/scanio.sh b/scanio.sh
@@ -139,6 +139,9 @@ declare -a arr=(
   "flydns.net"
   "launchrock.com"
   "\.strikinglydns.com"
+  "cname.landingi.com"
+  "\.helprace.com"
+  "cname.canny.io"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -119,6 +119,26 @@ declare -a arr=(
   "\.uptimerobot.com"
   "\.messagingengine.com"
   "\.readmessl.com"
+  "proxy-ssl.webflow.com"
+  "-portal.apigee.net"
+  "domains.smugmug.com"
+  "unbouncepages.com"
+  "pageserve.co"
+  "smartling.com"
+  "acquia.com"
+  "pantheon.io"
+  "stats.pingdom.com"
+  "tilda.ws"
+  "privatedomain.surveygizmo.eu"
+  "privatedomain.sgizmo.com"
+  "privatedomain.sgizmoca.com"
+  "mashery.com"
+  "readme.io"
+  "\.myjetbrains.com"
+  "flyio.tk"
+  "flydns.net"
+  "launchrock.com"
+  "\.strikinglydns.com"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -118,7 +118,6 @@ declare -a arr=(
   "\.airee.ru"
   "\.uptimerobot.com"
   "\.messagingengine.com"
-  "\.surge.sh"
   "\.readmessl.com"
 )
 

diff --git a/scanio.sh b/scanio.sh
@@ -44,7 +44,7 @@ function run() {
 # Instructions for if you already have a file downloaded:
 # 1. Comment or delete the next four lines.
 # 2. Comment out the line below with the downloaded .gz file
-file=$(echo "insertfilehere.gz")
+file=$(echo "opendata_fdns_cname.json")
 
 #latest=$(curl -s "https://opendata.rapid7.com/sonar.fdns_v2/" | grep "<td><a" | tail -1 | cut -d'"' -f2)
 #file=$(echo $latest | cut -d'/' -f3)
@@ -55,7 +55,7 @@ file=$(echo "insertfilehere.gz")
 msg="Grepping for CNAME records."
 ech $msg &
 pid=$!
-zcat $file | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
+cat $file | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
   awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> cname_scanio
 die $pid "CNAME records grepped." $msg
 

diff --git a/scanio.sh b/scanio.sh
@@ -44,11 +44,12 @@ function run() {
 # Instructions for if you already have a file downloaded:
 # 1. Comment or delete the next four lines.
 # 2. Comment out the line below with the downloaded .gz file
-#file=$(echo "insertfilehere.gz")
-latest=$(curl -s "https://opendata.rapid7.com/sonar.fdns_v2/" | grep "<td><a" | tail -1 | cut -d'"' -f2)
-file=$(echo $latest | cut -d'/' -f3)
-cmd="wget -q https://opendata.rapid7.com$latest"
-run "Downloading $file (This may take a while)." "$cmd" "Finished Downloading $file"
+file=$(echo "insertfilehere.gz")
+
+#latest=$(curl -s "https://opendata.rapid7.com/sonar.fdns_v2/" | grep "<td><a" | tail -1 | cut -d'"' -f2)
+#file=$(echo $latest | cut -d'/' -f3)
+#cmd="wget -q https://opendata.rapid7.com$latest"
+#run "Downloading $file (This may take a while)." "$cmd" "Finished Downloading $file"
 
 # Parsing it into a file called cname_scanio
 msg="Grepping for CNAME records."
@@ -113,6 +114,12 @@ declare -a arr=(
   "\.cloudapp.net"
   "\.trafficmanager.net"
   "\.blob.core.windows.net"
+  "\.createsend.com"
+  "\.airee.ru"
+  "\.uptimerobot.com"
+  "\.messagingengine.com"
+  "\.surge.sh"
+  "\.readmessl.com"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -40,6 +40,11 @@ function run() {
 }
 
 # Gathering data from scans.io
+#
+# Instructions for if you already have a file downloaded:
+# 1. Comment or delete the next four lines.
+# 2. Comment out the line below with the downloaded .gz file
+#file=$(echo "insertfilehere.gz")
 latest=$(curl -s "https://opendata.rapid7.com/sonar.fdns_v2/" | grep "<td><a" | tail -1 | cut -d'"' -f2)
 file=$(echo $latest | cut -d'/' -f3)
 cmd="wget -q https://opendata.rapid7.com$latest"

diff --git a/scanio.sh b/scanio.sh
@@ -64,12 +64,10 @@ declare -a arr=(
   "s3-external"
   "s3-accelerate.amazonaws.com"
   "\.herokuapp.com"
-  "\.herokussl.com"
   "\.herokudns.com"
   "\.wordpress.com"
   "\.pantheonsite.io"
   "domains.tumblr.com"
-  "\.desk.com"
   "\.zendesk.com"
   "\.github.io"
   "\.global.fastly.net"

diff --git a/scanio.sh b/scanio.sh
@@ -83,8 +83,6 @@ declare -a arr=(
   "\.uservoice.com"
   "\.surge.sh"
   "\.bitbucket.io"
-  "\.mashery.com"
-  "\.unbouncepages.com"
   "custom.intercom.help"
   "proxy.webflow.com"
   "landing.subscribepage.com"

diff --git a/scanio.sh b/scanio.sh
@@ -42,7 +42,7 @@ function run() {
 # Gathering data from scans.io
 latest=$(curl -s "https://opendata.rapid7.com/sonar.fdns_v2/" | grep "<td><a" | tail -1 | cut -d'"' -f2)
 file=$(echo $latest | cut -d'/' -f3)
-cmd="wget -q https://opendata.rapid7.com/sonar.fdns_v2$latest"
+cmd="wget -q https://opendata.rapid7.com$latest"
 run "Downloading $file (This may take a while)." "$cmd" "Finished Downloading $file"
 
 # Parsing it into a file called cname_scanio

diff --git a/scanio.sh b/scanio.sh
@@ -40,9 +40,9 @@ function run() {
 }
 
 # Gathering data from scans.io
-latest=$(curl -s "https://scans.io/study/sonar.fdns_v2" | grep "fdns_a.json.gz" | tail -1 | cut -d'"' -f2)
-file=$(echo $latest | cut -d'/' -f7)
-cmd="wget -q $latest"
+latest=$(curl -s "https://opendata.rapid7.com/sonar.fdns_v2/" | grep "<td><a" | tail -1 | cut -d'"' -f2)
+file=$(echo $latest | cut -d'/' -f3)
+cmd="wget -q https://opendata.rapid7.com/sonar.fdns_v2$latest"
 run "Downloading $file (This may take a while)." "$cmd" "Finished Downloading $file"
 
 # Parsing it into a file called cname_scanio

diff --git a/scanio.sh b/scanio.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
-# Usage : ./scanio.sh <version number> <file>
-# Example: ./scanio.sh 2017-12-08-1512720001-fdns_a.json.gz cname_list.txt
+# Usage : ./scanio.sh <save file>
+# Example: ./scanio.sh cname_list.txt
 
 # Premium
 function ech() {
@@ -40,14 +40,16 @@ function run() {
 }
 
 # Gathering data from scans.io
-cmd="wget -q https://scans.io/data/rapid7/sonar.fdns_v2/$1"
-run "Downloading $1 (This may take a while)." "$cmd" "Finished Downloading $1"
+latest=$(curl -s "https://scans.io/study/sonar.fdns_v2" | grep "fdns_a.json.gz" | tail -1 | cut -d'"' -f2)
+file=$(echo $latest | cut -d'/' -f7)
+cmd="wget -q $latest"
+run "Downloading $file (This may take a while)." "$cmd" "Finished Downloading $file"
 
 # Parsing it into a file called cname_scanio
 msg="Grepping for CNAME records."
 ech $msg &
 pid=$!
-zcat $1 | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
+zcat $file | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
   awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> cname_scanio
 die $pid "CNAME records grepped." $msg
 
@@ -120,7 +122,7 @@ cmd="grep -Ei '${DOMAINS}' cname_scanio >> cname_db"
 run "Sorting CNAME records." "$cmd" "CNAME records sorted."
 
 # Sorting the CNAME list
-cmd="cat cname_db | cut -d' ' -f1 | sort | uniq >> $2"
+cmd="cat cname_db | cut -d' ' -f1 | sort | uniq >> $1"
 run "Cleaning up." "$cmd" "Cleaned up."
 
 # RM files.

diff --git a/scanio.sh b/scanio.sh
@@ -106,6 +106,10 @@ declare -a arr=(
   "simplebooklet.com"
   "\.gr8.com"
   "\.vendecommerce.com"
+  "\.azurewebsites.net"
+  "\.cloudapp.net"
+  "\.trafficmanager.net"
+  "\.blob.core.windows.net"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -76,7 +76,6 @@ declare -a arr=(
   "\.ghost.io"
   "cargocollective.com"
   "redirect.feedpress.me"
-  "\.freshdesk.com"
   "\.myshopify.com"
   "\.statuspage.io"
   "\.uservoice.com"

diff --git a/scanio.sh b/scanio.sh
@@ -84,6 +84,29 @@ declare -a arr=(
   "\.bitbucket.io"
   "\.mashery.com"
   "\.unbouncepages.com"
+  "custom.intercom.help"
+  "proxy.webflow.com"
+  "landing.subscribepage.com"
+  "endpoint.mykajabi.com"
+  "\.teamwork.com"
+  "\.thinkific.com"
+  "clientaccess.tave.com"
+  "wishpond.com"
+  "\.aftership.com"
+  "ideas.aha.io"
+  "domains.tictail.com"
+  "cname.mendix.net"
+  "\.bcvp0rtal.com"
+  "\.brightcovegallery.com"
+  "\.gallery.video"
+  "\.bigcartel.com"
+  "\.activehosted.com"
+  "\.createsend.com"
+  "\.acquia-test.co"
+  "\.proposify.biz"
+  "simplebooklet.com"
+  "\.gr8.com"
+  "\.vendecommerce.com"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -17,7 +17,7 @@ function ech() {
 function join_by() {
   local IFS=$1
   shift
-  echo $*
+  echo "$*"
 }
 
 # Kill function
@@ -94,7 +94,7 @@ cmd="grep -Ei '${DOMAINS}' cname_scanio >> cname_db"
 run "Sorting CNAME records." "$cmd" "CNAME records sorted."
 
 # Sorting the CNAME list
-cmd="cat cname_db | awk '{print $1}' | sort | uniq >> $2"
+cmd="cat cname_db | cut -d' ' -f1 | sort | uniq >> $2"
 run "Cleaning up." "$cmd" "Cleaned up."
 
 # RM files.

diff --git a/scanio.sh b/scanio.sh
@@ -67,7 +67,6 @@ declare -a arr=(
   "\.wordpress.com"
   "\.pantheonsite.io"
   "domains.tumblr.com"
-  "\.wpengine.com"
   "\.desk.com"
   "\.zendesk.com"
   "\.github.io"
@@ -84,6 +83,7 @@ declare -a arr=(
   "\.surge.sh"
   "\.bitbucket.io"
   "\.mashery.com"
+  "\.unbouncepages.com"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -82,6 +82,8 @@ declare -a arr=(
   "\.statuspage.io"
   "\.uservoice.com"
   "\.surge.sh"
+  "\.bitbucket.io"
+  "\.mashery.com"
 )
 
 # Prepare CNAME grep

diff --git a/scanio.sh b/scanio.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+# Usage : ./scanio.sh <version number> <file>
+# Example: ./scanio.sh 2017-12-08-1512720001-fdns_a.json.gz cname_list.txt
+
+# Premium
+function ech() {
+  spinner=( "|" "/" "-" "\\" )
+  while true; do
+    for i in ${spinner[@]}; do
+      echo -ne "\r[$i] $1"
+      sleep 0.15
+    done
+  done
+}
+
+# Joining elements together
+function join_by() {
+  local IFS=$1
+  shift
+  echo $*
+}
+
+# Kill function
+function die() {
+  disown $1
+  kill -9 $1
+
+  length=$(echo -n $3 | wc -m)
+  Count=$(($length + 5))
+  Clear=$(head -c $Count < /dev/zero | tr '\0' '\040')
+  echo -ne "\r $Clear"
+  echo -e "\r[*] $2"
+}
+
+function run() {
+  ech "$1" &
+  pid=$!
+  eval "$2"
+  die $pid "$3" "$1"
+}
+
+# Gathering data from scans.io
+cmd="wget -q https://scans.io/data/rapid7/sonar.fdns_v2/$1"
+run "Downloading $1 (This may take a while)." "$cmd" "Finished Downloading $1"
+
+# Parsing it into a file called cname_scanio
+msg="Grepping for CNAME records."
+ech $msg &
+pid=$!
+zcat $1 | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
+  awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> cname_scanio
+die $pid "CNAME records grepped." $msg
+
+# List of CNAMEs we're going to grep for
+declare -a arr=(
+  "\.cloudfront.net"
+  "\.s3-website"
+  "\.s3.amazonaws.com"
+  "w.amazonaws.com"
+  "1.amazonaws.com"
+  "2.amazonaws.com"
+  "s3-external"
+  "s3-accelerate.amazonaws.com"
+  "\.herokuapp.com"
+  "\.herokussl.com"
+  "\.herokudns.com"
+  "\.wordpress.com"
+  "\.pantheonsite.io"
+  "domains.tumblr.com"
+  "\.wpengine.com"
+  "\.desk.com"
+  "\.zendesk.com"
+  "\.github.io"
+  "\.global.fastly.net"
+  "\.helpjuice.com"
+  "\.helpscoutdocs.com"
+  "\.ghost.io"
+  "cargocollective.com"
+  "redirect.feedpress.me"
+  "\.freshdesk.com"
+  "\.myshopify.com"
+  "\.statuspage.io"
+  "\.uservoice.com"
+  "\.surge.sh"
+)
+
+# Prepare CNAME grep
+DOMAINS=$(join_by '|' ${arr[@]})
+
+# Grepping CNAMEs from the array
+cmd="grep -Ei '${DOMAINS}' cname_scanio >> cname_db"
+run "Sorting CNAME records." "$cmd" "CNAME records sorted."
+
+# Sorting the CNAME list
+cmd="cat cname_db | awk '{print $1}' | sort | uniq >> $2"
+run "Cleaning up." "$cmd" "Cleaned up."
+
+# RM files.
+rm cname_db cname_scanio
+echo "[+] Finished."