droberson · April 7, 2020 02:51 · Apr 7, 2020
diff --git a/masscan-favicon-grab.py b/masscan-favicon-grab.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import fcntl
+import errno
+import socket
+import hashlib
+import requests
+import urllib3
+from threading import Thread
+from queue import Queue
+
+
+Q = Queue()
+
+
+def valid_ipv4_address(ip_address):
+    """ valid_ipv4_address() - Validate an IPv4 address.
+    Args:
+        ip_address (str) - IP address to verify.
+    Returns:
+        True if ip_address is a valid IPv4 address.
+        False if ip_address is not a valid IPv4 address.
+    """
+    try:
+        socket.inet_aton(ip_address)
+    except socket.error:
+        return False
+    return True
+
+
+def log(host, port, message):
+    with open("favicon-%s" % sys.argv[1], "a") as logfile:
+        logfile.write("%d %s %s\n" %
+                      (time.time(), host, message))
+
+
+
+def grab_favicon(host, port, timeout=5):
+    url = "http://" + host + ":" + str(port) + "/favicon.ico"
+    try:
+        r = requests.get(url, timeout=timeout)
+    except requests.exceptions.ConnectionError:
+        #print("connect failed", url)
+        return False
+    except requests.exceptions.ReadTimeout:
+        #print("timed out", url)
+        return False
+    except requests.exceptions.TooManyRedirects:
+        print(" [-] Too many redirects: %s" % url)
+        return False
+    except requests.exceptions.ContentDecodingError:
+        print(" [-] ContentDecodingError: %s" % url)
+        return False
+    except urllib3.exceptions.LocationValueError:
+        print(" [-] LocationValueError: %s" % url)
+        return False
+    except urllib3.exceptions.ProtocolError:
+        print(" [-] ProtocolError: %s" % url)
+        return False
+    except requests.exceptions.ChunkedEncodingError:
+        print(" [-] ChunkedEncodingError: %s" % url)
+        return False
+    except requests.exceptions.InvalidSchema:
+        return False
+    except UnicodeError:
+        return False
+    except AttributeError:
+        return False
+    favicon_hash = hashlib.md5(r.content).hexdigest()
+    #print(favicon_hash, host, len(r.content))
+    log(host, port, "%s %d %d" % (favicon_hash, r.status_code, len(r.content)))
+    return True
+
+
+def thread_scan(thread_id, queue):
+    while True:
+        host, port = queue.get()
+        #print(" [*] Scanning %s:%s on thread %d" % (host, str(port), thread_id))
+        grab_favicon(host, port, timeout=5)
+        queue.task_done()
+
+
+def process_ip_list(ip_list, port=0):
+    for ip in ip_list:
+        if valid_ipv4_address(ip):
+            Q.put((ip, port))
+        else:
+            print("[-] invalid ip: %s" % ip)
+
+    print("[+] Scanning %s - %s (%d/%d %d%%) -- Elapsed: %s" %
+          (ip_list[0], ip_list[-1],
+           progress, ip_count, progress / ip_count * 100,
+           display_time(round(time.time() - start_time))))
+    for thread in range(32):
+        worker = Thread(target=thread_scan, args=(thread, Q))
+        worker.setDaemon(True)
+        worker.start()
+
+    Q.join()
+
+
+def linecount(filename):
+    with open(filename, "r") as file:
+        for i, _ in enumerate(file):
+            pass
+    return i + 1
+
+
+intervals = (
+    ("weeks", 604800), # 60 * 60 * 24 * 7
+    ("days", 86400),   # 60 * 60 * 24
+    ("hours", 3600),   # 60 * 60
+    ("minutes", 60),
+    ("seconds", 1)
+)
+
+
+def display_time(seconds, granularity=2):
+    result = []
+
+    if seconds == 0:
+        return "0 seconds"
+
+    for name, count in intervals:
+        value = seconds // count
+        if value:
+            seconds -= value * count
+            if value == 1:
+                name = name.rstrip("s")
+            result.append("{} {}".format(value, name))
+    return ", ". join(result[:granularity])
+
+
+print("[+] Scanning ips in %s" % sys.argv[1])
+ip_count = linecount(sys.argv[1])
+progress = 0
+start_time = time.time()
+
+with open(sys.argv[1], "r") as fp:
+    count = 0
+    cset = []
+
+    for line in fp:
+        count += 1
+        cset += [line.rstrip()]
+        if count % 1000 == 0:
+            progress += len(cset)
+            process_ip_list(cset, port=80)
+            cset = []
+    progress += len(cset)
+    if len(cset) > 0:
+        process_ip_list(cset, 80)
+
+print("[+] Finished. Elapsed: %s" % display_time(time.time() - start_time))
+