#!/usr/bin/env python3 # Dirty tiles in a specified area in OpenStreetMap import argparse import datetime import decimal import email.utils import math import re import sys import time import urllib.parse from typing import Generator, Set, Tuple, Union import requests Dec = decimal.Decimal USER_AGENT = 'dirty-osm.py/0.1' DN = Union[int, Dec] # Numbers def drange(a: DN, b: DN, jump: DN) -> Generator[DN, None, None]: if jump > 0: while a < b: yield a a += jump elif jump < 0: while a > b: yield a a += jump else: raise ValueError('drange() argument 3 must not be zero') def deg2num(lat_deg: float, lon_deg: float, zoom: int) -> Tuple[int, int]: lat_rad = math.radians(lat_deg) n = 2.0 ** zoom xtile = int((lon_deg + 180.0) / 360.0 * n) ytile = int((1.0 - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) / math.pi) / 2.0 * n) return (xtile, ytile) def dirty_coords(lat_deg: float, lon_deg: float, maxzoom=13) -> Set[str]: urls = set() for zoom in range(6, maxzoom): # Higher zoom levels should automatically refresh on edit # Roads don't appear at levels 1-5 so probably unneeded x, y = deg2num(lat_deg, lon_deg, zoom) url = f'https://a.tile.openstreetmap.org/{zoom}/{x}/{y}.png/dirty' urls.add(url) return urls def logical_order(url: str) -> tuple: parsed_url = urllib.parse.urlsplit(url) result = tuple((int(x) if x.isnumeric() else x) for x in re.split(r'(\d+)', parsed_url.path)) return result def dirty_area(minlat: DN, maxlat: DN, minlon: DN, maxlon: DN, maxzoom=13, dry_run=False) -> None: urls = set() if maxlat < minlat: minlat, maxlat = maxlat, minlat if maxlon < minlon: minlon, maxlon = maxlon, minlon for lat in drange(minlat, maxlat, Dec('0.005')): for lon in drange(minlon, maxlon, Dec('0.005')): urls |= dirty_coords(float(lat), float(lon), maxzoom) with requests.Session() as session: session.headers.update({'User-Agent': USER_AGENT}) delay = 0 for url in sorted(urls, key=logical_order): if dry_run: print('Would have retrieved', url, file=sys.stderr) continue r = session.get(url) while r.status_code in (429, 503): retry = r.headers.get('Retry-After') if retry: try: delay = int(retry) except ValueError: tdiff = (datetime.datetime.now() - email.utils.parsedate_to_datetime(retry)) delay = tdiff.seconds print('Waiting', delay+1, 'seconds', file=sys.stderr) time.sleep(delay+1) r = session.get(url) print(url, r) return if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--dry-run', '-n', action='store_true', help="don't download anything") parser.add_argument('lat1', type=Dec) parser.add_argument('lon1', type=Dec) parser.add_argument('lat2', type=Dec) parser.add_argument('lon2', type=Dec) parser.add_argument('maxzoom', type=int, nargs='?', default=12) options = parser.parse_args() dirty_area(options.lat1, options.lat2, options.lon1, options.lon2, options.maxzoom+1, options.dry_run)