iandees · September 7, 2016 20:26 · Sep 1, 2016 · Sep 1, 2016 · Sep 1, 2016 · Aug 31, 2016
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -34,6 +34,8 @@ These instructions replicate this layer with the more recent TIGER 2016 release.
 
 5. Run the resulting CSV through tippecanoe to generate an mbtiles file.
 
-
+   ```bash
+   (find . -type f -name '*.expanded.csv' -exec cat {} \;) | ./tippecanoe-master/tippecanoe -o expanded.mbtiles
+   ```
 
 6. Send the mbtiles file to MapBox for rendering.
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -28,9 +28,9 @@ These instructions replicate this layer with the more recent TIGER 2016 release.
 
 4. Use the included Python script to join the `ROADS` and `FEATNAMES` data sets and expand the abbreviated road names. The resulting data will be written as newline-separated GeoJSON features.
 
-```bash
-find . -name '*_roads.shp' -print | xargs -t -L1 --max-procs=4 -Ifile /bin/sh -c 'base=$(basename file _roads.shp) && python merge_tiger_roads.py $base/${base}_roads.shp.csv $base/${base}_featnames.dbf.csv $base/$base.expanded.json'
-```
+   ```bash
+   find . -name '*_roads.shp' -print | xargs -t -L1 --max-procs=4 -Ifile /bin/sh -c 'base=$(basename file _roads.shp) && python merge_tiger_roads.py $base/${base}_roads.shp.csv $base/${base}_featnames.dbf.csv $base/$base.expanded.json'
+   ```
 
 5. Run the resulting CSV through tippecanoe to generate an mbtiles file.
 

diff --git a/merge_tiger_roads.py b/merge_tiger_roads.py
@@ -1,6 +1,11 @@
 import unicodecsv
 import argparse
 import sys
+import simplejson as json
+from shapely import wkt
+from shapely.geometry import mapping
+import shapely.speedups
+shapely.speedups.enable()
 
 parser = argparse.ArgumentParser()
 parser.add_argument('roadfile', type=argparse.FileType('r'))
@@ -43,14 +48,17 @@
             ]))
         }
 
-outcsv = unicodecsv.DictWriter(args.outfile, ['LINEARID', 'MTFCC', 'NAME_EXPANDED', 'WKT'])
 for road in unicodecsv.DictReader(args.roadfile):
     linearid = road['LINEARID']
     featname = featnames.get(linearid)
+    parsed_shape = wkt.loads(road['WKT'])
     out = {
-        'LINEARID': linearid,
-        'WKT': road['WKT'],
+        'type': 'Feature',
+        'geometry': mapping(parsed_shape),
+        'properties': {
+            'LINEARID': linearid,
+        }
     }
     if featname:
-        out.update(featname)
-    outcsv.writerow(out)
+        out['properties'].update(featname)
+    args.outfile.write(json.dumps(out) + '\n')
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -12,23 +12,24 @@ These instructions replicate this layer with the more recent TIGER 2016 release.
    wget -e robots=off --quiet --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/ROADS/
    ```
 
-2. Unzip the TIGER data 
+2. Unzip the TIGER data into per-county directories.
 
    ```bash
-   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -t -L1 --max-procs=12 unzip -q
-   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -t -L1 --max-procs=12 unzip -q
+   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -t -L1 --max-procs=4 /bin/sh -c 'unzip -q file -d $(basename file _roads.zip)'
+   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -t -L1 --max-procs=4 /bin/sh -c 'unzip -q file -d $(basename file _featnames.zip)'
    ```
 
 3. Convert the `ROADS` Shapefiles and `FEATNAMES` DBF files into CSVs.
 
    ```bash
    find . -name '*.dbf' -print0 | xargs -t -0 --max-procs=4 -Ifile ogr2ogr -f CSV file.csv file
-   find . -name '*.shp' -print0 | xargs -t -0 --max-procs=8 -Ifile ogr2ogr -lco GEOMETRY=AS_WKT -f CSV file.csv file
+   find . -name '*.shp' -print0 | xargs -t -0 --max-procs=4 -Ifile ogr2ogr -lco GEOMETRY=AS_WKT -f CSV file.csv file
    ```
 
-4. Use the included Python script to join the `ROADS` and `FEATNAMES` data sets and expand the abbreviated road names. The geometries from the `ROADS` table will be encoded in Well-Known-Text format in a column of the CSV.
+4. Use the included Python script to join the `ROADS` and `FEATNAMES` data sets and expand the abbreviated road names. The resulting data will be written as newline-separated GeoJSON features.
 
 ```bash
+find . -name '*_roads.shp' -print | xargs -t -L1 --max-procs=4 -Ifile /bin/sh -c 'base=$(basename file _roads.shp) && python merge_tiger_roads.py $base/${base}_roads.shp.csv $base/${base}_featnames.dbf.csv $base/$base.expanded.json'
 ```
 
 5. Run the resulting CSV through tippecanoe to generate an mbtiles file.

diff --git a/merge_tiger_roads.py b/merge_tiger_roads.py
@@ -0,0 +1,56 @@
+import unicodecsv
+import argparse
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument('roadfile', type=argparse.FileType('r'))
+parser.add_argument('featnamefile', type=argparse.FileType('r'))
+parser.add_argument('outfile', type=argparse.FileType('w'), nargs='?', default=sys.stdout)
+args = parser.parse_args()
+
+with open('2016_feature_name_directionals.csv', 'r') as f:
+    directionals = dict(
+        (row['Direction Code'], row['Expanded Full Text'])
+        for row in unicodecsv.DictReader(f)
+    )
+
+with open('2016_feature_name_qualifiers.csv', 'r') as f:
+    qualifiers = dict(
+        (row['Qualifier Code'], row['Expanded Full Text'])
+        for row in unicodecsv.DictReader(f)
+    )
+
+with open('2016_feature_name_types.csv', 'r') as f:
+    types = dict(
+        (row['Type Code'], row['Expanded Full Text'])
+        for row in unicodecsv.DictReader(f)
+    )
+
+featnames = dict()
+for featname in unicodecsv.DictReader(args.featnamefile):
+    linearid = featname['LINEARID']
+    if linearid not in featnames:
+        featnames[linearid] = {
+            'MTFCC': featname.get('MTFCC'),
+            'NAME_EXPANDED': ' '.join(filter(None, [
+                qualifiers.get(featname.get('PREQUAL')),
+                directionals.get(featname.get('PREDIR')),
+                types.get(featname.get('PRETYP')),
+                featname.get('NAME'),
+                types.get(featname.get('SUFTYP')),
+                directionals.get(featname.get('SUFDIR')),
+                qualifiers.get(featname.get('SUFQUAL')),
+            ]))
+        }
+
+outcsv = unicodecsv.DictWriter(args.outfile, ['LINEARID', 'MTFCC', 'NAME_EXPANDED', 'WKT'])
+for road in unicodecsv.DictReader(args.roadfile):
+    linearid = road['LINEARID']
+    featname = featnames.get(linearid)
+    out = {
+        'LINEARID': linearid,
+        'WKT': road['WKT'],
+    }
+    if featname:
+        out.update(featname)
+    outcsv.writerow(out)
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -15,11 +15,24 @@ These instructions replicate this layer with the more recent TIGER 2016 release.
 2. Unzip the TIGER data 
 
    ```bash
-   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -L1 -P4 unzip -q
-   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -L1 -P4 unzip -q
+   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -t -L1 --max-procs=12 unzip -q
+   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -t -L1 --max-procs=12 unzip -q
    ```
 
 3. Convert the `ROADS` Shapefiles and `FEATNAMES` DBF files into CSVs.
+
+   ```bash
+   find . -name '*.dbf' -print0 | xargs -t -0 --max-procs=4 -Ifile ogr2ogr -f CSV file.csv file
+   find . -name '*.shp' -print0 | xargs -t -0 --max-procs=8 -Ifile ogr2ogr -lco GEOMETRY=AS_WKT -f CSV file.csv file
+   ```
+
 4. Use the included Python script to join the `ROADS` and `FEATNAMES` data sets and expand the abbreviated road names. The geometries from the `ROADS` table will be encoded in Well-Known-Text format in a column of the CSV.
+
+```bash
+```
+
 5. Run the resulting CSV through tippecanoe to generate an mbtiles file.
+
+
+
 6. Send the mbtiles file to MapBox for rendering.
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -19,4 +19,7 @@ These instructions replicate this layer with the more recent TIGER 2016 release.
    find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -L1 -P4 unzip -q
    ```
 
-3. 
+3. Convert the `ROADS` Shapefiles and `FEATNAMES` DBF files into CSVs.
+4. Use the included Python script to join the `ROADS` and `FEATNAMES` data sets and expand the abbreviated road names. The geometries from the `ROADS` table will be encoded in Well-Known-Text format in a column of the CSV.
+5. Run the resulting CSV through tippecanoe to generate an mbtiles file.
+6. Send the mbtiles file to MapBox for rendering.
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -1,4 +1,9 @@
-Based on [Eric Fischer's TIGER2015 work](https://github.com/ericfischer/tiger-delta).
+Building TIGER 2016 Road Tiles
+==============================
+
+US Census Bureau's TIGER dataset is one of the primary nationwide geographic datasets. Roughly 10 years ago, it was imported into OpenStreetMap and huge swaths of it haven't been touched since, even though the TIGER dataset is updated yearly. Based on earlier work that OpenStreetMap US did, [Eric Fischer's TIGER2015 layer](https://github.com/ericfischer/tiger-delta) provides an overlay that helps mappers identify roads that are missing from OpenStreetMap and gives a way to find street names for roads that might not have names in OpenStreetMap.
+
+These instructions replicate this layer with the more recent TIGER 2016 release. The TIGER dataset includes a `ROADS` and `FEATNAMES` dataset. The `ROADS` dataset includes geometries and a `linearid` that can be joined with the `linearid` in the `FEATNAMES` dataset. In `FEATNAMES` the road names are broken into several pieces, which we expand (unabbreviate) and concatenate to form a display label. Finally, the resulting joined data is built into a mbtiles file with [tippecanoe]() and uploaded to MapBox Studio for styling.
 
 1. Download TIGER data. `ROADS` for the geometries, `FEATNAMES` for the split-apart road names.
 

diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -3,61 +3,15 @@ Based on [Eric Fischer's TIGER2015 work](https://github.com/ericfischer/tiger-de
 1. Download TIGER data. `ROADS` for the geometries, `FEATNAMES` for the split-apart road names.
 
    ```bash
-   wget -e robots=off --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/
-   wget -e robots=off --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/ROADS/
+   wget -e robots=off --quiet --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/
+   wget -e robots=off --quiet --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/ROADS/
    ```
 
 2. Unzip the TIGER data 
 
    ```bash
-   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -L1 -P8 unzip -q
-   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -L1 -P8 unzip -q
+   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -L1 -P4 unzip -q
+   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -L1 -P4 unzip -q
    ```
 
-3. Import data into a PostGIS database
-
-   ```bash
-   createuser -E tiger
-   createdb -E utf8 -O tiger tiger
-   psql -d tiger -c "CREATE EXTENSION postgis;"
-   shp2pgsql -n -p tl_2016_55079_featnames.dbf featnames | psql -q -d tiger -U tiger
-   for fn in *_featnames.dbf; do echo $fn; shp2pgsql -n -a $fn featnames | psql -q -d tiger -U tiger; done
-   shp2pgsql -p tl_2016_55079_roads.dbf roads | psql -q -d tiger -U tiger
-   for fn in *_roads.shp; do echo $fn; shp2pgsql -a $fn roads | psql -q -d tiger -U tiger; done
-   ```
-
-4. Add an index on the feature names linearid column to speed later joining.
-
-   ```bash
-   psql -d tiger -c "create index on featnames (linearid);"
-   ```
-
-5. Add the feature name feature code tables.
-
-   ```bash
-   ## Directionals
-   wget https://gist.github.com/iandees/5ec04ace6a9eb47b1ef8f7aeaa5f5bd2/raw/6d6025b73f4a670ef476515865d504c552b11742/2016_feature_name_directionals.csv
-   psql -d tiger -U tiger -c "create table feature_name_directionals (direction_code varchar, expanded_full varchar, display_abbr varchar, spanish varchar, translation varchar)"
-   psql -d tiger -U tiger -c "\copy feature_name_directionals FROM '2016_feature_name_directionals.csv' delimiter ',' header csv"
-   psql -d tiger -U tiger -c "create index on feature_name_directionals (direction_code);"
-
-   ## Qualifiers
-   wget https://gist.github.com/iandees/5ec04ace6a9eb47b1ef8f7aeaa5f5bd2/raw/6d6025b73f4a670ef476515865d504c552b11742/2016_feature_name_qualifiers.csv
-   psql -d tiger -U tiger -c "CREATE TABLE feature_name_qualifiers (qualifier_code varchar, expanded_full varchar, display_abbr varchar, prefix_qual varchar, suffix_qual varchar)"
-   psql -d tiger -U tiger -c "\copy feature_name_qualifiers FROM '2016_feature_name_qualifiers.csv' delimiter ',' header csv"
-   psql -d tiger -U tiger -c "create index on feature_name_qualifiers (qualifier_code);"
-
-   ## Types
-   wget https://gist.github.com/iandees/5ec04ace6a9eb47b1ef8f7aeaa5f5bd2/raw/6d6025b73f4a670ef476515865d504c552b11742/2016_feature_name_types.csv
-   psql -d tiger -U tiger -c "CREATE TABLE feature_name_types (type_code varchar, expanded_full varchar, display_abbr varchar, spanish varchar, translation varchar, prefix_type varchar, suffix_type varchar)"
-   psql -d tiger -U tiger -c "\copy feature_name_types FROM '2016_feature_name_types.csv' delimiter ',' header csv"
-   psql -d tiger -U tiger -c "create index on feature_name_types (type_code);"
-   ```
-
-6. Create a view with the merged and expanded feature names and roads data.
-
-   ```bash
-   psql -d tiger -U tiger -c "create table feature_name_directionals (direction_code varchar, expanded_full varchar, display_abbr varchar, spanish varchar, translation varchar)"
-   ```
-
-6. 
+3. 
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -26,7 +26,13 @@ Based on [Eric Fischer's TIGER2015 work](https://github.com/ericfischer/tiger-de
    for fn in *_roads.shp; do echo $fn; shp2pgsql -a $fn roads | psql -q -d tiger -U tiger; done
    ```
 
-4. Add the feature name feature code tables.
+4. Add an index on the feature names linearid column to speed later joining.
+
+   ```bash
+   psql -d tiger -c "create index on featnames (linearid);"
+   ```
+
+5. Add the feature name feature code tables.
 
    ```bash
    ## Directionals
@@ -48,4 +54,10 @@ Based on [Eric Fischer's TIGER2015 work](https://github.com/ericfischer/tiger-de
    psql -d tiger -U tiger -c "create index on feature_name_types (type_code);"
    ```
 
-5. 
+6. Create a view with the merged and expanded feature names and roads data.
+
+   ```bash
+   psql -d tiger -U tiger -c "create table feature_name_directionals (direction_code varchar, expanded_full varchar, display_abbr varchar, spanish varchar, translation varchar)"
+   ```
+
+6. 
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -26,4 +26,26 @@ Based on [Eric Fischer's TIGER2015 work](https://github.com/ericfischer/tiger-de
    for fn in *_roads.shp; do echo $fn; shp2pgsql -a $fn roads | psql -q -d tiger -U tiger; done
    ```
 
-4. 
+4. Add the feature name feature code tables.
+
+   ```bash
+   ## Directionals
+   wget https://gist.github.com/iandees/5ec04ace6a9eb47b1ef8f7aeaa5f5bd2/raw/6d6025b73f4a670ef476515865d504c552b11742/2016_feature_name_directionals.csv
+   psql -d tiger -U tiger -c "create table feature_name_directionals (direction_code varchar, expanded_full varchar, display_abbr varchar, spanish varchar, translation varchar)"
+   psql -d tiger -U tiger -c "\copy feature_name_directionals FROM '2016_feature_name_directionals.csv' delimiter ',' header csv"
+   psql -d tiger -U tiger -c "create index on feature_name_directionals (direction_code);"
+
+   ## Qualifiers
+   wget https://gist.github.com/iandees/5ec04ace6a9eb47b1ef8f7aeaa5f5bd2/raw/6d6025b73f4a670ef476515865d504c552b11742/2016_feature_name_qualifiers.csv
+   psql -d tiger -U tiger -c "CREATE TABLE feature_name_qualifiers (qualifier_code varchar, expanded_full varchar, display_abbr varchar, prefix_qual varchar, suffix_qual varchar)"
+   psql -d tiger -U tiger -c "\copy feature_name_qualifiers FROM '2016_feature_name_qualifiers.csv' delimiter ',' header csv"
+   psql -d tiger -U tiger -c "create index on feature_name_qualifiers (qualifier_code);"
+
+   ## Types
+   wget https://gist.github.com/iandees/5ec04ace6a9eb47b1ef8f7aeaa5f5bd2/raw/6d6025b73f4a670ef476515865d504c552b11742/2016_feature_name_types.csv
+   psql -d tiger -U tiger -c "CREATE TABLE feature_name_types (type_code varchar, expanded_full varchar, display_abbr varchar, spanish varchar, translation varchar, prefix_type varchar, suffix_type varchar)"
+   psql -d tiger -U tiger -c "\copy feature_name_types FROM '2016_feature_name_types.csv' delimiter ',' header csv"
+   psql -d tiger -U tiger -c "create index on feature_name_types (type_code);"
+   ```
+
+5. 
diff --git a/tiger2016_tiles.md b/tiger2016_tiles.md
@@ -0,0 +1,29 @@
+Based on [Eric Fischer's TIGER2015 work](https://github.com/ericfischer/tiger-delta).
+
+1. Download TIGER data. `ROADS` for the geometries, `FEATNAMES` for the split-apart road names.
+
+   ```bash
+   wget -e robots=off --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/
+   wget -e robots=off --mirror --no-parent --continue http://www2.census.gov/geo/tiger/TIGER2016/ROADS/
+   ```
+
+2. Unzip the TIGER data 
+
+   ```bash
+   find www2.census.gov/geo/tiger/TIGER2016/ROADS/ -name '*.zip' -print | xargs -L1 -P8 unzip -q
+   find www2.census.gov/geo/tiger/TIGER2016/FEATNAMES/ -name '*.zip' -print | xargs -L1 -P8 unzip -q
+   ```
+
+3. Import data into a PostGIS database
+
+   ```bash
+   createuser -E tiger
+   createdb -E utf8 -O tiger tiger
+   psql -d tiger -c "CREATE EXTENSION postgis;"
+   shp2pgsql -n -p tl_2016_55079_featnames.dbf featnames | psql -q -d tiger -U tiger
+   for fn in *_featnames.dbf; do echo $fn; shp2pgsql -n -a $fn featnames | psql -q -d tiger -U tiger; done
+   shp2pgsql -p tl_2016_55079_roads.dbf roads | psql -q -d tiger -U tiger
+   for fn in *_roads.shp; do echo $fn; shp2pgsql -a $fn roads | psql -q -d tiger -U tiger; done
+   ```
+
+4.