Skip to content

Instantly share code, notes, and snippets.

@caarloshugo

caarloshugo/download-census-shp.sh

Forked from diegovalle/download-census-shp.sh
Last active August 29, 2015 14:10
Show Gist options
  • Save caarloshugo/ebe2cb7d4b7f53d74031 to your computer and use it in GitHub Desktop.
Save caarloshugo/ebe2cb7d4b7f53d74031 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @diegovalle diegovalle revised this gist Mar 22, 2014. 1 changed file with 24 additions and 16 deletions.
    40 changes: 24 additions & 16 deletions download-census-shp.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -15,11 +15,10 @@
    # sudo apt-get install innoextract
    # chmod +x download-census-shp.sh

    # This script has been tested on Ubuntu, on a Mac you should try and
    # use 'brew install wget' and 'brew install rename', and on Windows use
    # cygwin. The script will create a directory called 'shps' where all
    # the shapefiles are located, if something goes wrong when dowloading
    # be sure to delete it and try again
    # As of now, this script has not been tested on Windows and Mac
    # systems, only on Ubuntu. The script will create a directory
    # called 'shps' where all the shapefiles are located, if something
    # goes wrong when dowloading be sure to delete it and try again

    # Projection compatible with Google Maps
    PROJECTION="+proj=longlat +ellps=WGS84 +no_defs +towgs84=0,0,0"
    @@ -30,8 +29,11 @@ WGET="wget -w 5 --random-wait --tries=10 "
    declare -a files=("ageb_urb" "eje_vial" "estatal" "loc_rur" "loc_urb"
    "manzanas" "municipal" "servicios_a" "servicios_l" "servicios_p")

    # List of files for the national (not state level) data
    declare -a national_files=("estatal" "loc_urb" "nacional" "loc_rur" "municipal" "zonas_metro")

    # State abbreviations
    declare -a states=("ags" "bc" "bcs" "camp" "coah" "col" "chis" "chih"
    declare -a states=("national" "ags" "bc" "bcs" "camp" "coah" "col" "chis" "chih"
    "df" "dgo" "gto" "gro" "hgo" "jal" "mex" "mich" "mor" "nay" "nl" "oax"
    "pue" "qro" "qroo" "slp" "sin" "son" "tab" "tamps" "tlax" "ver" "yuc"
    "zac");
    @@ -42,7 +44,9 @@ declare -a states=("ags" "bc" "bcs" "camp" "coah" "col" "chis" "chih"
    # Second argument: the state abbreviation
    # TODO: convert the encoding from windows-1252 to utf-8
    function reproject {
    for i in "${files[@]}"
    name=$3[@]
    arr=("${!name}")
    for i in "${arr[@]}"
    do
    ogr2ogr $1/$2_$i.shp "$1"/$i.shp -t_srs "$PROJECTION"
    rm "$1"/$i*
    @@ -54,8 +58,8 @@ function reproject {
    cd ../../..
    }

    # For each of the 32 states download and reproject
    for i in {1..32}
    # For each of the 32 states (and national data == 00) download and reproject
    for i in `seq 0 32`;
    do
    # The INEGI uses a leading zero for all one digit numbers
    if [ $i -lt 10 ]
    @@ -66,20 +70,25 @@ do
    fi
    # download the files from the inegi server. 'idusr' is the id you get
    # when you register at the INEGI (yes, I'm 12 years old)
    $WGET "http://www.inegi.org.mx/sistemas/consulta_resultados/scince2010.aspx?_file=/sistemas/consulta_resultados/scince2010/Scince2010_$FILENUM.exe&idusr=80085" -O ${states[$i-1]}_scince.exe
    $WGET "http://www.inegi.org.mx/sistemas/consulta_resultados/scince2010.aspx?_file=/sistemas/consulta_resultados/scince2010/Scince2010_$FILENUM.exe&idusr=80085" -O ${states[$i]}_scince.exe
    # Extract the shapefiles from the inno setup installer windows executable
    innoextract --lowercase --silent ${states[$i-1]}_scince.exe
    innoextract --lowercase --silent ${states[$i]}_scince.exe
    # Create a directory called "shps" to store the shapefiles
    mkdir -p shps/${states[$i-1]}
    mkdir -p shps/${states[$i]}
    # Copy the shapefiles to the new directory
    cp -r app/$FILENUM/* shps/${states[$i-1]}
    cp -r app/$FILENUM/* shps/${states[$i]}
    # Delete the temp files from innoextract (note that it doesn't
    # allow you to specify the directory to extract the files)
    rm -rf app
    rm -rf tmp
    rm -rf ${states[$i - 1]}_scince.exe
    rm -rf ${states[$i]}_scince.exe
    # call the reproject function above
    reproject shps/${states[$i - 1]} ${states[$i - 1]}
    if [ $i -eq 0 ]
    then
    reproject shps/${states[$i]} ${states[$i]} national_files
    else
    reproject shps/${states[$i]} ${states[$i]} files
    fi
    # give the server a rest before downloading the next file
    sleep 20
    done
    @@ -92,4 +101,3 @@ done
    #done
    # Filter attributes (e.g. only include total population in the dbf)
    #ogr2ogr -select POB1 mexico_manazanas.shp merge.shp

  2. @diegovalle diegovalle revised this gist Jun 26, 2013. 1 changed file with 5 additions and 4 deletions.
    9 changes: 5 additions & 4 deletions download-census-shp.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -15,10 +15,11 @@
    # sudo apt-get install innoextract
    # chmod +x download-census-shp.sh

    # As of now, this script has not been tested on Windows and Mac
    # systems, only on Ubuntu. The script will create a directory
    # called 'shps' where all the shapefiles are located, if something
    # goes wrong when dowloading be sure to delete it and try again
    # This script has been tested on Ubuntu, on a Mac you should try and
    # use 'brew install wget' and 'brew install rename', and on Windows use
    # cygwin. The script will create a directory called 'shps' where all
    # the shapefiles are located, if something goes wrong when dowloading
    # be sure to delete it and try again

    # Projection compatible with Google Maps
    PROJECTION="+proj=longlat +ellps=WGS84 +no_defs +towgs84=0,0,0"
  3. @diegovalle diegovalle revised this gist Jun 24, 2013. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions download-census-shp.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -30,9 +30,9 @@ declare -a files=("ageb_urb" "eje_vial" "estatal" "loc_rur" "loc_urb"
    "manzanas" "municipal" "servicios_a" "servicios_l" "servicios_p")

    # State abbreviations
    declare -a states=("ags" "bc" "bcs" "camp" "coah" "col""chis" "chih"
    "df" "dgo" "gto" "gro" "hgo" "jal" "mex" "mich""mor" "nay" "nl" "oax"
    "pue" "qro" "qroo" "slp" "sin" "son""tab" "tamps" "tlax" "ver" "yuc"
    declare -a states=("ags" "bc" "bcs" "camp" "coah" "col" "chis" "chih"
    "df" "dgo" "gto" "gro" "hgo" "jal" "mex" "mich" "mor" "nay" "nl" "oax"
    "pue" "qro" "qroo" "slp" "sin" "son" "tab" "tamps" "tlax" "ver" "yuc"
    "zac");

    # Use gdal to reproject, and then rename the shapefiles to include
  4. @diegovalle diegovalle revised this gist Jun 23, 2013. 1 changed file with 27 additions and 15 deletions.
    42 changes: 27 additions & 15 deletions download-census-shp.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -1,20 +1,28 @@
    #! /bin/bash
    # Author: Diego Valle-Jones
    # Web: www.diegovalle.net
    # Web: http://www.diegovalle.net
    # Purpose: Download shapefiles of manzanas (blocks), agebs (census areas), ejes
    # viales (streets), interesting areas and a whole bunch of other stuff

    # Note that you'll need a recent version of innoextract
    # http://constexpr.org/innoextract/, one that can work with files
    # created by version 5.5 of Inno Setup, the one in Ubuntu universe is
    # not recent enough. The version available at
    # https://launchpad.net/~arx/+archive/release is good enough. As of
    # now, this script has not been tested on Windows and Mac systems,
    # only on Ubuntu
    # https://launchpad.net/~arx/+archive/release is good enough.

    # sudo add-apt-repository ppa:arx/release
    # sudo apt-get update
    # sudo apt-get install innoextract
    # chmod +x download-census-shp.sh

    # As of now, this script has not been tested on Windows and Mac
    # systems, only on Ubuntu. The script will create a directory
    # called 'shps' where all the shapefiles are located, if something
    # goes wrong when dowloading be sure to delete it and try again

    # Projection compatible with Google Maps
    PROJECTION="+proj=longlat +ellps=WGS84 +no_defs +towgs84=0,0,0"
    # wget command
    WGET="wget -w 5 --random-wait --tries=10 "

    # The list of shapefiles of manzanas, agebs, etc
    @@ -27,8 +35,9 @@ declare -a states=("ags" "bc" "bcs" "camp" "coah" "col""chis" "chih"
    "pue" "qro" "qroo" "slp" "sin" "son""tab" "tamps" "tlax" "ver" "yuc"
    "zac");

    # Use gdal to reproject and rename the shapefiles
    # First argument: directori of shapefiles shps/state_abbreviation
    # Use gdal to reproject, and then rename the shapefiles to include
    # a user friendly abbreviation instead of a number
    # First argument: directory of shapefiles shps/state_abbreviation
    # Second argument: the state abbreviation
    # TODO: convert the encoding from windows-1252 to utf-8
    function reproject {
    @@ -44,39 +53,42 @@ function reproject {
    cd ../../..
    }

    #For each of the 32 states download and reproject
    # For each of the 32 states download and reproject
    for i in {1..32}
    do
    #The INEGI uses a leading zero for all one digit numbers
    # The INEGI uses a leading zero for all one digit numbers
    if [ $i -lt 10 ]
    then
    FILENUM="0$i"
    else
    FILENUM="$i"
    fi
    #download the files from the inegi server idusr is the id you get
    #when you register at the INEGI (yes, I'm 12 years old)
    # download the files from the inegi server. 'idusr' is the id you get
    # when you register at the INEGI (yes, I'm 12 years old)
    $WGET "http://www.inegi.org.mx/sistemas/consulta_resultados/scince2010.aspx?_file=/sistemas/consulta_resultados/scince2010/Scince2010_$FILENUM.exe&idusr=80085" -O ${states[$i-1]}_scince.exe
    #Extract the shapefiles from the inno setup installer windows executable
    # Extract the shapefiles from the inno setup installer windows executable
    innoextract --lowercase --silent ${states[$i-1]}_scince.exe
    # Create a directory called "shps" to store the shapefiles
    mkdir -p shps/${states[$i-1]}
    # Copy the shapefiles to the new directory
    cp -r app/$FILENUM/* shps/${states[$i-1]}
    # Delete the temp files from innoextract (note that it doesn't
    # allow you to specify the directory to exctract the files)
    # allow you to specify the directory to extract the files)
    rm -rf app
    rm -rf tmp
    rm -rf ${states[$i - 1]}_scince.exe
    # call the reproject function above
    reproject shps/${states[$i - 1]} ${states[$i - 1]}
    # give the server a rest before downloading the next file
    sleep 20
    done

    # You could use the code below to merge all the states into one giant
    # shapefile of Mexico change manzanas to agebs, eje_vial or whatever
    # shapefile of Mexico. Change '_manzanas' to '_agebs' or '_eje_vial' or whatever
    #for file in $(find shps -maxdepth 2 -name "*_manzanas.shp" )
    #do
    # ogr2ogr -update -append mexico_manzanas.shp $file -f "esri shapefile" -nln merge
    #done
    # Filter attributes
    #ogr2ogr -select POB1 merge2.shp merge.shp
    # Filter attributes (e.g. only include total population in the dbf)
    #ogr2ogr -select POB1 mexico_manazanas.shp merge.shp

  5. @diegovalle diegovalle created this gist Jun 23, 2013.
    82 changes: 82 additions & 0 deletions download-census-shp.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,82 @@
    #! /bin/bash
    # Author: Diego Valle-Jones
    # Web: www.diegovalle.net
    # Purpose: Download shapefiles of manzanas (blocks), agebs (census areas), ejes
    # viales (streets), interesting areas and a whole bunch of other stuff

    # Note that you'll need a recent version of innoextract
    # http://constexpr.org/innoextract/, one that can work with files
    # created by version 5.5 of Inno Setup, the one in Ubuntu universe is
    # not recent enough. The version available at
    # https://launchpad.net/~arx/+archive/release is good enough. As of
    # now, this script has not been tested on Windows and Mac systems,
    # only on Ubuntu


    # Projection compatible with Google Maps
    PROJECTION="+proj=longlat +ellps=WGS84 +no_defs +towgs84=0,0,0"
    WGET="wget -w 5 --random-wait --tries=10 "

    # The list of shapefiles of manzanas, agebs, etc
    declare -a files=("ageb_urb" "eje_vial" "estatal" "loc_rur" "loc_urb"
    "manzanas" "municipal" "servicios_a" "servicios_l" "servicios_p")

    # State abbreviations
    declare -a states=("ags" "bc" "bcs" "camp" "coah" "col""chis" "chih"
    "df" "dgo" "gto" "gro" "hgo" "jal" "mex" "mich""mor" "nay" "nl" "oax"
    "pue" "qro" "qroo" "slp" "sin" "son""tab" "tamps" "tlax" "ver" "yuc"
    "zac");

    # Use gdal to reproject and rename the shapefiles
    # First argument: directori of shapefiles shps/state_abbreviation
    # Second argument: the state abbreviation
    # TODO: convert the encoding from windows-1252 to utf-8
    function reproject {
    for i in "${files[@]}"
    do
    ogr2ogr $1/$2_$i.shp "$1"/$i.shp -t_srs "$PROJECTION"
    rm "$1"/$i*
    done
    # rename the extra census data that comes with the shapefiles
    cd $1/tablas
    rename "s/^cpv2010/$2_cpv2010/" cpv2010*
    rm -rf cpv2010*
    cd ../../..
    }

    #For each of the 32 states download and reproject
    for i in {1..32}
    do
    #The INEGI uses a leading zero for all one digit numbers
    if [ $i -lt 10 ]
    then
    FILENUM="0$i"
    else
    FILENUM="$i"
    fi
    #download the files from the inegi server idusr is the id you get
    #when you register at the INEGI (yes, I'm 12 years old)
    $WGET "http://www.inegi.org.mx/sistemas/consulta_resultados/scince2010.aspx?_file=/sistemas/consulta_resultados/scince2010/Scince2010_$FILENUM.exe&idusr=80085" -O ${states[$i-1]}_scince.exe
    #Extract the shapefiles from the inno setup installer windows executable
    innoextract --lowercase --silent ${states[$i-1]}_scince.exe
    # Create a directory called "shps" to store the shapefiles
    mkdir -p shps/${states[$i-1]}
    # Copy the shapefiles to the new directory
    cp -r app/$FILENUM/* shps/${states[$i-1]}
    # Delete the temp files from innoextract (note that it doesn't
    # allow you to specify the directory to exctract the files)
    rm -rf app
    rm -rf tmp
    rm -rf ${states[$i - 1]}_scince.exe
    reproject shps/${states[$i - 1]} ${states[$i - 1]}
    sleep 20
    done

    # You could use the code below to merge all the states into one giant
    # shapefile of Mexico change manzanas to agebs, eje_vial or whatever
    #for file in $(find shps -maxdepth 2 -name "*_manzanas.shp" )
    #do
    # ogr2ogr -update -append mexico_manzanas.shp $file -f "esri shapefile" -nln merge
    #done
    # Filter attributes
    #ogr2ogr -select POB1 merge2.shp merge.shp