Skip to content

Instantly share code, notes, and snippets.

@moklick

moklick/geojson-streaming.js

Last active August 29, 2015 14:20
Show Gist options
  • Save moklick/d87f64651d879b49002d to your computer and use it in GitHub Desktop.
Save moklick/d87f64651d879b49002d to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. moklick revised this gist May 12, 2015. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions geojson-streaming.js
    View file
    Original file line number Diff line number Diff line change
    @@ -17,6 +17,7 @@ var geoJsonEnd = '}';

    // here we can check if current geometry is valid, inside a polygon, etc
    var propManipulator = through.obj(function(chunk, enc, callback){
    // if(isValid)
    this.push(getCleanObj(chunk));
    callback();
    });
    @@ -41,9 +42,8 @@ function getCleanObj(dirtyObj){
    coordinates : dirtyObj.geometry.coordinates
    },
    properties: {
    gml_id: dirtyObj.properties.gml_id,
    floors: dirtyObj.properties.GESCHOSS,
    btype : dirtyObj.properties.Bezeichnun
    id: dirtyObj.properties.gml_id,
    height: dirtyObj.properties.floors * 3.5
    }
    };
    }
  2. moklick revised this gist May 11, 2015. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions geojson-streaming.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,5 @@
    // boilerplate for processing large geojson/json files..
    // npm install --save through2 JSONStream

    var through = require('through2');
    var JSONStream = require('JSONStream');
  3. moklick revised this gist May 11, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion geojson-streaming.js
    View file
    Original file line number Diff line number Diff line change
    @@ -14,7 +14,7 @@ var startTime = +new Date();
    var geoJsonStart = '{"type": "FeatureCollection", "features":';
    var geoJsonEnd = '}';

    // here we check if current geometry is valid, inside a polygon, etc
    // here we can check if current geometry is valid, inside a polygon, etc
    var propManipulator = through.obj(function(chunk, enc, callback){
    this.push(getCleanObj(chunk));
    callback();
  4. moklick created this gist May 11, 2015.
    48 changes: 48 additions & 0 deletions geojson-streaming.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,48 @@
    // boilerplate for processing large geojson/json files..

    var through = require('through2');
    var JSONStream = require('JSONStream');
    var fs = require('fs');

    var inputStream = fs.createReadStream('./test.geojson');
    var outputStream = fs.createWriteStream('./cleaned-geojson.json');

    // for measuring the process time
    var startTime = +new Date();

    // kind of hacky ?!
    var geoJsonStart = '{"type": "FeatureCollection", "features":';
    var geoJsonEnd = '}';

    // here we check if current geometry is valid, inside a polygon, etc
    var propManipulator = through.obj(function(chunk, enc, callback){
    this.push(getCleanObj(chunk));
    callback();
    });

    outputStream.write(geoJsonStart);

    inputStream
    .pipe(JSONStream.parse('features.*'))
    .pipe(propManipulator)
    .pipe(JSONStream.stringify('[',',',']'))
    .on('end', function(){
    outputStream.write(geoJsonEnd);
    console.log('Finished in', (+new Date() - startTime) / 1000, 'seconds.');
    })
    .pipe(outputStream)


    // example helper function
    function getCleanObj(dirtyObj){
    return {
    geometry : {
    coordinates : dirtyObj.geometry.coordinates
    },
    properties: {
    gml_id: dirtyObj.properties.gml_id,
    floors: dirtyObj.properties.GESCHOSS,
    btype : dirtyObj.properties.Bezeichnun
    }
    };
    }