Skip to content

Instantly share code, notes, and snippets.

@fabiodr

fabiodr/example.js

Forked from esjewett/example.js
Created March 6, 2016 03:58
Show Gist options
  • Save fabiodr/796898439cfbca14286c to your computer and use it in GitHub Desktop.
Save fabiodr/796898439cfbca14286c to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @esjewett esjewett revised this gist Jan 30, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion example.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    // This example data set (not provided) is over 100,000 records. The following code calculates 30-day moving
    // averages (over 3 million aggregations) using Crossfilter and the Reductio helper library. It takes about
    // 3 seconds for the initial aggregation in Chrome on a 2.3 GHz Core i7, mid-2012 rMBP.
    // 3 seconds for the initial aggregation in Chrome Canary (42.0.2291.0) on a 2.3 GHz Core i7, mid-2012 rMBP.

    d3.csv('dataJan-29-2015.csv', function (data) {

  2. @esjewett esjewett revised this gist Jan 30, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion example.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    // This example data set (not provided) is over 100,000 records. The following code calculates 30-day moving
    // averages (over 3 million aggregations) using Crossfilter and the Reductio helper library. It takes about
    // 3 seconds for the initial aggregation on a 2.3 GHz Core i7, mid-2012 rMBP.
    // 3 seconds for the initial aggregation in Chrome on a 2.3 GHz Core i7, mid-2012 rMBP.

    d3.csv('dataJan-29-2015.csv', function (data) {

  3. @esjewett esjewett revised this gist Jan 30, 2015. 1 changed file with 4 additions and 0 deletions.
    4 changes: 4 additions & 0 deletions example.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,7 @@
    // This example data set (not provided) is over 100,000 records. The following code calculates 30-day moving
    // averages (over 3 million aggregations) using Crossfilter and the Reductio helper library. It takes about
    // 3 seconds for the initial aggregation on a 2.3 GHz Core i7, mid-2012 rMBP.

    d3.csv('dataJan-29-2015.csv', function (data) {

    //convert the iso timestamps to JS Dates
  4. @esjewett esjewett renamed this gist Jan 30, 2015. 1 changed file with 0 additions and 0 deletions.
    0 gistfile1.js → example.js
    View file
    File renamed without changes.
  5. @esjewett esjewett revised this gist Jan 30, 2015. No changes.
  6. @esjewett esjewett revised this gist Jan 30, 2015. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions gistfile1.js
    View file
    Original file line number Diff line number Diff line change
    @@ -29,9 +29,9 @@ d3.csv('dataJan-29-2015.csv', function (data) {
    arr = [];
    // If the relevant field isn't populated, don't bother looping.
    if(record.ShipDate) {
    // Manually parse dates - don't use the d3 date formatters. They are up to 10x slower.
    tempDate = new Date(+record.ShipDate.slice(0,4), +record.ShipDate.slice(5,7)-1, +record.ShipDate.slice(8,10));
    for(i=0; i<30; i++) {
    // Manually parse dates - don't use the d3 date formatters. They are up to 10x slower.
    tempDate = new Date(+record.ShipDate.slice(0,4), +record.ShipDate.slice(5,7)-1, +record.ShipDate.slice(8,10));
    // Javascript Date wraps at month- and year-end (Dec 32 is Jan 1 of the following year)
    tempDate.setDate(tempDate.getDate() + i);
    // Again, don't use the date formatters. Be careful to get the right number of digits in months & days.
  7. @esjewett esjewett revised this gist Jan 30, 2015. 1 changed file with 6 additions and 0 deletions.
    6 changes: 6 additions & 0 deletions gistfile1.js
    View file
    Original file line number Diff line number Diff line change
    @@ -15,20 +15,26 @@ d3.csv('dataJan-29-2015.csv', function (data) {

    var cs_data = crossfilter(data);

    // Always force dimension values into the right data type.
    var ShipDateDim = cs_data.dimension(function(d) {return "" + d.ShipDate;});
    var countryDim = cs_data.dimension(function (d) {return "" + d.ShipToCountryCode;});
    var country_total = countryDim.group().reduceSum(dc.pluck('DollarValue'));

    // Declare variables used in the group function *outside* of the function.
    var arr = [];
    var tempDate = null;
    var i;
    var tempStr = "";
    var groupFunction = function(record) {
    arr = [];
    // If the relevant field isn't populated, don't bother looping.
    if(record.ShipDate) {
    for(i=0; i<30; i++) {
    // Manually parse dates - don't use the d3 date formatters. They are up to 10x slower.
    tempDate = new Date(+record.ShipDate.slice(0,4), +record.ShipDate.slice(5,7)-1, +record.ShipDate.slice(8,10));
    // Javascript Date wraps at month- and year-end (Dec 32 is Jan 1 of the following year)
    tempDate.setDate(tempDate.getDate() + i);
    // Again, don't use the date formatters. Be careful to get the right number of digits in months & days.
    tempStr = tempDate.getFullYear() + '-' +
    (tempDate.getMonth()<9 ? ('0' + (tempDate.getMonth()+1)) : (tempDate.getMonth()+1)) +
    '-' +
  8. @esjewett esjewett created this gist Jan 30, 2015.
    68 changes: 68 additions & 0 deletions gistfile1.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,68 @@
    d3.csv('dataJan-29-2015.csv', function (data) {

    //convert the iso timestamps to JS Dates
    var parseDate = d3.time.format("%Y-%m-%dT%H:%M:%S").parse;
    var ymd = d3.time.format("%Y-%m-%d");
    data.forEach(function(d) {
    d.DollarValue = parseFloat(d.DollarValue);
    d.ShipDate = ymd(parseDate(d.ShipDate.split("+")[0]));

    // fix obsolete or missing salesperson names
    if (d.SalespersonName.indexOf('do not use') > -1 || d.SalespersonName.indexOf('DO NOT USE') > -1 || d.SalespersonName === "") {
    d.SalespersonName = d.SalespersonNo;
    }
    });

    var cs_data = crossfilter(data);

    var ShipDateDim = cs_data.dimension(function(d) {return "" + d.ShipDate;});
    var countryDim = cs_data.dimension(function (d) {return "" + d.ShipToCountryCode;});
    var country_total = countryDim.group().reduceSum(dc.pluck('DollarValue'));

    var arr = [];
    var tempDate = null;
    var i;
    var tempStr = "";
    var groupFunction = function(record) {
    arr = [];
    if(record.ShipDate) {
    for(i=0; i<30; i++) {
    tempDate = new Date(+record.ShipDate.slice(0,4), +record.ShipDate.slice(5,7)-1, +record.ShipDate.slice(8,10));
    tempDate.setDate(tempDate.getDate() + i);
    tempStr = tempDate.getFullYear() + '-' +
    (tempDate.getMonth()<9 ? ('0' + (tempDate.getMonth()+1)) : (tempDate.getMonth()+1)) +
    '-' +
    (tempDate.getDate()<10 ? ('0' + (tempDate.getDate())) : (tempDate.getDate()));
    arr.push(tempStr);
    }
    }
    return arr;
    };

    var reducer = reductio().groupAll(groupFunction).count(true).sum(function(d) { return +d.DollarValue; }).avg(true);

    console.time("Total Crossfilter calculation");
    console.time("Build initial groupAll");
    var shipments_moving_avg = ShipDateDim.groupAll();
    console.timeEnd("Build initial groupAll");
    console.time("Apply reducers");
    reducer(shipments_moving_avg);
    console.timeEnd("Apply reducers");

    console.time("groupAll.value()");
    shipments_moving_avg.value();
    console.timeEnd("groupAll.value()");

    console.time("groupAll.value() second time");
    shipments_moving_avg.value();
    console.timeEnd("groupAll.value() second time");

    console.time("filter on country");
    countryDim.filter("Canada");
    console.timeEnd("filter on country");

    console.time("groupAll.value() after filter");
    shipments_moving_avg.value();
    console.timeEnd("groupAll.value() after filter");
    console.timeEnd("Total Crossfilter calculation");
    });