Skip to content

Instantly share code, notes, and snippets.

@teckl

teckl/ingress-crawl-portal-level.js

Last active November 6, 2015 20:05
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save teckl/8eff6b59ee24ffb1f615 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save teckl/8eff6b59ee24ffb1f615 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. teckl revised this gist Oct 23, 2015. 1 changed file with 125 additions and 0 deletions.
    125 changes: 125 additions & 0 deletions td-agent.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,125 @@
    <source>
    type tail
    format ltsv
    # time_format %d/%b/%Y:%H:%M:%S %z
    path /home/hoge/portal_detail.ltsv
    pos_file /var/log/td-agent/portal_detail.ltsv.pos
    tag data.portal.detail
    refresh_interval 3m
    </source>
    <match debug.**>
    type stdout
    </match>

    <match data.portal.detail>
    type copy
    <store>
    type stdout
    </store>
    <store>
    type datacounter
    count_interval 3m
    count_key level
    pattern1 p8 8
    pattern2 p7 7
    pattern3 p6 6
    pattern4 p5 5
    outcast_unmatched yes
    tag datacount.portal.level
    </store>
    <store>
    type datacounter
    count_interval 3m
    count_key team
    pattern1 Resistance R
    pattern2 Enlightened E
    outcast_unmatched yes
    tag datacount.portal.team
    </store>
    </match>


    <match datacount.portal.level>
    type copy
    <store>
    type stdout
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p8.count","value" => record["data.portal.detail_p8_count"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p7.count","value" => record["data.portal.detail_p7_count"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p6.count","value" => record["data.portal.detail_p6_count"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p8.rate","value" => record["data.portal.detail_p8_rate"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p7.rate","value" => record["data.portal.detail_p7_rate"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p6.rate","value" => record["data.portal.detail_p6_rate"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p8.percentage","value" => record["data.portal.detail_p8_percentage"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p7.percentage","value" => record["data.portal.detail_p7_percentage"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.p6.percentage","value" => record["data.portal.detail_p6_percentage"].to_i}]
    </store>
    </match>

    <match datacount.portal.team>
    type copy
    <store>
    type stdout
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Resistance.count","value" => record["data.portal.detail_Resistance_count"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Enlightened.count","value" => record["data.portal.detail_Enlightened_count"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Resistance.rate","value" => record["data.portal.detail_Resistance_rate"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Enlightened.rate","value" => record["data.portal.detail_Enlightened_rate"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Resistance.percentage","value" => record["data.portal.detail_Resistance_percentage"].to_i}]
    </store>
    <store>
    type map
    map ["stat." + tag, time, {"metric" => "stat.ingress.portal.Enlightened.percentage","value" => record["data.portal.detail_Enlightened_percentage"].to_i}]
    </store>
    </match>

    <match stat.datacount.portal.**>
    type copy
    <store>
    type stdout
    </store>
    <store>
    type dd
    dd_api_key your_datadoc_api_key
    </store>
    </match>
  2. teckl created this gist Oct 23, 2015.
    459 changes: 459 additions & 0 deletions ingress-crawl-portal-level.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,459 @@
    /**
    * @file Ingress portal level crawler with PhantomJS
    * @author teckl (https://github.com/teckl)
    * @version 0.0.1
    * @license MIT
    * @see {@link https://github.com/nibogd/tg-ingress-scorebot|GitHub }
    * @see {@link https://github.com/jonatkins/ingress-intel-total-conversion|GitHub }
    */

    "use strict";
    //Initialize

    var fs = require('fs');
    var system = require('system');

    //var args = system.args;
    var cookiespath = 'iced_cookies';
    var cookieSACSID, cookieCSRF;

    var settingsfile = fs.open('bot.json', 'r');
    var settings = JSON.parse(settingsfile.read());
    settingsfile.close();
    var l = settings['gmail'];
    var p = settings['password'];
    var area = settings['link'];
    var my_team = settings['my_team'];
    var v = 10000;

    /*global phantom */
    /*global idleReset */

    /**
    * Counter for number of screenshots
    */
    var version = '0.0.1';

    /**
    * Delay between logging in and checking if successful
    * @default
    */
    var loginTimeout = 10 * 1000;

    /**
    * twostep auth trigger
    */
    var twostep = 0;
    var page = require('webpage').create();
    // page.onConsoleMessage = function () {};
    // page.onError = function () {};

    /** @function setVieportSize */

    page.viewportSize = {
    width: 42,
    height: 42
    }

    var resourceCounter = 0;
    page.onResourceRequested = function (requestData, networkRequest) {
    console.log(resourceCounter++ + '.\t', requestData.url);
    };

    page.onResourceReceived = function(response) {
    if (/ingress.com\/r\/getPlexts/.test(response.url)) {
    console.log('Response (#' + response.id + ', stage "' + response.stage + '"): ' + JSON.stringify(response));
    }
    };

    page.onResourceError = function (resourceError) {
    system.stderr.writeLine('Unable to load resource (#' + resourceError.id + 'URL:' + resourceError.url + ')');
    system.stderr.writeLine('Error code: ' + resourceError.errorCode + '. Description: ' + resourceError.errorString);
    };


    function captureAjaxResponsesToConsole() {
    // logs ajax response contents to console so sublime's onConsoleMessage can use the contents
    // credit to Ionuț G. Stan
    // http://stackoverflow.com/questions/629671/how-can-i-intercept-xmlhttprequests-from-a-greasemonkey-script
    page.evaluate(function() {
    (function(open) {
    XMLHttpRequest.prototype.open = function(method, url, async, user, pass) {
    this.addEventListener("readystatechange", function() {
    if (this.readyState === 4) {
    var res = {'response':this.responseText, 'url' : url};
    console.log(JSON.stringify(res));
    }
    }, false);
    open.call(this, method, url, async, user, pass);
    };
    })(XMLHttpRequest.prototype.open);
    return 1;
    });
    }

    window.parsePortalEntities = function(data) {
    var entities = JSON.parse(data);
    var m = entities.result.map;
    var portal_count = 0, team_count = 0, p8_count = 0;
    for (var id in m) {
    var val = m[id];
    var gameEntities = val.gameEntities;
    for (var i in gameEntities) {
    var ent = gameEntities[i]
    if (ent[2][0] == 'p') {
    var level = parseInt(ent[2][4])||0;
    var team = ent[2][1];
    var health = ent[2][5];
    var resCount = ent[2][6];
    var image = ent[2][7];
    var title = ent[2][8];
    var timestamp = ent[2][13];
    announce('level : ' + level + ' team : ' + team + ' title : ' + title);

    var detail_data = 'time:' + getDateTime(2) + '\tmetric:portal.detail' + '\tteam:' + team + '\tresCount:' + resCount + '\tlevel:' + level + '\ttitle:' + title + '\n';
    announce(detail_data);
    fs.write('portal_detail.ltsv', detail_data, 'a');

    portal_count++;
    if (team == my_team) {
    team_count++;
    if (level >= 8) {
    p8_count++;
    }
    }
    }
    }
    }

    var portal_level_percent = Math.round(p8_count / portal_count * 100);
    var team_percent = Math.round(team_count / portal_count * 100);
    console.log('portal_count : ' + portal_count + ' p8_count : ' + p8_count + ' portal_level_percent : ' + portal_level_percent + ' team_percent : ' + team_percent);
    var data = {
    date : getDateTime(2),
    portal_count : portal_count,
    team_count : team_count,
    team_percent : team_percent,
    p8_count : p8_count,
    p8_percent : portal_level_percent,
    };
    fs.write('portal_summary.json', JSON.stringify(data, null, ' '), 'a');

    }

    page.onConsoleMessage = function (msg) {
    var res;
    try {
    res = JSON.parse(msg);
    console.log('URL:' + res.url);
    if (/\/r\/getEntities/.test(res.url)) {
    parsePortalEntities(res.response);
    }
    if (/\/r\/getPlexts/.test(res.url)) {
    console.log('getPlexts: ' + res.response);
    }
    } catch (e) {
    console.log(msg);
    console.log(e);
    }
    };



    //Functions

    /**
    * console.log() wrapper
    * @param {String} str - what to announce
    */
    function announce(str) {
    console.log(getDateTime(0) + ': ' + str);
    }

    /**
    * Returns Date and time
    * @param {number} format - the format of output, 0 for DD.MM.YYY HH:MM:SS, 1 for YYYY-MM-DD--HH-MM-SS (for filenames)
    * @returns {String} date
    */
    function getDateTime(format) {
    var now = new Date();
    var year = now.getFullYear();
    var month = now.getMonth()+1;
    var day = now.getDate();
    var hour = now.getHours();
    var minute = now.getMinutes();
    var second = now.getSeconds();
    if(month.toString().length === 1) {
    month = '0' + month;
    }
    if(day.toString().length === 1) {
    day = '0' + day;
    }
    if(hour.toString().length === 1) {
    hour = '0' + hour;
    }
    if(minute.toString().length === 1) {
    minute = '0' + minute;
    }
    if(second.toString().length === 1) {
    second = '0' + second;
    }
    var dateTime;
    if (format === 1) {
    dateTime = year + '-' + month + '-' + day + '--' + hour + '-' + minute + '-' + second;
    } else if (format === 2) {
    dateTime = year + '-' + month + '-' + day + 'T' + hour + ':' + minute + ':' + second;
    } else {
    dateTime = day + '.' + month + '.' + year + ' '+hour+':'+minute+':'+second;
    }
    return dateTime;
    }


    /**
    * Quit if an error occured
    * @param {String} err - the error text
    */
    function quit(err) {
    if (err) {
    announce('crawler crashed. Reason: ' + err + ' T_T');
    } else {
    announce('Quit');
    }
    phantom.exit();
    }


    /**
    * Log in to google. Doesn't use post, because URI may change.
    * Fixed in 3.0.0 -- obsolete versions will not work (google changed login form)
    * @param l - google login
    * @param p - google password
    */
    function login(l, p) {
    page.evaluate(function (l) {
    document.getElementById('Email').value = l;
    }, l);
    page.evaluate(function () {
    document.querySelector("#next").click();
    });
    window.setInterval(function () {
    page.evaluate(function (p) {
    document.getElementById('Passwd').value = p;
    }, p);
    page.evaluate(function () {
    document.querySelector("#next").click();
    });
    page.evaluate(function () {
    document.getElementById('gaia_loginform').submit();
    });
    }, loginTimeout / 10);
    }

    /**
    * Check if logged in successfully, quit if failed, accept appEngine request if needed and prompt for two step code if needed.
    */
    function checkLogin() {

    announce('URI is now ' + page.url.substring(0,40) + '...');

    if (page.url.substring(0,40) === 'https://accounts.google.com/ServiceLogin') {quit('login failed: wrong email and/or password');}
    if (page.url.substring(0,40) === 'https://appengine.google.com/_ah/loginfo') {
    announce('Accepting appEngine request...');
    page.evaluate(function () {
    document.getElementById('persist_checkbox').checked = true;
    document.getElementsByTagName('form').submit();
    });
    }

    if (page.url.substring(0,40) === 'https://accounts.google.com/SecondFactor') {
    announce('Using two-step verification, please enter your code:');
    twostep = system.stdin.readLine();
    }

    if (twostep) {
    page.evaluate(function (code) {
    document.getElementById('smsUserPin').value = code;
    }, twostep);
    page.evaluate(function () {
    document.getElementById('gaia_secondfactorform').submit();
    });
    }
    }

    /**
    * Does all stuff needed after login/password authentication
    * @since 3.1.0
    */
    function afterPlainLogin() {
    window.setTimeout(function () {
    announce('Verifying login...');
    checkLogin();
    announce('before_storeCookies...');
    window.setTimeout(function () {
    page.open(area, function () {
    storeCookies();
    setTimeout(function () {
    setTimeout(main, v);
    }, loginTimeout);
    });
    }, loginTimeout);
    }, loginTimeout);
    }

    /**
    * Checks if user is signed in by looking for the "Sign in" button
    * @returns {boolean}
    * @author mfcanovas (github.com/mfcanovas)
    * @since 3.2.0
    */
    function isSignedIn() {
    return page.evaluate(function() {
    var btns = document.getElementsByClassName('button_link');
    for(var i = 0; i<btns.length;i++) {
    if(btns[i].innerText.trim() === 'Sign in') return false;
    }
    return true;
    });

    }

    /**
    * Main function. Wrapper for others.
    */
    function main() {
    var data = page.evaluate(function() {return document.getElementById('rs_score_history_scores')});
    window.setTimeout(function() {
    announce('Crawler finished');
    fs.write('crawled.ice', data.innerHTML, 'w');
    phantom.exit();
    }, 1000);
    }

    /**
    * Checks if cookies file exists. If so, it sets SACSID and CSRF vars
    * @returns {boolean}
    * @author mfcanovas (github.com/mfcanovas)
    * @since 3.2.0
    */
    function cookiesFileExists() {
    if(fs.exists(cookiespath)) {
    var stream = fs.open(cookiespath, 'r');

    while(!stream.atEnd()) {
    var line = stream.readLine();
    var res = line.split('=');
    if(res[0] === 'SACSID') {
    cookieSACSID = res[1];
    } else if(res[0] === 'csrftoken') {
    cookieCSRF = res[1];
    }
    }
    stream.close();
    return true;
    } else {
    return false;
    }
    }

    /**
    * Remove cookies file if exists
    * @author mfcanovas (github.com/mfcanovas)
    * @since 3.2.0
    */
    function removeCookieFile() {
    if(fs.exists(cookiespath)) {
    fs.remove(cookiespath);
    }
    }

    function storeCookies() {
    var cookies = page.cookies;
    announce('storeCookies...');
    fs.write(cookiespath, '', 'w');
    for(var i in cookies) {
    announce('Cookies_write...');
    fs.write(cookiespath, cookies[i].name + '=' + cookies[i].value +'\n', 'a');
    }
    }

    /**
    * Fires plain login
    */
    function firePlainLogin() {
    cookieSACSID = '';
    cookieCSRF = '';
    page.open('https://www.ingress.com/intel', function (status) {

    if (status !== 'success') {quit('cannot connect to remote server');}

    var link = page.evaluate(function () {
    return document.getElementsByTagName('a')[0].href;
    });

    announce('Logging in...');
    page.open(link, function () {
    login(l, p);
    afterPlainLogin();
    });
    });
    }

    //MAIN SCRIPT

    announce('Crawler loaded')
    if (!cookiesFileExists()) {
    firePlainLogin();
    } else {
    announce('Using stored cookie');
    addCookies(cookieSACSID, cookieCSRF);
    afterCookieLogin();
    }


    /**
    * Log in using cookies
    * @param {String} sacsid
    * @param {String} csrf
    * @since 3.1.0
    */
    function addCookies(sacsid, csrf) {
    phantom.addCookie({
    name: 'SACSID',
    value: sacsid,
    domain: 'www.ingress.com',
    path: '/',
    httponly: true,
    secure: true
    });
    phantom.addCookie({
    name: 'csrftoken',
    value: csrf,
    domain: 'www.ingress.com',
    path: '/'
    });
    }


    /**
    * Does all stuff needed after cookie authentication
    * @since 3.1.0
    */
    function afterCookieLogin() {
    page.open(area, function (status) {
    if (status === 'success') {
    captureAjaxResponsesToConsole();
    }

    if(!isSignedIn()) {
    removeCookieFile();
    if(l && p) {
    firePlainLogin();
    return;
    } else {
    quit('User not logged in');
    }
    }
    setTimeout(main, loginTimeout);
    });
    }