DavidWittman · February 20, 2016 03:57 · Feb 20, 2016 · Feb 20, 2016 · Feb 20, 2016 · Feb 20, 2016
diff --git a/logstash.conf b/logstash.conf
@@ -86,7 +86,7 @@ output {
       "es1.example.com",
       "es2.example.com",
       "es3.example.com",
-      "es4.example.com",
+      "es4.example.com"
     ]
     port => 9200
     protocol => "http"

diff --git a/000-mongostat-to-elasticsearch.md b/000-mongostat-to-elasticsearch.md
@@ -23,4 +23,6 @@ $ mongostat --json -h localhost --port 27017 -u bob -ppassword --authenticationD
 All of these configs should work fine on Logstash 2.x as well, with the exception of two things:
 
  1. The `logstash-filter-bytes2human` plugin [has not yet been updated](https://github.com/robin13/logstash-filter-bytes2human/pull/1) for Logstash 2.x support. You'll need to manually install a patched version of this plugin if you wish to run 2.x.
- 2. The syntax for the `logstash-output-elasticsearch` plugin has changed. `host` is now `hosts`, among other things.
+ 2. The syntax for the `logstash-output-elasticsearch` plugin has changed
+   1. `host` is now `hosts`
+   2. The `protocol` option has been removed in favor of a separate plugin, `logstash-output-elasticsearch_java`, for communicating over protocols other than HTTP.
diff --git a/logstash.conf b/logstash.conf
@@ -83,10 +83,12 @@ output {
     user => "bob"
     password => "password"
     host => [
-      "es1.example.com:9200",
-      "es2.example.com:9200",
-      "es3.example.com:9200",
-      "es4.example.com:9200",
+      "es1.example.com",
+      "es2.example.com",
+      "es3.example.com",
+      "es4.example.com",
     ]
+    port => 9200
+    protocol => "http"
   }
 }
diff --git a/000-mongostat-to-elasticsearch.md b/000-mongostat-to-elasticsearch.md
@@ -3,12 +3,24 @@
 ## Requirements
 
  - [Logstash 1.5.x](https://download.elastic.co/logstash/logstash/logstash-1.5.6.tar.gz)
-    - With the `logstash-output-elasticsearch` and `logstash-filter-bytes2human` plugins installed (`bin/plugin install logstash-filter-bytes2human` should do the trick).
-    - It should work fine with Logstash 2.x as well, but the `bytes2human` filter [hasn't yet been updated](https://github.com/robin13/logstash-filter-bytes2human/pull/1) to support it
+    - If you want to run Logstash 2.x, see the section titled Logstash 2.x below
  - Mongostat 2.8+ (for support of the `--json` flag)
 
+## Preparation
+
+ 1. Copy all of the files from this gist into your Logstash directory. Put `logstash.conf` and `es-template-mongostat.json` in `$LOGSTASH_DIR/conf/`.
+ 2. Edit the connection information in `conf/logstash.conf` for your Elasticsearch output
+   1. If you change the index name, you'll also need to update the template name in `es-template-mongostat.json`.
+
 ## Run
 
 ``` bash
 $ mongostat --json -h localhost --port 27017 -u bob -ppassword --authenticationDatabase admin --discover | python mongostat-filter.py | bin/logstash -f conf/logstash.conf
-```
+```
+
+## Logstash 2.x
+
+All of these configs should work fine on Logstash 2.x as well, with the exception of two things:
+
+ 1. The `logstash-filter-bytes2human` plugin [has not yet been updated](https://github.com/robin13/logstash-filter-bytes2human/pull/1) for Logstash 2.x support. You'll need to manually install a patched version of this plugin if you wish to run 2.x.
+ 2. The syntax for the `logstash-output-elasticsearch` plugin has changed. `host` is now `hosts`, among other things.
diff --git a/logstash.conf b/logstash.conf
@@ -76,14 +76,17 @@ filter {
 
 output {
 #  stdout { codec => rubydebug }
-  elasticsearch_java {
+  elasticsearch {
     index => "mongostat-%{+YYYY.MM.dd}"
     template => "conf/es-template-mongostat.json"
-    cluster => "cluster_name"
-    network_host => "localhost:9200"
-    hosts => [
-      "localhost:9200"
-    ]
     template_name => "mongostat"
+    user => "bob"
+    password => "password"
+    host => [
+      "es1.example.com:9200",
+      "es2.example.com:9200",
+      "es3.example.com:9200",
+      "es4.example.com:9200",
+    ]
   }
 }
diff --git a/000-mongostat-to-elasticsearch.md b/000-mongostat-to-elasticsearch.md
@@ -2,8 +2,9 @@
 
 ## Requirements
 
- - Logstash 2.x
+ - [Logstash 1.5.x](https://download.elastic.co/logstash/logstash/logstash-1.5.6.tar.gz)
     - With the `logstash-output-elasticsearch` and `logstash-filter-bytes2human` plugins installed (`bin/plugin install logstash-filter-bytes2human` should do the trick).
+    - It should work fine with Logstash 2.x as well, but the `bytes2human` filter [hasn't yet been updated](https://github.com/robin13/logstash-filter-bytes2human/pull/1) to support it
  - Mongostat 2.8+ (for support of the `--json` flag)
 
 ## Run

diff --git a/000-mongostat-to-elasticsearch.md b/000-mongostat-to-elasticsearch.md
@@ -1,5 +1,13 @@
 # Piping data from Mongostat to Elasticsearch with Logstash
 
+## Requirements
+
+ - Logstash 2.x
+    - With the `logstash-output-elasticsearch` and `logstash-filter-bytes2human` plugins installed (`bin/plugin install logstash-filter-bytes2human` should do the trick).
+ - Mongostat 2.8+ (for support of the `--json` flag)
+
+## Run
+
 ``` bash
 $ mongostat --json -h localhost --port 27017 -u bob -ppassword --authenticationDatabase admin --discover | python mongostat-filter.py | bin/logstash -f conf/logstash.conf
 ```
diff --git a/logstash.conf b/logstash.conf
@@ -6,21 +6,76 @@ input {
 }
 
 filter {
+  grok {
+    match => {
+      "qr|qw" => "%{INT:qr:int}\|%{INT:qw:int}"
+    }
+    remove_field => [ "qr|qw" ]
+  }
+  grok {
+    match => {
+      "ar|aw" => "%{INT:ar:int}\|%{INT:aw:int}"
+    }
+    remove_field => [ "ar|aw" ]
+  }
+  grok {
+    match => {
+      "command" => "%{INT:commands_local:int}\|%{INT:commands_replicated:int}"
+    }
+    remove_field => [ "command" ]
+  }
+  grok {
+    match => {
+      "host" => "%{HOSTNAME:hostname}:%{POSINT:port:int}"
+    }
+  }
+
+  mutate { remove_field => [ "time" ] }
+
+  # Append 'ib' to all the byte values for bytes2human
+  mutate {
+    gsub => [
+      "netOut", "(?<!b)$", "ib",
+      "netIn", "(?<!b)$", "ib",
+      "vsize", "(?<!b)$", "ib",
+      "res", "(?<!b)$", "ib",
+      "mapped", "(?<!b)$", "ib",
+      "non-mapped", "(?<!b)$", "ib"
+    ]
+  }
+
+  # logstash-filter-bytes2human is required for these conversions
+  bytes2human {
+    convert => [
+      "netOut", "bytes",
+      "netIn", "bytes",
+      "vsize", "bytes",
+      "res", "bytes"
+    ]
+  }
+
   mutate {
     convert => {
       "conn" => "integer"
       "insert" => "integer"
       "update" => "integer"
+      "delete" => "integer"
       "query" => "integer"
       "flushes" => "integer"
       "faults" => "integer"
       "getmore" => "integer"
+      "netIn" => "integer"
+      "netOut" => "integer"
+      "mapped" => "integer"
+      "non-mapped" => "integer"
+      "vsize" => "integer"
+      "res" => "integer"
     }
   }
 }
 
 output {
-
+#  stdout { codec => rubydebug }
   elasticsearch_java {
     index => "mongostat-%{+YYYY.MM.dd}"
     template => "conf/es-template-mongostat.json"

diff --git a/mongostat-to-elasticsearch.md → 000-mongostat-to-elasticsearch.md b/mongostat-to-elasticsearch.md → 000-mongostat-to-elasticsearch.md
diff --git a/es-template-mongostat.json b/es-template-mongostat.json
@@ -0,0 +1,95 @@
+{
+  "template" : "mongostat-*",
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "_all" : {"enabled" : true, "omit_norms" : true},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" }
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" },
+            "fields" : {
+              "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
+            }
+          }
+        }
+      }, {
+        "float_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "float",
+          "mapping" : { "type" : "float", "doc_values" : true }
+        }
+      }, {
+        "double_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "double",
+          "mapping" : { "type" : "double", "doc_values" : true }
+        }
+      }, {
+        "byte_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "byte",
+          "mapping" : { "type" : "byte", "doc_values" : true }
+        }
+      }, {
+        "short_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "short",
+          "mapping" : { "type" : "short", "doc_values" : true }
+        }
+      }, {
+        "integer_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "integer",
+          "mapping" : { "type" : "integer", "doc_values" : true }
+        }
+      }, {
+        "long_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "long",
+          "mapping" : { "type" : "long", "doc_values" : true }
+        }
+      }, {
+        "date_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "date",
+          "mapping" : { "type" : "date", "doc_values" : true }
+        }
+      }, {
+        "geo_point_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "geo_point",
+          "mapping" : { "type" : "geo_point", "doc_values" : true }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date", "doc_values" : true },
+        "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
+        "geoip"  : {
+          "type" : "object",
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip", "doc_values" : true },
+            "location" : { "type" : "geo_point", "doc_values" : true },
+            "latitude" : { "type" : "float", "doc_values" : true },
+            "longitude" : { "type" : "float", "doc_values" : true }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/logstash.conf b/logstash.conf
@@ -0,0 +1,34 @@
+input {
+  stdin {
+    codec => json
+    tags => ["mongostat"]
+  }
+}
+
+filter {
+  mutate {
+    convert => {
+      "conn" => "integer"
+      "insert" => "integer"
+      "update" => "integer"
+      "query" => "integer"
+      "flushes" => "integer"
+      "faults" => "integer"
+      "getmore" => "integer"
+    }
+  }
+}
+
+output {
+
+  elasticsearch_java {
+    index => "mongostat-%{+YYYY.MM.dd}"
+    template => "conf/es-template-mongostat.json"
+    cluster => "cluster_name"
+    network_host => "localhost:9200"
+    hosts => [
+      "localhost:9200"
+    ]
+    template_name => "mongostat"
+  }
+}
diff --git a/mongostat-filter.py b/mongostat-filter.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import json
+import sys
+
+while True:
+    try:
+        data = json.loads(sys.stdin.readline())
+        for metric in data.itervalues():
+            print json.dumps(metric)
+    except Exception as e:
+        sys.stderr.write("Error: %s" % e)
+        sys.stderr.flush()
diff --git a/mongostat-to-elasticsearch.md b/mongostat-to-elasticsearch.md
@@ -0,0 +1,5 @@
+# Piping data from Mongostat to Elasticsearch with Logstash
+
+``` bash
+$ mongostat --json -h localhost --port 27017 -u bob -ppassword --authenticationDatabase admin --discover | python mongostat-filter.py | bin/logstash -f conf/logstash.conf
+```