Skip to content

Instantly share code, notes, and snippets.

@fxtentacle

fxtentacle/agent.conf

Created October 31, 2011 20:44
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save fxtentacle/1328849 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save fxtentacle/1328849 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. fxtentacle created this gist Oct 31, 2011.
    408 changes: 408 additions & 0 deletions agent.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,408 @@

    input {
    exec {
    type => "dstat"
    command => "dstat -cdngypms --nocolor 1 0"
    interval => 13
    }

    exec {
    type => "apache-benchmark"
    command => "ab -t 5 -n 1 -c 1 -d -S 'https://spratshop-rails.spratshop.com/' | grep -E '(requests:|Time taken)'"
    interval => 17
    }

    file {
    type => "linux-syslog"
    path => [
    "/var/log/alternatives.log",
    "/var/log/auth.log",
    "/var/log/daemon.log",
    "/var/log/dpkg.log",
    "/var/log/faillog",
    "/var/log/fontconfig.log",
    "/var/log/kern.log",
    "/var/log/lastlog",
    "/var/log/lpr.log",
    "/var/log/mail.log",
    "/var/log/mysql.log",
    "/var/log/pycentral.log",
    # "/var/log/rabbitmq_logger.log",
    "/var/log/user.log",

    "/var/log/messages",
    "/var/log/syslog"
    ]


    }

    file {
    type => "apache-access"
    path => [
    "/var/log/apache2/access.log",
    "/var/log/apache2/eth0-0-NOS-1-001_spratshop_images-access.log",
    "/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-access.log",
    "/var/log/apache2/eth0-0-SSL-1-001_spratshop_images-access.log",
    "/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-access.log",
    "/var/log/apache2/other_vhosts_access.log"
    ]
    }

    file {
    type => "apache-rewrite"
    path => [
    "/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-rewrite.log",
    "/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-rewrite.log",
    "/var/log/apache2/rewrite.log"
    ]
    }

    file {
    type => "apache-error"
    path => [
    "/var/log/apache2/error.log",
    "/var/log/apache2/eth0-0-NOS-1-001_spratshop_images-error.log",
    "/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-error.log",
    "/var/log/apache2/eth0-0-SSL-1-001_spratshop_images-error.log",
    "/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-error.log"
    ]
    }

    file {
    type => "rabbitmq-post-bridge"
    path => [
    "/var/log/rabbitmq_post_bridge.log"
    ]
    }

    file {
    type => "spratshop-rails"
    path => [
    "/opt/spratshop/rails/shared/log/production.log",
    "/opt/spratshop/rails/shared/log/development.log"
    ]
    }

    file {
    type => "spratshop-file-sender"
    path => [
    "/var/log/spratshop_file_sender.log"
    ]
    }

    file {
    type => "spratshop-ftp-retry-cron"
    path => [
    "/var/log/spratshop_ftp_retry_cron.log"
    ]
    }

    file {
    type => "spratshop-status-checker"
    path => [
    "/var/log/spratshop_status_checker.log"
    ]
    }

    file {
    type => "spratshop-usage-cron"
    path => [
    "/var/log/spratshop_usage_cron.log"
    ]
    }

    file {
    type => "vsftpd"
    path => [
    "/var/log/vsftpd.log"
    ]
    }

    file {
    type => "vsftpd-to-rabbitmq"
    path => [
    "/var/log/vsftpd_to_rabbitmq.log"
    ]
    }

    file {
    type => "spratshop-image-processor"
    path => [
    "/var/log/spratshop_image_processor-1.log",
    "/var/log/spratshop_image_processor-2.log",
    "/var/log/spratshop_image_processor-3.log",
    "/var/log/spratshop_image_processor-4.log",
    "/var/log/spratshop_image_processor.log"
    ]
    }


    amqp {
    host => ...
    user => ...
    password => ...

    name => "from-vsftpd"
    key => "spratshop"
    type => "from-vsftpd"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_from-vsftpd_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }
    amqp {
    host => ...
    user => ...
    password => ...

    name => "incoming-images"
    key => "spratshop"
    type => "incoming-images"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_incoming-images_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }
    amqp {
    host => ...
    user => ...
    password => ...

    name => "processed-images"
    key => "spratshop"
    type => "processed-images"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_processed-images_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }
    amqp {
    host => ...
    user => ...
    password => ...

    name => "send-file-todo"
    key => "spratshop"
    type => "send-file-todo"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_send-file-todo_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }
    amqp {
    host => ...
    user => ...
    password => ...

    name => "sent-files"
    key => "spratshop"
    type => "sent-files"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_sent-files_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }
    amqp {
    host => ...
    user => ...
    password => ...

    name => "status-checker-to-system"
    key => "spratshop"
    type => "status-checker-to-system"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_status-checker-to-system_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }
    amqp {
    host => ...
    user => ...
    password => ...

    name => "status-system-to-checker"
    key => "spratshop"
    type => "status-system-to-checker"
    # message_format => ""

    durable => true
    exchange_type => "direct"
    format => "json"

    queue_durable => false
    queue_name => "logstash_status-system-to-checker_spratshop"

    # port => ... # number, default: 5672
    # ssl => ... # boolean
    # tags => ... # array
    #
    # verify_ssl => ... # boolean
    # vhost => ... # string, default: "/"
    }

    }



    filter {
    grok {
    type => "linux-syslog"
    pattern => ["%{SYSLOG_SUDO}", "%{SYSLOG_KERNEL}", "%{SYSLOGLINE}" ]
    }

    date {
    type => "linux-syslog"
    timestamp => "MMM dd HH:mm:ss"
    timestamp8601 => ISO8601
    }

    grok {
    type => "apache-access"
    pattern => "%{COMBINEDAPACHELOG}"
    }

    # grok {
    # type => "apache-error"
    # match {
    # "APACHE_LOG_LEVEL" => "(?:emerg|alert|crit|error|warn|notice|info|debug)"
    # "APACHE_ERROR_LOG" => "\[%{DATESTAMP_OTHER:timestamp}\] \[%{APACHE_LOG_LEVEL:level}\] %{GREEDYDATA:message}"
    # }
    # pattern => "%{APACHE_ERROR_LOG}"
    # }

    multiline {
    type => "spratshop-file-sender"
    pattern => "^\\s"
    what => "previous"
    }
    multiline {
    type => "spratshop-image-processor"
    pattern => "^\\s"
    what => "previous"
    }
    multiline {
    type => "spratshop-status-checker"
    pattern => "^\\s"
    what => "previous"
    }

    multiline {
    type => "spratshop-rails"
    pattern => "^$"
    negate => true
    what => next
    }

    grep {
    type => "spratshop-rails"
    match => [
    "@message", "^$"
    ]
    negate => true
    }


    multiline {
    type => "spratshop-ftp-retry-cron"
    pattern => "^$"
    negate => true
    what => next
    }

    }


    output {
    # Emit events to stdout for easy debugging of what is going through
    # logstash.
    # stdout { }

    # This will use elasticsearch to store your logs.
    # The 'embedded' option will cause logstash to run the elasticsearch
    # server in the same process, so you don't have to worry about
    # how to download, configure, or run elasticsearch!
    # elasticsearch {
    # cluster => "default"
    # host => "127.0.0.1"
    # bind_host => "127.0.0.1"
    #}

    amqp {
    host => ...
    user => ...
    password => ...

    exchange_type => "fanout"

    name => "rawlogs"
    durable => true
    }

    }