Skip to content

Instantly share code, notes, and snippets.

@mister2d

mister2d/loki_nomad_consul_connect.hcl

Last active February 12, 2021 14:47
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save mister2d/d2fcb12d0abf8f88cc0ed9be61ea84cf to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save mister2d/d2fcb12d0abf8f88cc0ed9be61ea84cf to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. mister2d revised this gist Feb 1, 2021. 1 changed file with 49 additions and 57 deletions.
    106 changes: 49 additions & 57 deletions loki_nomad_consul_connect.hcl
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,7 @@
    locals {
    config = jsondecode(file("runnervars.json"))
    }

    job "logging" {
    datacenters = ["dc1"]
    type = "service"
    @@ -17,20 +21,25 @@ job "logging" {
    static = 1514
    to = 1514
    }
    port "loki_grpc" { static = 3101 }
    port "loki_http" { static = 3100 }
    port "promtail_grpc" { static = 9081 }
    port "promtail_http" { static = 9080 }
    port "promtail_syslog" { static = 2514 }
    }

    service {
    name = "loki"
    port = "3100"
    port = "loki_http"
    task = "loki"
    address_mode = "alloc"

    check {
    name = "healthcheck"
    type = "http"
    path = "/ready"
    interval = "10s"
    timeout = "2s"
    name = "healthcheck"
    type = "http"
    path = "/ready"
    interval = "10s"
    timeout = "2s"
    address_mode = "alloc"
    }

    @@ -41,17 +50,17 @@ job "logging" {

    service {
    name = "loki-promtail-syslog"
    port = "2514"
    port = "promtail_syslog"
    task = "loki-promtail"
    address_mode = "alloc"

    check {
    name = "healthcheck"
    type = "http"
    port = "9080"
    path = "/ready"
    interval = "10s"
    timeout = "2s"
    name = "healthcheck"
    type = "http"
    port = "promtail_http"
    path = "/ready"
    interval = "10s"
    timeout = "2s"
    address_mode = "alloc"
    }
    }
    @@ -63,32 +72,31 @@ job "logging" {
    address_mode = "host"

    check {
    name = "healthcheck"
    type = "tcp"
    port = "syslog"
    interval = "10s"
    timeout = "2s"
    name = "healthcheck"
    type = "tcp"
    port = "syslog"
    interval = "10s"
    timeout = "2s"
    }
    }

    task "loki" {
    driver = "docker"

    resources {
    cpu = 100
    cpu = 500
    memory = 256
    }

    config {
    image = "grafana/loki:2.1.0"
    image = "grafana/loki:${local.config.LOKI_VERSION}"

    args = [
    "-config.expand-env=true",
    "-config.file=/local/config/local-config.yaml",
    "-server.http-listen-address=0.0.0.0",
    "-server.http-listen-port=3100",
    "-server.http-listen-port=${NOMAD_PORT_loki_http}",
    "-server.grpc-listen-address=0.0.0.0",
    "-server.grpc-listen-port=3101"
    "-server.grpc-listen-port=${NOMAD_PORT_loki_grpc}"
    ]

    mounts = [
    @@ -103,7 +111,7 @@ job "logging" {
    options = [
    {
    device=":/srv/pool/data/loki/data"
    o="addr=nfs.server.local,rw,vers=4.2,rsize=1048576,wsize=1048576,hard,noatime,lookupcache=positive,timeo=600,retrans=2"
    o="addr=nfs.local,rw,vers=4.2,rsize=1048576,wsize=1048576,hard,noatime,lookupcache=positive,timeo=600,retrans=2"
    type="nfs"
    }
    ]
    @@ -161,16 +169,19 @@ EOH
    driver = "docker"

    resources {
    cpu = 100
    cpu = 200
    memory = 256
    }

    config {
    image = "grafana/promtail:2.1.0"
    image = "grafana/promtail:${local.config.PROMTAIL_VERSION}"

    args = [
    "-config.expand-env=true",
    "-config.file=/local/config/config.yml"
    "-config.file=/local/config/config.yml",
    "-server.http-listen-address=0.0.0.0",
    "-server.http-listen-port=${NOMAD_PORT_promtail_http}",
    "-server.grpc-listen-address=0.0.0.0",
    "-server.grpc-listen-port=${NOMAD_PORT_promtail_grpc}"
    ]
    }

    @@ -181,20 +192,16 @@ EOH

    template {
    data = <<EOH
    server:
    http_listen_port: 9080
    grpc_listen_port: 0
    positions:
    filename: /tmp/positions.yaml
    clients:
    - url: http://localhost:3100/loki/api/v1/push
    - url: http://localhost:{{ env "NOMAD_PORT_loki_http" }}/loki/api/v1/push
    scrape_configs:
    - job_name: syslog
    syslog:
    listen_address: 0.0.0.0:2514
    listen_address: 0.0.0.0:{{ env "NOMAD_PORT_promtail_syslog" }}
    idle_timeout: 60s
    label_structured_data: yes
    labels:
    @@ -218,15 +225,12 @@ EOH
    }

    config {
    image = "balabit/syslog-ng:3.30.1"
    image = "registry.ingress.service.consul/local/syslog-ng:${local.config.SYSLOGNG_VERSION}"

    args = [
    "--no-caps"
    ]

    volumes = [
    "local/config/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf"
    ]
    auth {
    username = "${local.config.REGISTRY_USER}"
    password = "${local.config.REGISTRY_PASSWORD}"
    }
    }

    lifecycle {
    @@ -236,30 +240,18 @@ EOH

    template {
    data = <<EOH
    @version: 3.30
    @include "scl.conf"
    source s_tcp {
    tcp(ip("0.0.0.0")
    port("1514")
    keep_hostname(yes)
    use_dns(yes)
    use_fqdn(yes)
    );
    };
    destination d_loki_promtail {
    syslog("localhost" transport("tcp") port(2514));
    syslog("localhost" transport("tcp") port({{ env "NOMAD_PORT_promtail_syslog" }}));
    };
    log {
    source(s_tcp);
    source(s_docker);
    destination(d_loki_promtail);
    };
    EOH
    change_mode = "signal"
    change_signal = "SIGHUP"
    destination = "local/config/syslog-ng.conf"
    destination = "local/conf.d/loki.conf"
    }
    }
    }
  2. mister2d created this gist Jan 29, 2021.
    266 changes: 266 additions & 0 deletions loki_nomad_consul_connect.hcl
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,266 @@
    job "logging" {
    datacenters = ["dc1"]
    type = "service"

    constraint {
    attribute = "${attr.cpu.arch}"
    value = "amd64"
    }

    group "loki" {
    count = 1

    network {
    mode = "bridge"

    port "syslog" {
    static = 1514
    to = 1514
    }
    }

    service {
    name = "loki"
    port = "3100"
    task = "loki"
    address_mode = "alloc"

    check {
    name = "healthcheck"
    type = "http"
    path = "/ready"
    interval = "10s"
    timeout = "2s"
    address_mode = "alloc"
    }

    connect {
    sidecar_service {}
    }
    }

    service {
    name = "loki-promtail-syslog"
    port = "2514"
    task = "loki-promtail"
    address_mode = "alloc"

    check {
    name = "healthcheck"
    type = "http"
    port = "9080"
    path = "/ready"
    interval = "10s"
    timeout = "2s"
    address_mode = "alloc"
    }
    }

    service {
    name = "syslog"
    port = "syslog"
    task = "syslog"
    address_mode = "host"

    check {
    name = "healthcheck"
    type = "tcp"
    port = "syslog"
    interval = "10s"
    timeout = "2s"
    }
    }

    task "loki" {
    driver = "docker"

    resources {
    cpu = 100
    memory = 256
    }

    config {
    image = "grafana/loki:2.1.0"

    args = [
    "-config.expand-env=true",
    "-config.file=/local/config/local-config.yaml",
    "-server.http-listen-address=0.0.0.0",
    "-server.http-listen-port=3100",
    "-server.grpc-listen-address=0.0.0.0",
    "-server.grpc-listen-port=3101"
    ]

    mounts = [
    {
    type = "volume"
    target = "/srv/loki"
    source = "loki_data"
    readonly = false
    volume_options = {
    driver_config = {
    name = "local"
    options = [
    {
    device=":/srv/pool/data/loki/data"
    o="addr=nfs.server.local,rw,vers=4.2,rsize=1048576,wsize=1048576,hard,noatime,lookupcache=positive,timeo=600,retrans=2"
    type="nfs"
    }
    ]
    }
    }
    }
    ]
    }

    template {
    data = <<EOH
    ---
    auth_enabled: false
    ingester:
    lifecycler:
    ring:
    kvstore:
    store: inmemory
    replication_factor: 1
    final_sleep: 0s
    chunk_idle_period: 5m
    chunk_retain_period: 30s
    schema_config:
    configs:
    - from: 2020-10-24
    store: boltdb
    object_store: filesystem
    schema: v11
    index:
    prefix: index_
    period: 24h
    storage_config:
    boltdb:
    directory: /srv/loki/index
    filesystem:
    directory: /srv/loki/chunks
    limits_config:
    enforce_metric_name: false
    reject_old_samples: true
    reject_old_samples_max_age: 168h
    EOH

    change_mode = "signal"
    change_signal = "SIGHUP"
    destination = "local/config/local-config.yaml"
    }
    }

    task "loki-promtail" {
    driver = "docker"

    resources {
    cpu = 100
    memory = 256
    }

    config {
    image = "grafana/promtail:2.1.0"

    args = [
    "-config.expand-env=true",
    "-config.file=/local/config/config.yml"
    ]
    }

    lifecycle {
    sidecar = true
    hook = "poststart"
    }

    template {
    data = <<EOH
    server:
    http_listen_port: 9080
    grpc_listen_port: 0
    positions:
    filename: /tmp/positions.yaml
    clients:
    - url: http://localhost:3100/loki/api/v1/push
    scrape_configs:
    - job_name: syslog
    syslog:
    listen_address: 0.0.0.0:2514
    idle_timeout: 60s
    label_structured_data: yes
    labels:
    job: "syslog"
    relabel_configs:
    - source_labels: ['__syslog_message_hostname']
    target_label: 'host'
    EOH
    change_mode = "signal"
    change_signal = "SIGHUP"
    destination = "local/config/config.yml"
    }
    }

    task "syslog" {
    driver = "docker"

    resources {
    cpu = 100
    memory = 256
    }

    config {
    image = "balabit/syslog-ng:3.30.1"

    args = [
    "--no-caps"
    ]

    volumes = [
    "local/config/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf"
    ]
    }

    lifecycle {
    sidecar = true
    hook = "poststart"
    }

    template {
    data = <<EOH
    @version: 3.30
    @include "scl.conf"
    source s_tcp {
    tcp(ip("0.0.0.0")
    port("1514")
    keep_hostname(yes)
    use_dns(yes)
    use_fqdn(yes)
    );
    };
    destination d_loki_promtail {
    syslog("localhost" transport("tcp") port(2514));
    };
    log {
    source(s_tcp);
    destination(d_loki_promtail);
    };
    EOH
    change_mode = "signal"
    change_signal = "SIGHUP"
    destination = "local/config/syslog-ng.conf"
    }
    }
    }
    }