ruanbekker · April 3, 2025 17:58 · Apr 21, 2020 · Apr 21, 2020 · Jan 29, 2020
diff --git a/promtail-journal.yml b/promtail-journal.yml
@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+positions:
+  filename: /var/lib/promtail/positions.yaml
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 1h
+      path: /var/log/journal
+      labels:
+        job: systemd
+        env: production
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
diff --git a/promtail-nginx_with_pipeline.yml b/promtail-nginx_with_pipeline.yml
@@ -0,0 +1,30 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+positions:
+  filename: /var/lib/promtail/positions.yaml
+clients:
+  - url: http://prometheus.internal.eng.mamamoney.co.za:3100/loki/api/v1/push
+scrape_configs:
+- job_name: nginx
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      app: nginx
+      env: production
+      host: ruan-prod-nginx
+      __path__: /var/log/nginx/*.log
+  pipeline_stages:
+  - match:
+      selector: '{app="nginx"}'
+      stages:
+      - regex:
+          # logline example: 127.0.0.1 - - [21/Apr/2020:13:59:45 +0000] "GET /?foo=bar HTTP/1.1" 200 612 "http://example.com/lekkebot.html" "curl/7.58.0"
+          expression: '^(?P<host>[\w\.]+) - (?P<user>[^ ]*) \[(?P<timestamp>.*)\] "(?P<method>[^ ]*) (?P<request_url>[^ ]*) (?P<request_http_protocol>[^ ]*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+) "(?P<http_referer>[^"]*)" "(?P<user_agent>[^"]*)"?'
+      - labels:
+          host:
+          method:
+          request_url:
+          status:
+          user_agent:
diff --git a/dnsmasq.conf b/dnsmasq.conf
@@ -0,0 +1,7 @@
+log-queries
+log-facility=/var/log/dnsmasq.log
+no-resolv
+server=8.8.4.4
+server=8.8.8.8
+address=/router/10.1.1.1
+address=/server/10.1.1.2
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,73 @@
+version: '2.4'
+
+networks:
+  loki:
+
+services:
+
+  loki:
+    image: grafana/loki:master-2739551
+    ports:
+      - "3100:3100"
+    networks:
+      - loki
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:6.4.4
+    ports:
+      - "3000:3000"
+    networks:
+      - loki
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:master-2739551
+    networks:
+      - loki
+    volumes:
+      - ./promtail.yml:/etc/promtail/promtail.yml
+    command: 
+      - '-config.file=/etc/promtail/promtail.yml'
+    restart: unless-stopped
+
+  promtail-nginx:
+    image: grafana/promtail:master-2739551
+    networks:
+      - loki
+    volumes:
+      - ./promtail-nginx.yml:/etc/promtail/promtail.yml
+      - ./nginx:/var/log/nginx
+    command:
+      - '-config.file=/etc/promtail/promtail.yml'
+    restart: unless-stopped
+
+  nginx:
+    image: nginx
+    networks:
+      - loki
+    volumes:
+      - ./nginx:/var/log/nginx
+    ports:
+      - 8084:80
+
+  promtail-dnsmasq:
+    image: grafana/promtail:master-2739551
+    networks:
+      - loki
+    volumes:
+      - ./promtail-dnsmasq.yml:/etc/promtail/promtail.yml
+      - ./dnsmasq:/var/log
+    command:
+      - '-config.file=/etc/promtail/promtail.yml'
+    restart: unless-stopped
+
+  dnsmasq:
+    image: strm/dnsmasq
+    networks:
+      - loki
+    cap_add:
+      - NET_ADMIN
+    volumes:
+      - ./dnsmasq:/var/log
+      - ./dnsmasq.conf:/etc/dnsmasq.conf 
diff --git a/promtail-dnsmasq.yml b/promtail-dnsmasq.yml
@@ -0,0 +1,31 @@
+server:
+
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+client:
+
+  url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+
+  - job_name: dnsmasq
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: dnsmasq
+        env: production
+        host: ruan-prod-dnsmasq
+        __path__: /var/log/dnsmasq.log
+    pipeline_stages:
+    - match:
+        selector: '{job="dnsmasq"}'
+        stages:
+        - regex:
+            expression: ".*query\\[A\\] (?P<query>.*\\s)"
+        - labels:
+            query:
diff --git a/promtail-nginx.yml b/promtail-nginx.yml
@@ -0,0 +1,23 @@
+server:
+
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+client:
+
+  url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+
+  - job_name: nginx
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx
+        env: production
+        host: ruan-prod-nginx
+        __path__: /var/log/nginx/*.log
diff --git a/promtail.yml b/promtail.yml
@@ -0,0 +1,23 @@
+server:
+
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+client:
+
+  url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+
+  - job_name: messages
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: all-logs
+        env: dev
+        host: monitoring-dev
+        __path__: /var/log/*.log
diff --git a/queries.md b/queries.md
@@ -0,0 +1,52 @@
+## Prometheus (loki) as Datasource
+
+Requests for last 60 Seconds:
+
+```
+count_over_time( {job="nginx"} [60s])
+```
+
+Rate over 60s:
+
+```
+rate( ( {env="production", job="nginx"} ) [60s])
+```
+
+Show metrics with filter patterns:
+
+```
+rate( ( {env="production", job="nginx"} |~ "GET (/er|/ax)" ) [10s])
+```
+
+Dnsmasq Queries:
+
+```
+sum (rate( ( {env="production",query=~".*\\..*"} )[5m] )) by (query)
+
+```
+
+## Loki as a Datasource
+
+All logs:
+
+```
+{job="all-logs"}
+```
+
+Logs for nginx job:
+
+```
+{job="nginx"}
+```
+
+## Generating Queries on DnsMasq Container
+
+```
+apk add bind-tools
+for domain in yahoo.com google.com facebook.com; do dig A $domain @127.0.0.1; done
+```
+
+## More
+- https://itnext.io/grafana-loki-the-logqls-prometheus-like-counters-aggregation-functions-and-dnsmasq-s-1e622c25c7e4
+- https://grafana.com/blog/2019/07/15/lokis-path-to-ga-docker-logging-driver-plugin-support-for-systemd/
+- https://medium.com/@mightywomble/using-grafana-to-visualise-syslog-files-with-loki-9a8869081fb5 (includes queries)