busyboy77 · January 15, 2024 09:31 · Nov 2, 2020 · Nov 2, 2020 · Oct 29, 2020 · Oct 29, 2020
diff --git a/cluster-logging.yaml → clusteroutput-es.yaml b/cluster-logging.yaml → clusteroutput-es.yaml
diff --git a/clusteroutput-s3.yaml b/clusteroutput-s3.yaml
@@ -0,0 +1,55 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Output
+metadata:
+ name: cluster-output-s3
+ namespace: cattle-logging-system
+spec:
+ s3:
+   aws_key_id:
+     valueFrom:
+       secretKeyRef:
+         name: s3-write-logs
+         key: awsAccessKeyId
+   aws_sec_key:
+     valueFrom:
+       secretKeyRef:
+         name: s3-write-logs
+         key: awsSecretAccessKey
+   s3_bucket: log-archive-cluster-foo
+   s3_region: eu-central-1
+   path: logs/${tag}/%Y/%m/%d/
+   buffer:
+     timekey: 10m
+     timekey_wait: 30s
+     timekey_use_utc: true
+
+
+
+
+
+    s3_bucket "${s3_bucket}"
+    s3_region "${s3_region}"
+
+    path "${s3_prefix}%Y/%m/%d/%H/$${tag}/"
+    s3_object_key_format "%%{path}%%{time_slice}_%%{hex_random}_%%{index}_$${chunk_id}.%%{file_extension}"
+    storage_class "${storage_class}"
+
+    auto_create_bucket false
+    check_bucket true
+
+    <buffer tag,time>
+      @type file
+
+      path /td-agent/buffer/s3-buffer
+
+      timekey 3600
+      timekey_wait 10m
+      timekey_use_utc true
+
+      flush_at_shutdown true
+      flush_thread_count 8
+      retry_forever true
+      retry_max_interval 128s
+    </buffer>
+  </store>
+  %{ endif }
diff --git a/cluster-logging.yaml b/cluster-logging.yaml
@@ -11,6 +11,17 @@ spec:
     port: 443
     scheme: https
     ssl_verify: false
+    user: username
+    password:
+      valueFrom:
+        secretKeyRef:
+         name: es-log-writer
+         key: token
+    ca_file:
+      valueFrom:
+        secretKeyRef:
+         name: es-log-writer
+         key: ca.pem
     logstash_format: true
     logstash_prefix: k8s-cluster-foo
     logstash_dateformat: YYYY-MM-DD
@@ -21,14 +32,14 @@ spec:
     buffer:
       type: file
       chunk_limit_size: 4M # Determines HTTP payload size
-      total_limit_size: 256MB # Max total buffer size
+      total_limit_size: 1024MB # Max total buffer size
       flush_mode: interval
       flush_interval: 10s
       flush_thread_count: 2 # Parallel send of logs
       overflow_action: block
       retry_forever: true # Never discard buffer chunks
       retry_type: exponential_backoff
-      retry_max_interval: 60
+      retry_max_interval: 60s
 ---
 apiVersion: logging.banzaicloud.io/v1beta1
 kind: ClusterFlow

diff --git a/clusterflow-catchall.yaml → clusterflow-archive.yaml b/clusterflow-catchall.yaml → clusterflow-archive.yaml
@@ -1,7 +1,7 @@
 apiVersion: logging.banzaicloud.io/v1beta1
 kind: ClusterFlow
 metadata:
-  name: catchall
+  name: archive
 spec:
   match:
     - select: {}

diff --git a/clusterflow-catchall.yaml b/clusterflow-catchall.yaml
@@ -0,0 +1,9 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: ClusterFlow
+metadata:
+  name: catchall
+spec:
+  match:
+    - select: {}
+  outputRefs:
+    - s3
diff --git a/cluster-logging.yaml b/cluster-logging.yaml
@@ -20,13 +20,13 @@ spec:
     reload_on_failure: true  
     buffer:
       type: file
-      chunk_limit_size: 4M
-      queue_limit_length: 32
+      chunk_limit_size: 4M # Determines HTTP payload size
+      total_limit_size: 256MB # Max total buffer size
       flush_mode: interval
       flush_interval: 10s
-      flush_thread_count: 2
+      flush_thread_count: 2 # Parallel send of logs
       overflow_action: block
-      retry_forever: true
+      retry_forever: true # Never discard buffer chunks
       retry_type: exponential_backoff
       retry_max_interval: 60
 ---

diff --git a/cluster-logging.yaml b/cluster-logging.yaml
@@ -15,9 +15,9 @@ spec:
     logstash_prefix: k8s-cluster-foo
     logstash_dateformat: YYYY-MM-DD
     include_tag_key: true
-    reload_connections false
-    reconnect_on_error true
-    reload_on_failure true  
+    reload_connections: false
+    reconnect_on_error: true
+    reload_on_failure: true  
     buffer:
       type: file
       chunk_limit_size: 4M

diff --git a/cluster-logging.yaml b/cluster-logging.yaml
@@ -0,0 +1,45 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: ClusterOutput
+metadata:
+  name: cluster-output-es
+  namespace: cattle-logging-system
+  labels:
+    someLabel: foo
+spec:
+  elasticsearch:
+    host: elasticsearch-elasticsearch-cluster.default.svc.cluster.local
+    port: 443
+    scheme: https
+    ssl_verify: false
+    logstash_format: true
+    logstash_prefix: k8s-cluster-foo
+    logstash_dateformat: YYYY-MM-DD
+    include_tag_key: true
+    reload_connections false
+    reconnect_on_error true
+    reload_on_failure true  
+    buffer:
+      type: file
+      chunk_limit_size: 4M
+      queue_limit_length: 32
+      flush_mode: interval
+      flush_interval: 10s
+      flush_thread_count: 2
+      overflow_action: block
+      retry_forever: true
+      retry_type: exponential_backoff
+      retry_max_interval: 60
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: ClusterFlow
+metadata:
+  name: log-all-elastic
+  namespace: cattle-logging-system
+  labels:
+    someLabel: foo
+spec:
+  filters:
+    - tag_normaliser: {}
+  selectors: {}
+  globalOutputRefs:
+    - cluster-output-es
No results found