dtaivpp · March 14, 2025 06:08 · Feb 4, 2023 · Feb 4, 2023 · Feb 4, 2023 · Feb 4, 2023
diff --git a/logging_demo.py b/logging_demo.py
@@ -82,6 +82,10 @@
 logger.info("Look @ my pretty log")
 
 
+
+
+
+
 # ... but the log wasn't output ...
 
 
@@ -113,6 +117,7 @@
 #
 import logging
 import logging.config
+from logging.config import dictConfig
 CONFIG = '''
 {
     "version": 1,
@@ -147,8 +152,7 @@
     }
 }
 '''
-
-
+dictConfig(CONFIG)
 
 
 

diff --git a/logging_demo.py b/logging_demo.py
@@ -1,5 +1,5 @@
 # Download this file with:
-# curl -L https://gist.github.com/dtaivpp/a9b00957aa7d9cfe33e92aff8d50c835/raw/logging_demo.py -o logging_demo.py
+# curl -L https://gist.githubusercontent.com/dtaivpp/a9b00957aa7d9cfe33e92aff8d50c835/raw/logging_demo.py -o logging_demo.py
 import logging
 
 #
@@ -48,14 +48,15 @@
 #
 # Handlers
 #
-fileHandle = logging.FileHandler('test.log')
-logger.addHandler(fileHandle)
-
+import sys
+console_handle = logging.StreamHandler(sys.stdout)
+logger.addHandler(console_handle)
 
 # Setting Handlers Level
-fileHandle.setLevel(logging.WARNING)
+console_handle.setLevel(logging.WARNING)
 
-logger.info("This won't show in test.log")
+# Sending Logs
+logger.info("This won't show in the console")
 logger.error("This will")
 
 
@@ -75,13 +76,18 @@
 #
 # Formatters 
 #
-consoleHandle = logging.StreamHandler()
-consoleHandle.setLevel(logging.INFO)
+console_handle.setLevel(logging.INFO)
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-consoleHandle.setFormatter(formatter)
-logger.addHandler(consoleHandle)
+console_handle.setFormatter(formatter)
+logger.info("Look @ my pretty log")
+
+
+# ... but the log wasn't output ...
 
 
+logger.setLevel(logging.DEBUG)
+logger.info("Look @ my pretty log")
+
 
 
 
@@ -95,8 +101,8 @@
 #           |
 #           V
 #        Formatters - Are attached to handlers and enrich the output 
-
-
+#
+#        *all of these filter by log level*
 
 
 
@@ -157,9 +163,12 @@
 #
 
 # eg:
-http_handle = SMTPHandler(url="https://some-log-server.com")
-http_handle.setLevel(logging.INFO)
-logger.addHandler(http_handle)
+smtp_handler = SMTPHandler("smtp.email.com", 
+                           fromaddr="[email protected]", 
+                           toaddrs="[email protected]", 
+                           subject="Wont work")
+smtp_handler.setLevel(logging.INFO)
+logger.addHandler(smtp_handler)
 
 for record in ["Some long list"]:
   logger.info("test")
@@ -177,11 +186,12 @@
 import queue
 from logging.handlers import QueueHandler, QueueListener
 
-que = queue.Queue(-1)  # no limit on size
-queue_handler = QueueHandler(que)
-handler = logging.StreamHandler()
-listener = QueueListener(que, handler)
-
+q = queue.Queue(-1)  # no limit on size
+queue_handler = QueueHandler(q)
+other_stream = logging.StreamHandler(sys.stdout)
+listener = QueueListener(q, other_stream)
+logger.addHandler(queue_handler)
+listener.start()
 
 
 
@@ -190,10 +200,10 @@
 # Pulling it all together with OpenSearch
 # 
 # Download the following docker compose file: 
-# curl -L https://gist.github.com/dtaivpp/77e310917716e49d6fafa489283847ea/raw/docker-compose.yml -o docker-compose.yml
+# curl -L https://gist.githubusercontent.com/dtaivpp/77e310917716e49d6fafa489283847ea/raw/docker-compose.yml -o docker-compose.yml
 # Run with `docker compose up -d`
 #
-# `pip install opensearch-py``
+# `pip install opensearch-py`
 # Then you can run with the following:
 #
 import logging
@@ -214,8 +224,7 @@ class OpenSearchHandler(logging.Handler):
     """
     def __init__(self):
         logging.Handler.__init__(self)
-        self.opensearch_client = get_os_client()
-
+        self.opensearch_client = opensearch_client
 
     def __index_name_builder(self, name: str):
         """This method creates a standardized way to build index names."""
@@ -241,5 +250,8 @@ def emit(self, record):
 os_logger.setLevel(logging.INFO)
 os_handler = OpenSearchHandler()
 os_handler.setLevel(logging.INFO)
+os_logger.addHandler(os_handler)
 
 os_logger.info("This is my log~!")
+
+# Congrats now you do more logging than 99% of the population!
diff --git a/logging_demo.py b/logging_demo.py
@@ -190,7 +190,7 @@
 # Pulling it all together with OpenSearch
 # 
 # Download the following docker compose file: 
-# curl -L https://gist.githubusercontent.com/dtaivpp/77e310917716e49d6fafa489283847ea/raw/docker-compose.yml -o docker-compose.yml
+# curl -L https://gist.github.com/dtaivpp/77e310917716e49d6fafa489283847ea/raw/docker-compose.yml -o docker-compose.yml
 # Run with `docker compose up -d`
 #
 # `pip install opensearch-py``

diff --git a/logging_demo.py b/logging_demo.py
@@ -1,5 +1,5 @@
 # Download this file with:
-# curl -L https://gist.githubusercontent.com/dtaivpp/a9b00957aa7d9cfe33e92aff8d50c835/raw/logging_demo.py -o logging_demo.py
+# curl -L https://gist.github.com/dtaivpp/a9b00957aa7d9cfe33e92aff8d50c835/raw/logging_demo.py -o logging_demo.py
 import logging
 
 #

diff --git a/logging_demo.py b/logging_demo.py
@@ -1,14 +1,21 @@
+# Download this file with:
+# curl -L https://gist.github.com/dtaivpp/a9b00957aa7d9cfe33e92aff8d50c835/raw/logging_demo.py -o logging_demo.py
 import logging
 
-
+#
 # Named loggers
+#
 logger = logging.getLogger(__name__)
 print(logger.name)
 
 
 
 
+
+
+#
 # Log Levels
+#
 log_levels = [
   logging.NOTSET,
   logging.DEBUG,
@@ -25,14 +32,22 @@
 
 
 
+
+
+#
 # Emmitting records
+#
 logger.info("Emmitting Info Record")
 # Here the logger named "__main__" is emmitting an info record
 
 
 
 
+
+
+#
 # Handlers
+#
 fileHandle = logging.FileHandler('test.log')
 logger.addHandler(fileHandle)
 
@@ -49,20 +64,31 @@
     RotatingFileHandler, \
     SysLogHandler, \
     HTTPHandler, \
-    TimedRotatingFileHandler
+    TimedRotatingFileHandler, \
+    SMTPHandler
+
+
+
 
 
+
+#
 # Formatters 
+#
 consoleHandle = logging.StreamHandler()
 consoleHandle.setLevel(logging.INFO)
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 consoleHandle.setFormatter(formatter)
 logger.addHandler(consoleHandle)
 
 
+
+
+
+
 # Top to bottom
 #
-#        Logger - Emits Records
+#        Logger - Emits Records at some level
 #           |
 #           V
 #        Handlers - Recieve logs and send to specified output 
@@ -71,12 +97,14 @@
 #        Formatters - Are attached to handlers and enrich the output 
 
 
+
+
+
+
 #
 # Setting up loggers one line at a time can be confusing
 #    so we can use a json logging config. 
 #
-
-import json
 import logging
 import logging.config
 CONFIG = '''
@@ -115,60 +143,103 @@
 '''
 
 
+
+
+
+
 #
 # There is however one other problem.... 
 #
 #
 #
 #
-# Loggers are thread blocking....
+# Loggers are blocking operations....
 #
 
 # eg:
-http_handle = HTTPHandler(url="https://some-log-server.com")
+http_handle = SMTPHandler(url="https://some-log-server.com")
 http_handle.setLevel(logging.INFO)
 logger.addHandler(http_handle)
 
 for record in ["Some long list"]:
   logger.info("test")
   print(record)
-
-
 
-
+
+
+
+
+
+#
+# In comes queues to save the day!
+# - note setting up queueing by dictConfig is only available in Python 3.12
+#
+import queue
+from logging.handlers import QueueHandler, QueueListener
+
+que = queue.Queue(-1)  # no limit on size
+queue_handler = QueueHandler(que)
+handler = logging.StreamHandler()
+listener = QueueListener(que, handler)
+
+
+
+
+
+
+# Pulling it all together with OpenSearch
+# 
+# Download the following docker compose file: 
+# curl -L https://gist.github.com/dtaivpp/77e310917716e49d6fafa489283847ea/raw/docker-compose.yml -o docker-compose.yml
+# Run with `docker compose up -d`
+#
+# `pip install opensearch-py``
+# Then you can run with the following:
+#
 import logging
 from datetime import datetime
-from search.opensearch import get_os_client
+from opensearchpy import OpenSearch
+
+opensearch_client = OpenSearch(
+        "https://admin:admin@localhost:9200",
+        use_ssl=True,
+        verify_certs=False,
+        ssl_show_warn=False
+    )
 
+# Custom Handler to send to OpenSearch
 class OpenSearchHandler(logging.Handler):
     """
     Custom handler to send certain logs to OpenSearch
     """
     def __init__(self):
-        try:
-            logging.Handler.__init__(self)
-            self.opensearch_client = get_os_client()
-        except Exception as e: 
-            print("Failed initialization")
+        logging.Handler.__init__(self)
+        self.opensearch_client = get_os_client()
+
 
-    def __index_name_builder(name: str):
+    def __index_name_builder(self, name: str):
         """This method creates a standardized way to build index names."""
         return f"{name}-{datetime.date(datetime.now())}"
 
     def emit(self, record):
         """
         Sends Log to OpenSearch
         """
+        created_time = datetime.fromtimestamp(record.created).isoformat()#strftime("%Y-%m-%d %H:%M:S")
 
         doc = {
-          "message": record.message, 
+          "message": record.msg, 
           "locator": str(record.funcName) + " : " + str(record.lineno),
           "exec_info": record.exc_info,
-          "@timestamp": record.asctime
+          "created": created_time
         }
-
-        try:
-            self.opensearch_client.index(index=self.__index_name_builder(record.name), body=doc)
-        except Exception as e: 
-            print(e)
-        return
+
+        self.opensearch_client.index(index=self.__index_name_builder(record.name), body=doc)
+
+# Setup Logging
+os_logger = logging.getLogger("log")
+os_logger.setLevel(logging.INFO)
+os_handler = OpenSearchHandler()
+os_handler.setLevel(logging.INFO)
+
+os_logger.info("This is my log~!")
diff --git a/logging_demo.py b/logging_demo.py
@@ -131,4 +131,44 @@
 
 for record in ["Some long list"]:
   logger.info("test")
-  print(record)
+  print(record)
+
+
+
+
+import logging
+from datetime import datetime
+from search.opensearch import get_os_client
+
+class OpenSearchHandler(logging.Handler):
+    """
+    Custom handler to send certain logs to OpenSearch
+    """
+    def __init__(self):
+        try:
+            logging.Handler.__init__(self)
+            self.opensearch_client = get_os_client()
+        except Exception as e: 
+            print("Failed initialization")
+
+    def __index_name_builder(name: str):
+        """This method creates a standardized way to build index names."""
+        return f"{name}-{datetime.date(datetime.now())}"
+
+    def emit(self, record):
+        """
+        Sends Log to OpenSearch
+        """
+
+        doc = {
+          "message": record.message, 
+          "locator": str(record.funcName) + " : " + str(record.lineno),
+          "exec_info": record.exc_info,
+          "@timestamp": record.asctime
+        }
+
+        try:
+            self.opensearch_client.index(index=self.__index_name_builder(record.name), body=doc)
+        except Exception as e: 
+            print(e)
+        return
diff --git a/logging_demo.py b/logging_demo.py
@@ -0,0 +1,134 @@
+import logging
+
+
+# Named loggers
+logger = logging.getLogger(__name__)
+print(logger.name)
+
+
+
+
+# Log Levels
+log_levels = [
+  logging.NOTSET,
+  logging.DEBUG,
+  logging.INFO,
+  logging.WARN,
+  logging.WARNING, 
+  logging.ERROR,
+  logging.CRITICAL
+]
+
+for level in log_levels:
+  print(level)
+
+
+
+
+# Emmitting records
+logger.info("Emmitting Info Record")
+# Here the logger named "__main__" is emmitting an info record
+
+
+
+
+# Handlers
+fileHandle = logging.FileHandler('test.log')
+logger.addHandler(fileHandle)
+
+
+# Setting Handlers Level
+fileHandle.setLevel(logging.WARNING)
+
+logger.info("This won't show in test.log")
+logger.error("This will")
+
+
+# Some common/useful handlers
+from logging.handlers import \
+    RotatingFileHandler, \
+    SysLogHandler, \
+    HTTPHandler, \
+    TimedRotatingFileHandler
+
+
+# Formatters 
+consoleHandle = logging.StreamHandler()
+consoleHandle.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+consoleHandle.setFormatter(formatter)
+logger.addHandler(consoleHandle)
+
+
+# Top to bottom
+#
+#        Logger - Emits Records
+#           |
+#           V
+#        Handlers - Recieve logs and send to specified output 
+#           |
+#           V
+#        Formatters - Are attached to handlers and enrich the output 
+
+
+#
+# Setting up loggers one line at a time can be confusing
+#    so we can use a json logging config. 
+#
+
+import json
+import logging
+import logging.config
+CONFIG = '''
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "format": "%(levelname)-8s - %(message)s"
+        }
+    },
+    "filters": {
+        "warnings_and_below": {
+            "()" : "__main__.filter_maker",
+            "level": "WARNING"
+        }
+    },
+    "handlers": {
+        "stdout": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout",
+            "filters": ["warnings_and_below"]
+        }
+    },
+    "root": {
+        "level": "DEBUG",
+        "handlers": [
+            "stderr",
+            "stdout",
+            "file"
+        ]
+    }
+}
+'''
+
+
+#
+# There is however one other problem.... 
+#
+#
+#
+#
+# Loggers are thread blocking....
+#
+
+# eg:
+http_handle = HTTPHandler(url="https://some-log-server.com")
+http_handle.setLevel(logging.INFO)
+logger.addHandler(http_handle)
+
+for record in ["Some long list"]:
+  logger.info("test")
+  print(record)