Skip to content

Instantly share code, notes, and snippets.

@roguh

roguh/jsonlogparser.py

Last active November 30, 2021 18:13
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save roguh/e914fd540061ff74b6a703c13113822c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save roguh/e914fd540061ff74b6a703c13113822c to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. roguh revised this gist Nov 30, 2021. 1 changed file with 29 additions and 3 deletions.
    32 changes: 29 additions & 3 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -11,6 +11,7 @@
    DEFAULT_JSON_INDENT = 4

    FORMAT = "{asctime} {severity} {name} {module}:{funcName}:{lineno} {message}"
    REQUIRED_KEYS = "{asctime} {severity} {name} {module}:{funcName}:{lineno} {message}"

    # Stack traces or exception tracebacks
    KEYS_WITH_STACK_TRACES = [
    @@ -39,6 +40,8 @@
    @dataclass
    class Arguments:
    files: List[str]
    fail_if_non_json: bool
    missing_keys_warnings: bool
    ignored_keys: List[str]
    show_stack_traces: bool
    show_extra_keys: bool
    @@ -73,11 +76,20 @@ def parse_loki_json(line):
    JSON_PARSERS = [parse_plain_json, parse_loki_json]


    def parse(line: StrOrBytes) -> Union[ParsedLog, StrOrBytes]:
    def parse(
    line: StrOrBytes, fail_if_non_json: bool = False
    ) -> Union[ParsedLog, StrOrBytes]:
    for parser in JSON_PARSERS:
    log_msg = parser(line)
    if log_msg is not None:
    return log_msg
    if fail_if_non_json:
    print(
    "FATAL: could not parse JSON with parsers",
    ", ".join(func.__name__ for func in JSON_PARSERS),
    )
    print(line[:-1])
    sys.exit(1)
    return line


    @@ -159,7 +171,9 @@ def format_log_msg(
    return None
    return msg

    except (ValueError, KeyError):
    except (ValueError, KeyError) as exc:
    if isinstance(exc, KeyError) and args.missing_keys_warnings:
    print("WARNING: missing keys", file=sys.stderr)
    if args.show_non_json:
    # Exclude trailing newline
    return original_line[:-1]
    @@ -175,6 +189,11 @@ def parse_args() -> Arguments:
    parser.add_argument(
    "files", help="Which log files to read. Reads from stdin as well.", nargs="*"
    )
    parser.add_argument(
    "--missing-keys-warnings",
    help="Print a warning to stderr if there are missing required keys in JSON logs.",
    action="store_true",
    )
    parser.add_argument(
    "--no-extra-keys",
    help="Do not print extra keys in a JSON log line",
    @@ -197,6 +216,11 @@ def parse_args() -> Arguments:
    help="Only show non-JSON input",
    action="store_true",
    )
    non_json_group.add_argument(
    "--fail-if-non-json",
    help="Exit immediately if some lines cannot be parsed as JSON.",
    action="store_true",
    )

    stack_trace_group = parser.add_mutually_exclusive_group()
    stack_trace_group.add_argument(
    @@ -275,10 +299,12 @@ def parse_args() -> Arguments:

    application_args = Arguments(
    files=args.files,
    missing_keys_warnings=args.missing_keys_warnings,
    show_stack_traces=not args.no_stack_traces and not args.only_non_json,
    show_extra_keys=not args.no_extra_keys and not args.only_non_json,
    show_message=not args.no_message and not args.only_non_json,
    show_non_json=args.only_non_json or not args.no_non_json,
    fail_if_non_json=args.fail_if_non_json,
    strip_stack_traces=args.strip_stack_traces,
    replace_stack_trace_newlines=args.replace_stack_trace_newlines,
    extra_keys_sep=" " if args.one_line or args.extra_keys_on_same_line else "\n",
    @@ -297,7 +323,7 @@ def main():
    with fileinput.input(files=args.files, mode="r") as fileinputinput:
    try:
    for line in fileinputinput:
    parsed_json = parse(line)
    parsed_json = parse(line, args.fail_if_non_json)
    readable_message = format_log_msg(parsed_json, line, args)
    if readable_message is not None:
    print(readable_message)
  2. roguh revised this gist Nov 30, 2021. 1 changed file with 106 additions and 59 deletions.
    165 changes: 106 additions & 59 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,16 +1,18 @@
    #!/usr/bin/env python3
    from argparse import ArgumentParser
    import sys
    from pprint import pprint
    from dataclasses import dataclass
    import fileinput
    from typing import Union, Dict, Any, List, Optional
    import json
    import re
    import sys
    from argparse import ArgumentParser
    from dataclasses import dataclass
    from pprint import pprint
    from typing import Any, Dict, List, Optional, Union

    DEFAULT_JSON_INDENT = 4

    FORMAT = "{asctime} {severity} {name} {module}:{funcName}:{lineno} {message}"

    # Stack traces or exception tracebacks
    KEYS_WITH_STACK_TRACES = [
    "exc_info",
    "stack_info",
    @@ -42,6 +44,8 @@ class Arguments:
    show_extra_keys: bool
    show_message: bool
    show_non_json: bool
    strip_stack_traces: bool
    replace_stack_trace_newlines: Optional[str]
    extra_keys_sep: str
    json_indent: Optional[int]

    @@ -77,6 +81,59 @@ def parse(line: StrOrBytes) -> Union[ParsedLog, StrOrBytes]:
    return line


    def format_extra_keys(log_msg: ParsedLog, args: Arguments) -> str:
    extra_items = {
    key: value
    for key, value in log_msg.items()
    if not (key in KNOWN_KEYS or key in args.ignored_keys)
    }

    output = ""
    if args.show_extra_keys and len(extra_items) > 0:
    # Add newlines between msg components
    if args.show_message:
    output += args.extra_keys_sep

    output += json.dumps(extra_items, indent=args.json_indent)
    return output


    def strip_stack_trace(stack_trace: str) -> str:
    lines = stack_trace.split("\n")
    last_file_location_index = len(lines) - 1
    for index, line in enumerate(lines):
    if re.match('\\s*File ".*", line .*', line):
    last_file_location_index = index
    return "\n".join(lines[:2] + lines[last_file_location_index:])


    def format_stack_traces(
    log_msg: ParsedLog,
    newline_at_start: bool,
    strip: bool,
    replace_newlines: Optional[str] = None,
    ) -> str:
    output = ""
    for i, field in enumerate(KEYS_WITH_STACK_TRACES):
    if field in log_msg:
    if log_msg[field] == "NoneType: None":
    continue

    # Add newlines between msg components
    if i == 0 and not newline_at_start:
    exc_sep = ""
    else:
    exc_sep = replace_newlines if replace_newlines is not None else "\n"

    stack_trace = log_msg[field]
    if strip:
    stack_trace = strip_stack_trace(stack_trace)
    if replace_newlines:
    stack_trace = stack_trace.replace("\n", replace_newlines)
    output += exc_sep + stack_trace
    return output


    def format_log_msg(
    log_msg: Union[ParsedLog, StrOrBytes], original_line: StrOrBytes, args: Arguments
    ) -> Optional[StrOrBytes]:
    @@ -86,34 +143,17 @@ def format_log_msg(

    msg = ""
    if args.show_message:
    msg = FORMAT.format(**{"name": "|"} | log_msg)

    extra_items = {
    key: value
    for key, value in log_msg.items()
    if not (key in KNOWN_KEYS or key in args.ignored_keys)
    }

    if args.show_extra_keys and len(extra_items) > 0:
    # Add newlines between msg components
    if args.show_message:
    msg += args.extra_keys_sep
    msg += FORMAT.format(**{"name": "|"} | log_msg)

    msg += json.dumps(extra_items, indent=args.json_indent)
    msg += format_extra_keys(log_msg, args)

    if args.show_stack_traces:
    for i, field in enumerate(KEYS_WITH_STACK_TRACES):
    if field in log_msg:
    if log_msg[field] == "NoneType: None":
    continue

    # Add newlines between msg components
    if i == 0 and not (args.show_extra_keys or args.show_message):
    exc_sep = ""
    else:
    exc_sep = "\n"

    msg += exc_sep + log_msg[field]
    msg += format_stack_traces(
    log_msg,
    newline_at_start=args.show_extra_keys or args.show_message,
    strip=args.strip_stack_traces,
    replace_newlines=args.replace_stack_trace_newlines,
    )

    if len(msg) == 0:
    return None
    @@ -129,7 +169,7 @@ def parse_args() -> Arguments:
    parser = ArgumentParser(
    description="""Convert JSON logs from stdin or files into readable output.
    The JSON keys are assumed to come from Python.
    Stack trace and exception information will be searched for in the keys: """
    Stack trace and exception tracebacks will be searched for in the keys: """
    + (", ".join(KEYS_WITH_STACK_TRACES))
    )
    parser.add_argument(
    @@ -161,38 +201,41 @@ def parse_args() -> Arguments:
    stack_trace_group = parser.add_mutually_exclusive_group()
    stack_trace_group.add_argument(
    "--no-stack-traces",
    help="Do not print Python stack traces or exception information.",
    help="Do not print Python stack traces or exception tracebacks.",
    action="store_true",
    )
    stack_trace_group.add_argument(
    "--only-stack-traces",
    help="Only print JSON log lines that contain Python stack traces or exception information.",
    help="Only print JSON log lines that contain Python stack traces or exception tracebacks.",
    action="store_true",
    )

    # TODO
    # parser.add_argument(
    # "--strip-stack-traces",
    # help="Print only the first and last lines of Python stack traces or exception information.",
    # action="store_true",
    # )
    # parser.add_argument(
    # "--replace-stack-trace-newlines",
    # help="Print Python stack traces or exception information, but replace newlines with the argument.",
    # action="store",
    # )
    parser.add_argument(
    "--strip-stack-traces",
    help="Print only the first two lines and last few lines, including the last file position, of Python stack traces or exception tracebacks.",
    action="store_true",
    )
    parser.add_argument(
    "--replace-stack-trace-newlines",
    help="Print Python stack traces or exception tracebacks, but replace newlines with the argument.",
    action="store",
    default=None,
    )

    parser.add_argument(
    "--extra-keys-on-same-line",
    help="Print extra keys on the same line as the formatted log message",
    action="store_true",
    )

    # TODO
    # one line with stripped stack traces
    parser.add_argument(
    "--one-line-output",
    help="Print the formatted log message on a single line.",
    "--stack-traces-one-line",
    help="Print the stack traces and exception tracebacks on a single line. Same as \"--replace-stack-trace-newlines ' '\"",
    action="store_true",
    )
    parser.add_argument(
    "--one-line",
    help="Print the formatted log message on a single line. Excludes stack traces and exception tracebacks.",
    action="store_true",
    )

    @@ -227,18 +270,19 @@ def parse_args() -> Arguments:
    args.no_extra_keys = True
    args.no_message = True

    if args.stack_traces_one_line:
    args.replace_stack_trace_newlines = " "

    application_args = Arguments(
    files=args.files,
    show_stack_traces=not args.no_stack_traces and not args.only_non_json,
    show_extra_keys=not args.no_extra_keys and not args.only_non_json,
    show_message=not args.no_message and not args.only_non_json,
    show_non_json=args.only_non_json or not args.no_non_json,
    extra_keys_sep=" "
    if args.one_line_output or args.extra_keys_on_same_line
    else "\n",
    json_indent=None
    if args.one_line_output or args.compact_json
    else args.json_indent,
    strip_stack_traces=args.strip_stack_traces,
    replace_stack_trace_newlines=args.replace_stack_trace_newlines,
    extra_keys_sep=" " if args.one_line or args.extra_keys_on_same_line else "\n",
    json_indent=None if args.one_line or args.compact_json else args.json_indent,
    ignored_keys=DEFAULT_IGNORED_KEYS if args.ignore is None else args.ignore,
    )

    @@ -251,11 +295,14 @@ def parse_args() -> Arguments:
    def main():
    args = parse_args()
    with fileinput.input(files=args.files, mode="r") as fileinputinput:
    for line in fileinputinput:
    parsed_json = parse(line)
    readable_message = format_log_msg(parsed_json, line, args)
    if readable_message is not None:
    print(readable_message)
    try:
    for line in fileinputinput:
    parsed_json = parse(line)
    readable_message = format_log_msg(parsed_json, line, args)
    if readable_message is not None:
    print(readable_message)
    except KeyboardInterrupt:
    return


    if __name__ == "__main__":
  3. roguh revised this gist Nov 30, 2021. 1 changed file with 21 additions and 7 deletions.
    28 changes: 21 additions & 7 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,7 @@
    #!/usr/bin/env python3
    from argparse import ArgumentParser
    import sys
    from pprint import pprint
    from dataclasses import dataclass
    import fileinput
    from typing import Union, Dict, Any, List, Optional
    @@ -44,6 +46,12 @@ class Arguments:
    json_indent: Optional[int]


    def dataclass_to_dict(dataclass_) -> Dict[str, Any]:
    return {
    key: getattr(dataclass_, key) for key in dataclass_.__dataclass_fields__.keys()
    }


    def parse_plain_json(line):
    try:
    return json.loads(line)
    @@ -119,12 +127,14 @@ def format_log_msg(

    def parse_args() -> Arguments:
    parser = ArgumentParser(
    description="Convert JSON logs into readable output."
    "The JSON keys are assumed to come from Python."
    "Stack trace and exception information will be searched for in the keys: "
    + (",".join(KEYS_WITH_STACK_TRACES))
    description="""Convert JSON logs from stdin or files into readable output.
    The JSON keys are assumed to come from Python.
    Stack trace and exception information will be searched for in the keys: """
    + (", ".join(KEYS_WITH_STACK_TRACES))
    )
    parser.add_argument(
    "files", help="Which log files to read. Reads from stdin as well.", nargs="*"
    )
    parser.add_argument("files", nargs="*")
    parser.add_argument(
    "--no-extra-keys",
    help="Do not print extra keys in a JSON log line",
    @@ -208,6 +218,8 @@ def parse_args() -> Arguments:
    action="store_true",
    )

    parser.add_argument("--print-arguments", action="store_true")

    args = parser.parse_args()

    # convenience for --no-extra --no-message
    @@ -229,8 +241,10 @@ def parse_args() -> Arguments:
    else args.json_indent,
    ignored_keys=DEFAULT_IGNORED_KEYS if args.ignore is None else args.ignore,
    )
    # TODO debug output
    # print(application_args)

    if args.print_arguments:
    print("Arguments:", file=sys.stderr)
    pprint(dataclass_to_dict(application_args), stream=sys.stderr)
    return application_args


  4. roguh revised this gist Nov 30, 2021. 1 changed file with 19 additions and 10 deletions.
    29 changes: 19 additions & 10 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -125,11 +125,6 @@ def parse_args() -> Arguments:
    + (",".join(KEYS_WITH_STACK_TRACES))
    )
    parser.add_argument("files", nargs="*")
    parser.add_argument(
    "--no-stack-traces",
    help="Do not print Python stack traces and exception information.",
    action="store_true",
    )
    parser.add_argument(
    "--no-extra-keys",
    help="Do not print extra keys in a JSON log line",
    @@ -153,19 +148,27 @@ def parse_args() -> Arguments:
    action="store_true",
    )

    # TODO
    # only show msgs with stack traces
    # convenience for --no-extra --no-message --no-non
    stack_trace_group = parser.add_mutually_exclusive_group()
    stack_trace_group.add_argument(
    "--no-stack-traces",
    help="Do not print Python stack traces or exception information.",
    action="store_true",
    )
    stack_trace_group.add_argument(
    "--only-stack-traces",
    help="Only print JSON log lines that contain Python stack traces or exception information.",
    action="store_true",
    )

    # TODO
    # parser.add_argument(
    # "--strip-stack-traces",
    # help="Print only the first and last lines of Python stack traces and exception information.",
    # help="Print only the first and last lines of Python stack traces or exception information.",
    # action="store_true",
    # )
    # parser.add_argument(
    # "--replace-stack-trace-newlines",
    # help="Print Python stack traces and exception information, but replace newlines with the argument.",
    # help="Print Python stack traces or exception information, but replace newlines with the argument.",
    # action="store",
    # )

    @@ -206,6 +209,12 @@ def parse_args() -> Arguments:
    )

    args = parser.parse_args()

    # convenience for --no-extra --no-message
    if args.only_stack_traces:
    args.no_extra_keys = True
    args.no_message = True

    application_args = Arguments(
    files=args.files,
    show_stack_traces=not args.no_stack_traces and not args.only_non_json,
  5. roguh revised this gist Nov 30, 2021. 1 changed file with 179 additions and 24 deletions.
    203 changes: 179 additions & 24 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,15 +1,15 @@
    #!/usr/bin/env python3
    from argparse import ArgumentParser
    from dataclasses import dataclass
    import fileinput
    from typing import Union, Dict, Any, List, Optional
    import json

    # add ability to not output exc_info or extra keys or output on same line
    # add ability to only lo JSON formatted or only log non-JSON

    JSON_INDENT=4
    DEFAULT_JSON_INDENT = 4

    FORMAT = "{asctime} {severity} {name} {module}:{funcName}:{lineno} {message}"

    KEYS_WITH_NEWLINES = [
    KEYS_WITH_STACK_TRACES = [
    "exc_info",
    "stack_info",
    ]
    @@ -21,12 +21,28 @@
    "funcName",
    "lineno",
    "message",
    ] + KEYS_WITH_NEWLINES
    IGNORED_KEYS = [
    ] + KEYS_WITH_STACK_TRACES

    DEFAULT_IGNORED_KEYS = [
    "filename",
    "pathname",
    ]

    StrOrBytes = Union[str, bytes]
    ParsedLog = Dict[str, Any]


    @dataclass
    class Arguments:
    files: List[str]
    ignored_keys: List[str]
    show_stack_traces: bool
    show_extra_keys: bool
    show_message: bool
    show_non_json: bool
    extra_keys_sep: str
    json_indent: Optional[int]


    def parse_plain_json(line):
    try:
    @@ -42,42 +58,181 @@ def parse_loki_json(line):
    return None


    def parse(line):
    for parser in [parse_plain_json, parse_loki_json]:
    JSON_PARSERS = [parse_plain_json, parse_loki_json]


    def parse(line: StrOrBytes) -> Union[ParsedLog, StrOrBytes]:
    for parser in JSON_PARSERS:
    log_msg = parser(line)
    if log_msg is not None:
    return log_msg
    return line


    def format_log_msg(log_msg, original_line):
    def format_log_msg(
    log_msg: Union[ParsedLog, StrOrBytes], original_line: StrOrBytes, args: Arguments
    ) -> Optional[StrOrBytes]:
    try:
    if not isinstance(log_msg, dict):
    raise ValueError(f"expected dict, received {type(log_msg)}")
    msg = FORMAT.format(**{"name": "|"} | log_msg)

    msg = ""
    if args.show_message:
    msg = FORMAT.format(**{"name": "|"} | log_msg)

    extra_items = {
    key: value
    for key, value in log_msg.items()
    if not (key in KNOWN_KEYS or key in IGNORED_KEYS)
    if not (key in KNOWN_KEYS or key in args.ignored_keys)
    }
    if len(extra_items) > 0:
    msg += "\n" + json.dumps(extra_items, indent=JSON_INDENT)
    for field in KEYS_WITH_NEWLINES:
    if field in log_msg:
    if log_msg[field] == "NoneType: None":
    continue
    exc_sep = "\n"
    msg += exc_sep + log_msg[field]

    if args.show_extra_keys and len(extra_items) > 0:
    # Add newlines between msg components
    if args.show_message:
    msg += args.extra_keys_sep

    msg += json.dumps(extra_items, indent=args.json_indent)

    if args.show_stack_traces:
    for i, field in enumerate(KEYS_WITH_STACK_TRACES):
    if field in log_msg:
    if log_msg[field] == "NoneType: None":
    continue

    # Add newlines between msg components
    if i == 0 and not (args.show_extra_keys or args.show_message):
    exc_sep = ""
    else:
    exc_sep = "\n"

    msg += exc_sep + log_msg[field]

    if len(msg) == 0:
    return None
    return msg

    except (ValueError, KeyError):
    # Exclude trailing newline
    return original_line[:-1]
    if args.show_non_json:
    # Exclude trailing newline
    return original_line[:-1]


    def parse_args() -> Arguments:
    parser = ArgumentParser(
    description="Convert JSON logs into readable output."
    "The JSON keys are assumed to come from Python."
    "Stack trace and exception information will be searched for in the keys: "
    + (",".join(KEYS_WITH_STACK_TRACES))
    )
    parser.add_argument("files", nargs="*")
    parser.add_argument(
    "--no-stack-traces",
    help="Do not print Python stack traces and exception information.",
    action="store_true",
    )
    parser.add_argument(
    "--no-extra-keys",
    help="Do not print extra keys in a JSON log line",
    action="store_true",
    )
    parser.add_argument(
    "--no-message",
    help="Do not print the main message. Only the stack traces or extra keys",
    action="store_true",
    )

    non_json_group = parser.add_mutually_exclusive_group()
    non_json_group.add_argument(
    "--no-non-json",
    help="Hide non-JSON input instead of printing it as is",
    action="store_true",
    )
    non_json_group.add_argument(
    "--only-non-json",
    help="Only show non-JSON input",
    action="store_true",
    )

    # TODO
    # only show msgs with stack traces
    # convenience for --no-extra --no-message --no-non

    # TODO
    # parser.add_argument(
    # "--strip-stack-traces",
    # help="Print only the first and last lines of Python stack traces and exception information.",
    # action="store_true",
    # )
    # parser.add_argument(
    # "--replace-stack-trace-newlines",
    # help="Print Python stack traces and exception information, but replace newlines with the argument.",
    # action="store",
    # )

    parser.add_argument(
    "--extra-keys-on-same-line",
    help="Print extra keys on the same line as the formatted log message",
    action="store_true",
    )

    # TODO
    # one line with stripped stack traces
    parser.add_argument(
    "--one-line-output",
    help="Print the formatted log message on a single line.",
    action="store_true",
    )

    parser.add_argument(
    "--ignore",
    help="Specify a key which must be ignored. Can be given multiple times",
    action="append",
    default=None,
    )
    # TODO
    # parser.add_argument("--message-format")
    # parser.add_argument("--known-keys")

    json_formatting = parser.add_mutually_exclusive_group()
    json_formatting.add_argument(
    "--json-indent",
    help="How much to indent extra keys when pretty-printing them as JSON.",
    default=DEFAULT_JSON_INDENT,
    )
    json_formatting.add_argument(
    "--compact-json",
    help="Print extra keys in a compact format",
    action="store_true",
    )

    args = parser.parse_args()
    application_args = Arguments(
    files=args.files,
    show_stack_traces=not args.no_stack_traces and not args.only_non_json,
    show_extra_keys=not args.no_extra_keys and not args.only_non_json,
    show_message=not args.no_message and not args.only_non_json,
    show_non_json=args.only_non_json or not args.no_non_json,
    extra_keys_sep=" "
    if args.one_line_output or args.extra_keys_on_same_line
    else "\n",
    json_indent=None
    if args.one_line_output or args.compact_json
    else args.json_indent,
    ignored_keys=DEFAULT_IGNORED_KEYS if args.ignore is None else args.ignore,
    )
    # TODO debug output
    # print(application_args)
    return application_args


    def main():
    for line in fileinput.input():
    print(format_log_msg(parse(line), line))
    args = parse_args()
    with fileinput.input(files=args.files, mode="r") as fileinputinput:
    for line in fileinputinput:
    parsed_json = parse(line)
    readable_message = format_log_msg(parsed_json, line, args)
    if readable_message is not None:
    print(readable_message)


    if __name__ == "__main__":
  6. roguh revised this gist Nov 29, 2021. 1 changed file with 45 additions and 13 deletions.
    58 changes: 45 additions & 13 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,51 +1,83 @@
    #!/usr/bin/env python3
    import fileinput
    import json
    from pprint import pformat

    FORMAT = "{asctime} {severity} {module}:{funcName}:{lineno} {message}"
    # add ability to not output exc_info or extra keys or output on same line
    # add ability to only lo JSON formatted or only log non-JSON

    JSON_INDENT=4

    FORMAT = "{asctime} {severity} {name} {module}:{funcName}:{lineno} {message}"

    KEYS_WITH_NEWLINES = [
    "exc_info",
    "stack_info",
    ]
    KNOWN_KEYS = [
    "name",
    "asctime",
    "severity",
    "module",
    "funcName",
    "lineno",
    "message",
    "exc_info",
    ]
    ] + KEYS_WITH_NEWLINES
    IGNORED_KEYS = [
    "filename",
    "pathname",
    ]


    def parse_plain_json(line):
    try:
    return json.loads(line)
    except json.decoder.JSONDecodeError:
    return None


    def parse_loki_json(line):
    try:
    return parse_plain_json(line.split("\t", 1)[1])
    except IndexError:
    return None


    def parse(line):
    for parser in [parse_plain_json, parse_loki_json]:
    log_msg = parser(line)
    if log_msg is not None:
    return log_msg
    return line


    def format_log_msg(log_msg, original_line):
    try:
    log_msg = json.loads(line)
    if not isinstance(log_msg, dict):
    raise ValueError(f"expected dict, received {type(log_msg)}")
    msg = FORMAT.format(**log_msg)
    msg = FORMAT.format(**{"name": "|"} | log_msg)
    extra_items = {
    key: value
    for key, value in log_msg.items()
    if not (key in KNOWN_KEYS or key in IGNORED_KEYS)
    }
    if len(extra_items) > 0:
    msg += "\n" + pformat(extra_items)
    if "exc_info" in log_msg:
    exc_sep = "\n" if len(extra_items) > 0 else " "
    msg += exc_sep + log_msg["exc_info"]
    msg += "\n" + json.dumps(extra_items, indent=JSON_INDENT)
    for field in KEYS_WITH_NEWLINES:
    if field in log_msg:
    if log_msg[field] == "NoneType: None":
    continue
    exc_sep = "\n"
    msg += exc_sep + log_msg[field]
    return msg

    except (json.decoder.JSONDecodeError, ValueError, KeyError):
    except (ValueError, KeyError):
    # Exclude trailing newline
    return line[:-1]
    return original_line[:-1]


    def main():
    for line in fileinput.input():
    print(parse(line))
    print(format_log_msg(parse(line), line))


    if __name__ == "__main__":
  7. roguh revised this gist Oct 8, 2021. 1 changed file with 25 additions and 2 deletions.
    27 changes: 25 additions & 2 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,18 +1,41 @@
    #!/usr/bin/env python3
    import fileinput
    import json
    from pprint import pformat

    FORMAT = "{asctime} {severity} {module}:{funcName}:{lineno} {message}"

    KNOWN_KEYS = [
    "asctime",
    "severity",
    "module",
    "funcName",
    "lineno",
    "message",
    "exc_info",
    ]
    IGNORED_KEYS = [
    "filename",
    "pathname",
    ]


    def parse(line):
    try:
    log_msg = json.loads(line)
    if not isinstance(log_msg, dict):
    raise ValueError(f"expected dict, received {type(log_msg)}")
    msg = FORMAT.format(**log_msg)
    extra_items = {
    key: value
    for key, value in log_msg.items()
    if not (key in KNOWN_KEYS or key in IGNORED_KEYS)
    }
    if len(extra_items) > 0:
    msg += "\n" + pformat(extra_items)
    if "exc_info" in log_msg:
    msg += " " + log_msg["exc_info"]
    exc_sep = "\n" if len(extra_items) > 0 else " "
    msg += exc_sep + log_msg["exc_info"]
    return msg

    except (json.decoder.JSONDecodeError, ValueError, KeyError):
    @@ -26,4 +49,4 @@ def main():


    if __name__ == "__main__":
    main()
    main()
  8. roguh created this gist Jul 9, 2021.
    29 changes: 29 additions & 0 deletions jsonlogparser.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,29 @@
    #!/usr/bin/env python3
    import fileinput
    import json

    FORMAT = "{asctime} {severity} {module}:{funcName}:{lineno} {message}"


    def parse(line):
    try:
    log_msg = json.loads(line)
    if not isinstance(log_msg, dict):
    raise ValueError(f"expected dict, received {type(log_msg)}")
    msg = FORMAT.format(**log_msg)
    if "exc_info" in log_msg:
    msg += " " + log_msg["exc_info"]
    return msg

    except (json.decoder.JSONDecodeError, ValueError, KeyError):
    # Exclude trailing newline
    return line[:-1]


    def main():
    for line in fileinput.input():
    print(parse(line))


    if __name__ == "__main__":
    main()