Skip to content

Instantly share code, notes, and snippets.

@miraculixx

miraculixx/markup.py

Last active April 12, 2023 17:13
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save miraculixx/900a28a94c375b7259b1f711b93417d3 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save miraculixx/900a28a94c375b7259b1f711b93417d3 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. miraculixx revised this gist Jun 13, 2020. 1 changed file with 7 additions and 0 deletions.
    7 changes: 7 additions & 0 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,10 @@
    # (c) miraculixx, licensed as by the terms of WTFPL, http://www.wtfpl.net/txt/copying/
    # License: DO WHATEVER YOU WANT TO with this code.
    #
    # THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
    # IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
    # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
    #
    from io import StringIO

    from contextlib import contextmanager
  2. miraculixx revised this gist Jun 13, 2020. 1 changed file with 21 additions and 24 deletions.
    45 changes: 21 additions & 24 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -7,18 +7,14 @@ def markup(file_or_str, parsers=None, direct=True, on_error='warn', default=None
    **kwargs):
    """
    a safe markup file reader, accepts json and yaml, returns a dict or a default
    Usage:
    file_or_str = filename|file-like|markup-str
    # try, return None if not readable, will issue a warning in the log
    data = markup(file_or_str)
    # try, return some other default, will issue a warning in the log
    data = markup(file_or_str, default={})
    # try and fail
    data = markup(file_or_str, on_error='fail')
    Args:
    file_or_str (None, str, file-like): any file-like, can be
    any object that the parsers accept
    @@ -31,10 +27,8 @@ def markup(file_or_str, parsers=None, direct=True, on_error='warn', default=None
    default (obj): return the obj if the input is None or in case of on_error=warn or silent
    **kwargs (dict): any kwargs passed on to read(), any entry that matches a parser
    function's module name will be passed on to the parser
    Returns:
    data parsed or default
    markups.exceptions contains list of exceptions raised, if any
    """
    import json
    @@ -99,22 +93,25 @@ def raises(fn, wanted_ex):

    if __name__ == '__main__':
    import sys
    from pprint import pprint

    if len(sys.argv):
    return markup(sys.argv[0])

    assert markup('foo: bar') == {'foo': 'bar'}
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
    assert markup('test.txt') == {'foo': 'bar'}
    assert raises(lambda : markup('xtest.txt', on_error='fail') == {'foo': 'bar'}, ValueError)
    assert isinstance(markup('xtest.txt', on_error='silent', default=markup).exceptions[0], FileNotFoundError)
    assert markup('failed: - bar') is None
    assert markup('failed: - bar', default={}) == {}
    assert raises(lambda : markup('failed: - bar', on_error='fail'), ValueError)
    assert markup('failed: - bar', on_error='silent') is None
    assert markup('failed: - bar', on_error='silent', default="nothing") == 'nothing'
    assert markup('', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='silent', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='fail')
    assert markup('foo: bar', direct=False) == markup and markup.read() == {'foo': 'bar'}
    if len(sys.argv) > 1:
    pprint(markup(sys.argv[1], on_error='fail'))
    else:
    print("testing...")
    assert markup('foo: bar') == {'foo': 'bar'}
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
    #assert markup('test.txt') == {'foo': 'bar'}
    assert raises(lambda : markup('xtest.txt', on_error='fail') == {'foo': 'bar'}, ValueError)
    assert isinstance(markup('xtest.txt', on_error='silent', default=markup).exceptions[0], FileNotFoundError)
    assert markup('failed: - bar') is None
    assert markup('failed: - bar', default={}) == {}
    assert raises(lambda : markup('failed: - bar', on_error='fail'), ValueError)
    assert markup('failed: - bar', on_error='silent') is None
    assert markup('failed: - bar', on_error='silent', default="nothing") == 'nothing'
    assert markup('', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='silent', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='fail')
    assert markup('foo: bar', direct=False) == markup and markup.read() == {'foo': 'bar'}
    print("ok. use as python markup.py '<file or markup>'")
  3. miraculixx revised this gist Jun 13, 2020. 1 changed file with 5 additions and 0 deletions.
    5 changes: 5 additions & 0 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -98,6 +98,11 @@ def raises(fn, wanted_ex):
    return True

    if __name__ == '__main__':
    import sys

    if len(sys.argv):
    return markup(sys.argv[0])

    assert markup('foo: bar') == {'foo': 'bar'}
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
  4. miraculixx revised this gist Jun 13, 2020. 1 changed file with 16 additions and 16 deletions.
    32 changes: 16 additions & 16 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -97,19 +97,19 @@ def raises(fn, wanted_ex):
    raise ValueError("did not raise {}".format(wanted_ex))
    return True


    assert markup('foo: bar') == {'foo': 'bar'}
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
    assert markup('test.txt') == {'foo': 'bar'}
    assert raises(lambda : markup('xtest.txt', on_error='fail') == {'foo': 'bar'}, ValueError)
    assert isinstance(markup('xtest.txt', on_error='silent', default=markup).exceptions[0], FileNotFoundError)
    assert markup('failed: - bar') is None
    assert markup('failed: - bar', default={}) == {}
    assert raises(lambda : markup('failed: - bar', on_error='fail'), ValueError)
    assert markup('failed: - bar', on_error='silent') is None
    assert markup('failed: - bar', on_error='silent', default="nothing") == 'nothing'
    assert markup('', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='silent', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='fail')
    assert markup('foo: bar', direct=False) == markup and markup.read() == {'foo': 'bar'}
    if __name__ == '__main__':
    assert markup('foo: bar') == {'foo': 'bar'}
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
    assert markup('test.txt') == {'foo': 'bar'}
    assert raises(lambda : markup('xtest.txt', on_error='fail') == {'foo': 'bar'}, ValueError)
    assert isinstance(markup('xtest.txt', on_error='silent', default=markup).exceptions[0], FileNotFoundError)
    assert markup('failed: - bar') is None
    assert markup('failed: - bar', default={}) == {}
    assert raises(lambda : markup('failed: - bar', on_error='fail'), ValueError)
    assert markup('failed: - bar', on_error='silent') is None
    assert markup('failed: - bar', on_error='silent', default="nothing") == 'nothing'
    assert markup('', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='silent', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='fail')
    assert markup('foo: bar', direct=False) == markup and markup.read() == {'foo': 'bar'}
  5. miraculixx revised this gist Jun 13, 2020. 1 changed file with 31 additions and 15 deletions.
    46 changes: 31 additions & 15 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,14 +1,12 @@
    # (c) miraculixx licensed by the terms of WTFPL, see http://www.wtfpl.net/
    from io import StringIO

    from contextlib import contextmanager
    import re

    def markup(file_or_str, parsers=None, direct=True, **kwargs):
    def markup(file_or_str, parsers=None, direct=True, on_error='warn', default=None, msg='could not read {}',
    **kwargs):
    """
    a safe markup file reader, accepts json and yaml, returns a dict or a default
    Requirements:
    $ pip install pyyaml
    Usage:
    file_or_str = filename|file-like|markup-str
    @@ -26,6 +24,13 @@ def markup(file_or_str, parsers=None, direct=True, **kwargs):
    any object that the parsers accept
    parsers (list): the list of parsers, defaults to json.load, yaml.safe_load,
    json.loads
    direct (bool): if True returns the result, else returns markup (self). then use
    .read() to actually read the contents
    on_error (str): 'fail' raises a ValueError in case of error, 'warn' outputs a warning to the log,
    and returns the default, 'silent' returns the default. Defaults to warn
    default (obj): return the obj if the input is None or in case of on_error=warn or silent
    **kwargs (dict): any kwargs passed on to read(), any entry that matches a parser
    function's module name will be passed on to the parser
    Returns:
    data parsed or default
    @@ -34,6 +39,7 @@ def markup(file_or_str, parsers=None, direct=True, **kwargs):
    """
    import json
    import yaml
    import logging

    parsers = parsers or (json.load, yaml.safe_load, json.loads)
    # path-like regex
    @@ -57,19 +63,19 @@ def fopen(filein, *args, **kwargs):
    throw = lambda ex: (_ for _ in ()).throw(ex)
    exceptions = []

    def read(on_error='warn', default=None, msg='could not read {}'):
    def read(**kwargs):
    if file_or_str is None:
    return default
    for fn in parsers:
    with fopen(file_or_str) as fin:
    try:
    try:
    with fopen(file_or_str) as fin:
    if hasattr(fin, 'seek'):
    fin.seek(0)
    data = fn(fin)
    except Exception as e:
    exceptions.append(e)
    else:
    return data
    data = fn(fin, **kwargs.get(fn.__module__, {}))
    except Exception as e:
    exceptions.append(e)
    else:
    return data
    # nothing worked so far
    actions = {
    'fail': lambda: throw(ValueError("Reading {} caused exceptions {}".format(file_or_str, exceptions))),
    @@ -86,7 +92,7 @@ def raises(fn, wanted_ex):
    try:
    fn()
    except Exception as e:
    assert isinstance(e, wanted_ex)
    assert isinstance(e, wanted_ex), "expected {}, raised {} instead".format(wanted_ex, e)
    else:
    raise ValueError("did not raise {}".format(wanted_ex))
    return True
    @@ -96,4 +102,14 @@ def raises(fn, wanted_ex):
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
    assert markup('test.txt') == {'foo': 'bar'}
    assert raises(lambda : markup('xtest.txt') == {'foo': 'bar'}, FileNotFoundError)
    assert raises(lambda : markup('xtest.txt', on_error='fail') == {'foo': 'bar'}, ValueError)
    assert isinstance(markup('xtest.txt', on_error='silent', default=markup).exceptions[0], FileNotFoundError)
    assert markup('failed: - bar') is None
    assert markup('failed: - bar', default={}) == {}
    assert raises(lambda : markup('failed: - bar', on_error='fail'), ValueError)
    assert markup('failed: - bar', on_error='silent') is None
    assert markup('failed: - bar', on_error='silent', default="nothing") == 'nothing'
    assert markup('', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='silent', default="nothing") == 'nothing'
    assert lambda : markup('.', on_error='fail')
    assert markup('foo: bar', direct=False) == markup and markup.read() == {'foo': 'bar'}
  6. miraculixx revised this gist Jun 13, 2020. 1 changed file with 3 additions and 0 deletions.
    3 changes: 3 additions & 0 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -6,6 +6,9 @@
    def markup(file_or_str, parsers=None, direct=True, **kwargs):
    """
    a safe markup file reader, accepts json and yaml, returns a dict or a default
    Requirements:
    $ pip install pyyaml
    Usage:
    file_or_str = filename|file-like|markup-str
  7. miraculixx revised this gist Jun 13, 2020. No changes.
  8. miraculixx created this gist Jun 13, 2020.
    96 changes: 96 additions & 0 deletions markup.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,96 @@
    # (c) miraculixx licensed by the terms of WTFPL, see http://www.wtfpl.net/
    from io import StringIO
    from contextlib import contextmanager
    import re

    def markup(file_or_str, parsers=None, direct=True, **kwargs):
    """
    a safe markup file reader, accepts json and yaml, returns a dict or a default
    Usage:
    file_or_str = filename|file-like|markup-str
    # try, return None if not readable, will issue a warning in the log
    data = markup(file_or_str)
    # try, return some other default, will issue a warning in the log
    data = markup(file_or_str, default={})
    # try and fail
    data = markup(file_or_str, on_error='fail')
    Args:
    file_or_str (None, str, file-like): any file-like, can be
    any object that the parsers accept
    parsers (list): the list of parsers, defaults to json.load, yaml.safe_load,
    json.loads
    Returns:
    data parsed or default
    markups.exceptions contains list of exceptions raised, if any
    """
    import json
    import yaml

    parsers = parsers or (json.load, yaml.safe_load, json.loads)
    # path-like regex
    # - \/? leading /, optional
    # - (?P<path>\w+/?)* any path-part followed by /, repeated 0 - n times
    # - (?P<ext>(\w*\.?\w*)+ any file.ext, at least once
    pathlike = lambda s: re.match(r"^\/?(?P<path>\w+/?)*(?P<ext>(\w*\.?\w*))$", s)

    @contextmanager
    def fopen(filein, *args, **kwargs):
    # https://stackoverflow.com/a/55032634/890242
    if isinstance(filein, str) and pathlike(filein): # filename
    with open(filein, *args, **kwargs) as f:
    yield f
    elif isinstance(filein, str): # some other string, make a file-like
    yield StringIO(filein)
    else:
    # file-like object
    yield filein

    throw = lambda ex: (_ for _ in ()).throw(ex)
    exceptions = []

    def read(on_error='warn', default=None, msg='could not read {}'):
    if file_or_str is None:
    return default
    for fn in parsers:
    with fopen(file_or_str) as fin:
    try:
    if hasattr(fin, 'seek'):
    fin.seek(0)
    data = fn(fin)
    except Exception as e:
    exceptions.append(e)
    else:
    return data
    # nothing worked so far
    actions = {
    'fail': lambda: throw(ValueError("Reading {} caused exceptions {}".format(file_or_str, exceptions))),
    'warn': lambda: logging.warning(msg.format(file_or_str)) or default,
    'silent': lambda: default,
    }
    return actions[on_error]()

    markup.read = read
    markup.exceptions = exceptions
    return markup.read(**kwargs) if direct else markup

    def raises(fn, wanted_ex):
    try:
    fn()
    except Exception as e:
    assert isinstance(e, wanted_ex)
    else:
    raise ValueError("did not raise {}".format(wanted_ex))
    return True


    assert markup('foo: bar') == {'foo': 'bar'}
    assert markup('{"foo": "bar"}') == {'foo': 'bar'}
    assert markup(StringIO("foo: bar")) == {'foo': 'bar'}
    assert markup('test.txt') == {'foo': 'bar'}
    assert raises(lambda : markup('xtest.txt') == {'foo': 'bar'}, FileNotFoundError)