Implicit resolvers and robust representers for human-friendly tuple and np.array support in ruamel.yaml

Question

I have a project where the user is expected to manually write a yaml file. This yaml file might have some of its entries formatted as tuples or numpy arrays. We distinguish tuples and list internally in python to provide a convenient interface to the user, e.g. (1, 2, 3) is different than [1, 2, 3].

For convenience, I'd like the user to be able to enter a tuple directly using parenthesis, like so name: (1,2,3). I'd also like the user to be able to provide with numpy arrays by entering something like other_name: np.array([1,2,3]). I know this won't preserve exact numerical accuracy of the numpy arrays, but we determined that this is a fair compromise for the improved user experience.

I'm using ruamel.yaml, mainly because it preserves cmets.

I managed to do something that works, but it's does not feel "right" to me, especially the Resolving part. There's basically no implicit resolver and I'm using a dirty eval instead. I did manage to find some information about implicit resolvers in ruamel.yaml online, on SO, and by rummaging through the source, but I could not really make sense of it.

Here's a minimal working example, with cmets pointing out where I feel like the implementation is not robust or unclean.

import sys
import numpy as np
import ruamel.yaml


def _tupleRepresenter(dumper, data):
    # TODO: Make this more robust
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', str(data))


def _numpyRepresenter(dumper, data):
    # TODO: Make this more robust
    as_string = 'np.array(' + np.array2string(data, max_line_width=np.inf, precision=16, prefix='np.array(', separator=', ', suffix=')') + ')'
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', as_string)


def load_yaml(file):
    # TODO: Resolve tuples and arrays properly when loading
    yaml = ruamel.yaml.YAML()
    yaml.Representer.add_representer(tuple, _tupleRepresenter)
    yaml.Representer.add_representer(np.ndarray, _numpyRepresenter)
    return yaml.load(file)


def dump_yaml(data, file):
    yaml = ruamel.yaml.YAML()
    yaml.Representer.add_representer(tuple, _tupleRepresenter)
    yaml.Representer.add_representer(np.ndarray, _numpyRepresenter)
    return yaml.dump(data, file)


yaml_file = """
test_tuple: (1, 2, 3)
test_array: np.array([4,5,6])
"""

data = load_yaml(yaml_file)

data['test_tuple'] = eval(data['test_tuple']) # This feels dirty
data['test_array'] = eval(data['test_array']) # This feels dirty

dump_yaml(data, sys.stdout)
# test_tuple: (1, 2, 3)
# test_array: np.array([4, 5, 6])

I welcome any help on improving this implementation with a proper implicit resolver, with robusts representers, and generally using ruamel.yaml more like it is intended to be.

---

Update:

With help from the cmets, I managed to do something that works almost completely. Let's ignore that I'd need to write a proper non-eval parser for now.

The only issue left is that the new tags are now exported as strings, so they are not properly interpreted when reloading. They become strings instead and they won't survive many roundtrips.

How can I avoid that?

Here's a minimal working example:

import sys
import numpy as np
import ruamel.yaml

# TODO: Replace evals by actual parsing
# TODO: Represent custom types without the string quotes

_tuple_re = "^(?:\((?:.|\n|\r)*,(?:.|\n|\r)*\){1}(?: |\n|\r)*$)"
_array_re = "^(?:(np\.|)array\(\[(?:.|\n|\r)*,(?:.|\n|\r)*\]\){1}(?: |\n|\r)*$)"
_complex_re = "^(?:(?:\d+(?:(?:\.\d+)?(?:e[+\-]\d+)?)?)?(?: *[+\-] *))?(?:\d+(?:(?:\.\d+)?(?:e[+\-]\d+)?)?)?[jJ]$"


def _tuple_constructor(self, node):
    return eval(self.construct_scalar(node))


def _array_constructor(self, node):
    value = node.value
    if not value.startswith('np.'):
        value = 'np.' + value
    return eval(value)


def _complex_constructor(self, node):
    return eval(node.value)


def _tuple_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', str(data))


def _array_representer(dumper, data):
    as_string = 'np.array(' + np.array2string(data, max_line_width=np.inf, precision=16, prefix='np.array(', separator=', ', suffix=')') + ')'
    as_string = as_string.replace(' ', '').replace(',', ', ')
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', as_string)


def _complex_representer(dumper, data):
    repr = str(data).replace('(', '').replace(')', '')
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', repr)


custom_types = {
    '!tuple':   {'re':_tuple_re,   'constructor': _tuple_constructor,   'representer':_tuple_representer,   'type': tuple,      'first':list('(')             },
    '!nparray': {'re':_array_re,   'constructor': _array_constructor,   'representer':_array_representer,   'type': np.ndarray, 'first':list('an')            },
    '!complex': {'re':_complex_re, 'constructor': _complex_constructor, 'representer':_complex_representer, 'type': complex,    'first':list('0123456789+-jJ')},
}


def load_yaml(file):
    yaml = ruamel.yaml.YAML()
    for tag,ct in custom_types.items():
        yaml.Constructor.add_constructor(tag, ct['constructor'])
        yaml.Resolver.add_implicit_resolver(tag, ruamel.yaml.util.RegExp(ct['re']), ct['first'])
        yaml.Representer.add_representer(ct['type'], ct['representer'])
    return yaml.load(file)


def dump_yaml(data, file):
    yaml = ruamel.yaml.YAML()
    for tag,ct in custom_types.items():
        yaml.Constructor.add_constructor(tag, ct['constructor'])
        yaml.Resolver.add_implicit_resolver(tag, ruamel.yaml.util.RegExp(ct['re']), ct['first'])
        yaml.Representer.add_representer(ct['type'], ct['representer'])
    return yaml.dump(data, file)

yaml_file = """
test_tuple: (1, 2, 3)
test_array: array([4.0,5+0j,6.0j])
test_complex: 3 + 2j
"""

data = load_yaml(yaml_file)

dump_yaml(data, sys.stdout)
# test_tuple: '(1, 2, 3)'
# test_array: 'np.array([4.+0.j, 5.+0.j, 0.+6.j])'
# test_complex: '3+2j'

Thank you!

Answer 1

A representer in ruamel.yaml is used to dump a Python type in a specific way as YAML, you cannot normally use it to create a Python type from some part of YAML. For the latter you need a constructor.

A constructor can be explicit, using a tag ( e.g. !!float, a list of these can be found here ), or implicit, i.e. recognise the input which in ruamel.yaml is done on scalars initiallly using regular expressions.

Your example seems to border on the need of expanding the collection types of YAML beyond the mapping and dict and I don't think you will succeed without rewriting most of the ruamel.yaml code. What I do recommend is that you write the code construct your numpy array from tagged input like the following first:

test_tuple: !np.array [1, 2, 3]

even though you don't want your users to write things that way. And also dump numpy arrays using that tag.

The next step would then to write a constructor that matches scalars that start and end with a opening and closing parenthesis, or start with np.array([ and end in ]) (even though there is a [ in there this doesn't start a sequence when in the middle of a scalar. You should keep track of which of the two formats was used to construct the NumPy array (e.g. using some unique attribute that has a tri-state for tagged-, parenthesis-, or np.array- input). You will need to parse the matching scalar, but you don't need to do that using eval(). For altneratives, look e.g. at the processing of !timestamp. Although your examples only have an array of integers, you might want to look at (also) accepting floats.

Once you have those additional non-tagged constructors, you can adapt the representer for the NumPy array to use the non-tagged formats depending on the attribute.

A good example for the above are the round-trip processing of floats (preserving scientific notation) and the aforementioned timestamps.

Answer 2

With help from Anthon in the cmets, and reading through his ruamel.yaml source, I managed to answer my question.

I'm putting a minimum viable solution here for reference. It'd probably be a good idea to replace the evals with an actual parser to avoid exploits if this is ever to be executed on a yaml file form a source you don't trust.

import sys
import numpy as np
import ruamel.yaml

from ruamel.yaml.comments import TaggedScalar

# TODO: Replace evals by actual parsing

_tuple_re = "^(?:((?:.|
|
)*,(?:.|
|
)*){1}(?: |
|
)*$)"
_array_re = "^(?:(np.|)array([(?:.|
|
)*,(?:.|
|
)*]){1}(?: |
|
)*$)"


def _tuple_constructor(self, node):
    return eval(self.construct_scalar(node))


def _array_constructor(self, node):
    value = node.value
    if not value.startswith('np.'):
        value = 'np.' + value
    return eval(value)


def _tuple_representer(dumper, data):
    repr = str(data)
    return dumper.represent_tagged_scalar(TaggedScalar(repr, style=None, tag='!tuple'))


def _array_representer(dumper, data):
    repr = 'np.array(' + np.array2string(data, max_line_width=np.inf, precision=16, prefix='np.array(', separator=', ', suffix=')') + ')'
    repr = repr.replace(' ', '').replace(',', ', ')
    return dumper.represent_tagged_scalar(TaggedScalar(repr, style=None, tag='!nparray'))


custom_types = {
    '!tuple':   {'re':_tuple_re,   'constructor': _tuple_constructor,   'representer':_tuple_representer,   'type': tuple,      'first':list('(')             },
    '!nparray': {'re':_array_re,   'constructor': _array_constructor,   'representer':_array_representer,   'type': np.ndarray, 'first':list('an')            },
}


def load_yaml(file):
    yaml = ruamel.yaml.YAML()
    for tag,ct in custom_types.items():
        yaml.Constructor.add_constructor(tag, ct['constructor'])
        yaml.Resolver.add_implicit_resolver(tag, ruamel.yaml.util.RegExp(ct['re']), ct['first'])
        yaml.Representer.add_representer(ct['type'], ct['representer'])
    return yaml.load(file)


def dump_yaml(data, file):
    yaml = ruamel.yaml.YAML()
    for tag,ct in custom_types.items():
        yaml.Constructor.add_constructor(tag, ct['constructor'])
        yaml.Resolver.add_implicit_resolver(tag, ruamel.yaml.util.RegExp(ct['re']), ct['first'])
        yaml.Representer.add_representer(ct['type'], ct['representer'])
    return yaml.dump(data, file)

yaml_file = """
test_tuple: (1, 2, 3)
test_array: array([4.0,5+0j,6.0j])
"""

data = load_yaml(yaml_file)

dump_yaml(data, sys.stdout)
# test_tuple: (1, 2, 3)
# test_array: np.array([4.+0.j, 5.+0.j, 0.+6.j])

Answer 3

I wanted to expand on the given answer by adding a way that does not use eval(). It also probably fails in some cases, and could probably do with using the tolist() method of numpy arrays for the dumping.

Added here is a complex parser, necessary so that yaml can itself load the internals of the numpy array as a list, so that it has support for this.

Some slight cosmetic change in the load and dump functions also happened.

import sys
import numpy as np
import ruamel.yaml
import re

from ruamel.yaml.comments import TaggedScalar

_tuple_re = "^(?:((?:.|
|
)*,(?:.|
|
)*){1}(?: |
|
)*$)"
_array_re = "^(?:(np.|)array([(?:.|
|
)*,(?:.|
|
)*]){1}(?: |
|
)*$)"
_complex_re= "^(?:(?:(|)(?:.|
|
)*[+-]|)(?:.|
|
)*j(?:)|)$"


yaml = ruamel.yaml.YAML()

def setup_yaml(yaml):
    for tag,ct in custom_types.items():
        yaml.Constructor.add_constructor(tag, ct['constructor'])
        yaml.Resolver.add_implicit_resolver(tag, ruamel.yaml.util.RegExp(ct['re']), ct['first'])
        yaml.Representer.add_representer(ct['type'], ct['representer'])
    

def _tuple_constructor_safe(self,node):
    value = node.value
    value = re.sub("^(","[",value)
    value = re.sub(")$","]",value)
    safe_l = yaml.load(value)
    return tuple(safe_l)
    
def _complex_constructor(self,node):
    return complex(node.value)

def _array_constructor_safe(self,node):
    value = node.value
    value = re.sub("^(?:np.|)array(","",value)
    value = re.sub(")","",value)
    safe_l = yaml.load(value)
    return np.array(safe_l)


def _tuple_representer(dumper, data):
    repr = str(data)
    return dumper.represent_tagged_scalar(TaggedScalar(repr, style=None, tag='!tuple'))

def _complex_representer(dumper,data):
    repr = str(data)
    repr = re.sub("()","",repr)
    return dumper.represent_tagged_scalar(TaggedScalar(repr, style=None, tag='!complex'))

    


custom_types = {
    '!tuple':   {'re':_tuple_re,   'constructor': _tuple_constructor_safe,   'representer':_tuple_representer,   'type': tuple,      'first':list('(')             },
    '!nparray': {'re':_array_re,   'constructor': _array_constructor_safe,   'representer':_array_representer,   'type': np.ndarray, 'first':list('an')},
    '!complex': {'re':_complex_re,   'constructor': _complex_constructor,   'representer':_complex_representer,   'type': complex, 'first':None }
}

def load_yaml(file):
    return yaml.load(file)


def dump_yaml(data, file):
    return yaml.dump(data, file)


setup_yaml(yaml)

yaml_file = """
test_tuple: (1, 2, 3)
test_array: array([4.0,5+0j,6.0j])
"""

data = load_yaml(yaml_file)

dump_yaml(data, sys.stdout)
# test_tuple: (1, 2, 3)
# test_array: np.array([4.+0.j, 5.+0.j, 0.+6.j])