master
py 687 lines 25 KB
Raw
1 # SPDX-License-Identifier: MIT
2
3 __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
4 'ConstructorError']
5
6 from .error import *
7 from .nodes import *
8
9 import collections, datetime, base64, binascii, re, sys, types
10
11 class ConstructorError(MarkedYAMLError):
12 pass
13
14 class BaseConstructor:
15
16 yaml_constructors = {}
17 yaml_multi_constructors = {}
18
19 def __init__(self):
20 self.constructed_objects = {}
21 self.recursive_objects = {}
22 self.state_generators = []
23 self.deep_construct = False
24
25 def check_data(self):
26 # If there are more documents available?
27 return self.check_node()
28
29 def get_data(self):
30 # Construct and return the next document.
31 if self.check_node():
32 return self.construct_document(self.get_node())
33
34 def get_single_data(self):
35 # Ensure that the stream contains a single document and construct it.
36 node = self.get_single_node()
37 if node is not None:
38 return self.construct_document(node)
39 return None
40
41 def construct_document(self, node):
42 data = self.construct_object(node)
43 while self.state_generators:
44 state_generators = self.state_generators
45 self.state_generators = []
46 for generator in state_generators:
47 for dummy in generator:
48 pass
49 self.constructed_objects = {}
50 self.recursive_objects = {}
51 self.deep_construct = False
52 return data
53
54 def construct_object(self, node, deep=False):
55 if node in self.constructed_objects:
56 return self.constructed_objects[node]
57 if deep:
58 old_deep = self.deep_construct
59 self.deep_construct = True
60 if node in self.recursive_objects:
61 raise ConstructorError(None, None,
62 "found unconstructable recursive node", node.start_mark)
63 self.recursive_objects[node] = None
64 constructor = None
65 tag_suffix = None
66 if node.tag in self.yaml_constructors:
67 constructor = self.yaml_constructors[node.tag]
68 else:
69 for tag_prefix in self.yaml_multi_constructors:
70 if node.tag.startswith(tag_prefix):
71 tag_suffix = node.tag[len(tag_prefix):]
72 constructor = self.yaml_multi_constructors[tag_prefix]
73 break
74 else:
75 if None in self.yaml_multi_constructors:
76 tag_suffix = node.tag
77 constructor = self.yaml_multi_constructors[None]
78 elif None in self.yaml_constructors:
79 constructor = self.yaml_constructors[None]
80 elif isinstance(node, ScalarNode):
81 constructor = self.__class__.construct_scalar
82 elif isinstance(node, SequenceNode):
83 constructor = self.__class__.construct_sequence
84 elif isinstance(node, MappingNode):
85 constructor = self.__class__.construct_mapping
86 if tag_suffix is None:
87 data = constructor(self, node)
88 else:
89 data = constructor(self, tag_suffix, node)
90 if isinstance(data, types.GeneratorType):
91 generator = data
92 data = next(generator)
93 if self.deep_construct:
94 for dummy in generator:
95 pass
96 else:
97 self.state_generators.append(generator)
98 self.constructed_objects[node] = data
99 del self.recursive_objects[node]
100 if deep:
101 self.deep_construct = old_deep
102 return data
103
104 def construct_scalar(self, node):
105 if not isinstance(node, ScalarNode):
106 raise ConstructorError(None, None,
107 "expected a scalar node, but found %s" % node.id,
108 node.start_mark)
109 return node.value
110
111 def construct_sequence(self, node, deep=False):
112 if not isinstance(node, SequenceNode):
113 raise ConstructorError(None, None,
114 "expected a sequence node, but found %s" % node.id,
115 node.start_mark)
116 return [self.construct_object(child, deep=deep)
117 for child in node.value]
118
119 def construct_mapping(self, node, deep=False):
120 if not isinstance(node, MappingNode):
121 raise ConstructorError(None, None,
122 "expected a mapping node, but found %s" % node.id,
123 node.start_mark)
124 mapping = {}
125 for key_node, value_node in node.value:
126 key = self.construct_object(key_node, deep=deep)
127 if not isinstance(key, collections.Hashable):
128 raise ConstructorError("while constructing a mapping", node.start_mark,
129 "found unhashable key", key_node.start_mark)
130 value = self.construct_object(value_node, deep=deep)
131 mapping[key] = value
132 return mapping
133
134 def construct_pairs(self, node, deep=False):
135 if not isinstance(node, MappingNode):
136 raise ConstructorError(None, None,
137 "expected a mapping node, but found %s" % node.id,
138 node.start_mark)
139 pairs = []
140 for key_node, value_node in node.value:
141 key = self.construct_object(key_node, deep=deep)
142 value = self.construct_object(value_node, deep=deep)
143 pairs.append((key, value))
144 return pairs
145
146 @classmethod
147 def add_constructor(cls, tag, constructor):
148 if not 'yaml_constructors' in cls.__dict__:
149 cls.yaml_constructors = cls.yaml_constructors.copy()
150 cls.yaml_constructors[tag] = constructor
151
152 @classmethod
153 def add_multi_constructor(cls, tag_prefix, multi_constructor):
154 if not 'yaml_multi_constructors' in cls.__dict__:
155 cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
156 cls.yaml_multi_constructors[tag_prefix] = multi_constructor
157
158 class SafeConstructor(BaseConstructor):
159
160 def construct_scalar(self, node):
161 if isinstance(node, MappingNode):
162 for key_node, value_node in node.value:
163 if key_node.tag == 'tag:yaml.org,2002:value':
164 return self.construct_scalar(value_node)
165 return super().construct_scalar(node)
166
167 def flatten_mapping(self, node):
168 merge = []
169 index = 0
170 while index < len(node.value):
171 key_node, value_node = node.value[index]
172 if key_node.tag == 'tag:yaml.org,2002:merge':
173 del node.value[index]
174 if isinstance(value_node, MappingNode):
175 self.flatten_mapping(value_node)
176 merge.extend(value_node.value)
177 elif isinstance(value_node, SequenceNode):
178 submerge = []
179 for subnode in value_node.value:
180 if not isinstance(subnode, MappingNode):
181 raise ConstructorError("while constructing a mapping",
182 node.start_mark,
183 "expected a mapping for merging, but found %s"
184 % subnode.id, subnode.start_mark)
185 self.flatten_mapping(subnode)
186 submerge.append(subnode.value)
187 submerge.reverse()
188 for value in submerge:
189 merge.extend(value)
190 else:
191 raise ConstructorError("while constructing a mapping", node.start_mark,
192 "expected a mapping or list of mappings for merging, but found %s"
193 % value_node.id, value_node.start_mark)
194 elif key_node.tag == 'tag:yaml.org,2002:value':
195 key_node.tag = 'tag:yaml.org,2002:str'
196 index += 1
197 else:
198 index += 1
199 if merge:
200 node.value = merge + node.value
201
202 def construct_mapping(self, node, deep=False):
203 if isinstance(node, MappingNode):
204 self.flatten_mapping(node)
205 return super().construct_mapping(node, deep=deep)
206
207 def construct_yaml_null(self, node):
208 self.construct_scalar(node)
209 return None
210
211 bool_values = {
212 'yes': True,
213 'no': False,
214 'true': True,
215 'false': False,
216 'on': True,
217 'off': False,
218 }
219
220 def construct_yaml_bool(self, node):
221 value = self.construct_scalar(node)
222 return self.bool_values[value.lower()]
223
224 def construct_yaml_int(self, node):
225 value = self.construct_scalar(node)
226 value = value.replace('_', '')
227 sign = +1
228 if value[0] == '-':
229 sign = -1
230 if value[0] in '+-':
231 value = value[1:]
232 if value == '0':
233 return 0
234 elif value.startswith('0b'):
235 return sign*int(value[2:], 2)
236 elif value.startswith('0x'):
237 return sign*int(value[2:], 16)
238 elif value[0] == '0':
239 return sign*int(value, 8)
240 elif ':' in value:
241 digits = [int(part) for part in value.split(':')]
242 digits.reverse()
243 base = 1
244 value = 0
245 for digit in digits:
246 value += digit*base
247 base *= 60
248 return sign*value
249 else:
250 return sign*int(value)
251
252 inf_value = 1e300
253 while inf_value != inf_value*inf_value:
254 inf_value *= inf_value
255 nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
256
257 def construct_yaml_float(self, node):
258 value = self.construct_scalar(node)
259 value = value.replace('_', '').lower()
260 sign = +1
261 if value[0] == '-':
262 sign = -1
263 if value[0] in '+-':
264 value = value[1:]
265 if value == '.inf':
266 return sign*self.inf_value
267 elif value == '.nan':
268 return self.nan_value
269 elif ':' in value:
270 digits = [float(part) for part in value.split(':')]
271 digits.reverse()
272 base = 1
273 value = 0.0
274 for digit in digits:
275 value += digit*base
276 base *= 60
277 return sign*value
278 else:
279 return sign*float(value)
280
281 def construct_yaml_binary(self, node):
282 try:
283 value = self.construct_scalar(node).encode('ascii')
284 except UnicodeEncodeError as exc:
285 raise ConstructorError(None, None,
286 "failed to convert base64 data into ascii: %s" % exc,
287 node.start_mark)
288 try:
289 if hasattr(base64, 'decodebytes'):
290 return base64.decodebytes(value)
291 else:
292 return base64.decodestring(value)
293 except binascii.Error as exc:
294 raise ConstructorError(None, None,
295 "failed to decode base64 data: %s" % exc, node.start_mark)
296
297 timestamp_regexp = re.compile(
298 r'''^(?P<year>[0-9][0-9][0-9][0-9])
299 -(?P<month>[0-9][0-9]?)
300 -(?P<day>[0-9][0-9]?)
301 (?:(?:[Tt]|[ \t]+)
302 (?P<hour>[0-9][0-9]?)
303 :(?P<minute>[0-9][0-9])
304 :(?P<second>[0-9][0-9])
305 (?:\.(?P<fraction>[0-9]*))?
306 (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
307 (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
308
309 def construct_yaml_timestamp(self, node):
310 value = self.construct_scalar(node)
311 match = self.timestamp_regexp.match(node.value)
312 values = match.groupdict()
313 year = int(values['year'])
314 month = int(values['month'])
315 day = int(values['day'])
316 if not values['hour']:
317 return datetime.date(year, month, day)
318 hour = int(values['hour'])
319 minute = int(values['minute'])
320 second = int(values['second'])
321 fraction = 0
322 if values['fraction']:
323 fraction = values['fraction'][:6]
324 while len(fraction) < 6:
325 fraction += '0'
326 fraction = int(fraction)
327 delta = None
328 if values['tz_sign']:
329 tz_hour = int(values['tz_hour'])
330 tz_minute = int(values['tz_minute'] or 0)
331 delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
332 if values['tz_sign'] == '-':
333 delta = -delta
334 data = datetime.datetime(year, month, day, hour, minute, second, fraction)
335 if delta:
336 data -= delta
337 return data
338
339 def construct_yaml_omap(self, node):
340 # Note: we do not check for duplicate keys, because it's too
341 # CPU-expensive.
342 omap = []
343 yield omap
344 if not isinstance(node, SequenceNode):
345 raise ConstructorError("while constructing an ordered map", node.start_mark,
346 "expected a sequence, but found %s" % node.id, node.start_mark)
347 for subnode in node.value:
348 if not isinstance(subnode, MappingNode):
349 raise ConstructorError("while constructing an ordered map", node.start_mark,
350 "expected a mapping of length 1, but found %s" % subnode.id,
351 subnode.start_mark)
352 if len(subnode.value) != 1:
353 raise ConstructorError("while constructing an ordered map", node.start_mark,
354 "expected a single mapping item, but found %d items" % len(subnode.value),
355 subnode.start_mark)
356 key_node, value_node = subnode.value[0]
357 key = self.construct_object(key_node)
358 value = self.construct_object(value_node)
359 omap.append((key, value))
360
361 def construct_yaml_pairs(self, node):
362 # Note: the same code as `construct_yaml_omap`.
363 pairs = []
364 yield pairs
365 if not isinstance(node, SequenceNode):
366 raise ConstructorError("while constructing pairs", node.start_mark,
367 "expected a sequence, but found %s" % node.id, node.start_mark)
368 for subnode in node.value:
369 if not isinstance(subnode, MappingNode):
370 raise ConstructorError("while constructing pairs", node.start_mark,
371 "expected a mapping of length 1, but found %s" % subnode.id,
372 subnode.start_mark)
373 if len(subnode.value) != 1:
374 raise ConstructorError("while constructing pairs", node.start_mark,
375 "expected a single mapping item, but found %d items" % len(subnode.value),
376 subnode.start_mark)
377 key_node, value_node = subnode.value[0]
378 key = self.construct_object(key_node)
379 value = self.construct_object(value_node)
380 pairs.append((key, value))
381
382 def construct_yaml_set(self, node):
383 data = set()
384 yield data
385 value = self.construct_mapping(node)
386 data.update(value)
387
388 def construct_yaml_str(self, node):
389 return self.construct_scalar(node)
390
391 def construct_yaml_seq(self, node):
392 data = []
393 yield data
394 data.extend(self.construct_sequence(node))
395
396 def construct_yaml_map(self, node):
397 data = {}
398 yield data
399 value = self.construct_mapping(node)
400 data.update(value)
401
402 def construct_yaml_object(self, node, cls):
403 data = cls.__new__(cls)
404 yield data
405 if hasattr(data, '__setstate__'):
406 state = self.construct_mapping(node, deep=True)
407 data.__setstate__(state)
408 else:
409 state = self.construct_mapping(node)
410 data.__dict__.update(state)
411
412 def construct_undefined(self, node):
413 raise ConstructorError(None, None,
414 "could not determine a constructor for the tag %r" % node.tag,
415 node.start_mark)
416
417 SafeConstructor.add_constructor(
418 'tag:yaml.org,2002:null',
419 SafeConstructor.construct_yaml_null)
420
421 SafeConstructor.add_constructor(
422 'tag:yaml.org,2002:bool',
423 SafeConstructor.construct_yaml_bool)
424
425 SafeConstructor.add_constructor(
426 'tag:yaml.org,2002:int',
427 SafeConstructor.construct_yaml_int)
428
429 SafeConstructor.add_constructor(
430 'tag:yaml.org,2002:float',
431 SafeConstructor.construct_yaml_float)
432
433 SafeConstructor.add_constructor(
434 'tag:yaml.org,2002:binary',
435 SafeConstructor.construct_yaml_binary)
436
437 SafeConstructor.add_constructor(
438 'tag:yaml.org,2002:timestamp',
439 SafeConstructor.construct_yaml_timestamp)
440
441 SafeConstructor.add_constructor(
442 'tag:yaml.org,2002:omap',
443 SafeConstructor.construct_yaml_omap)
444
445 SafeConstructor.add_constructor(
446 'tag:yaml.org,2002:pairs',
447 SafeConstructor.construct_yaml_pairs)
448
449 SafeConstructor.add_constructor(
450 'tag:yaml.org,2002:set',
451 SafeConstructor.construct_yaml_set)
452
453 SafeConstructor.add_constructor(
454 'tag:yaml.org,2002:str',
455 SafeConstructor.construct_yaml_str)
456
457 SafeConstructor.add_constructor(
458 'tag:yaml.org,2002:seq',
459 SafeConstructor.construct_yaml_seq)
460
461 SafeConstructor.add_constructor(
462 'tag:yaml.org,2002:map',
463 SafeConstructor.construct_yaml_map)
464
465 SafeConstructor.add_constructor(None,
466 SafeConstructor.construct_undefined)
467
468 class Constructor(SafeConstructor):
469
470 def construct_python_str(self, node):
471 return self.construct_scalar(node)
472
473 def construct_python_unicode(self, node):
474 return self.construct_scalar(node)
475
476 def construct_python_bytes(self, node):
477 try:
478 value = self.construct_scalar(node).encode('ascii')
479 except UnicodeEncodeError as exc:
480 raise ConstructorError(None, None,
481 "failed to convert base64 data into ascii: %s" % exc,
482 node.start_mark)
483 try:
484 if hasattr(base64, 'decodebytes'):
485 return base64.decodebytes(value)
486 else:
487 return base64.decodestring(value)
488 except binascii.Error as exc:
489 raise ConstructorError(None, None,
490 "failed to decode base64 data: %s" % exc, node.start_mark)
491
492 def construct_python_long(self, node):
493 return self.construct_yaml_int(node)
494
495 def construct_python_complex(self, node):
496 return complex(self.construct_scalar(node))
497
498 def construct_python_tuple(self, node):
499 return tuple(self.construct_sequence(node))
500
501 def find_python_module(self, name, mark):
502 if not name:
503 raise ConstructorError("while constructing a Python module", mark,
504 "expected non-empty name appended to the tag", mark)
505 try:
506 __import__(name)
507 except ImportError as exc:
508 raise ConstructorError("while constructing a Python module", mark,
509 "cannot find module %r (%s)" % (name, exc), mark)
510 return sys.modules[name]
511
512 def find_python_name(self, name, mark):
513 if not name:
514 raise ConstructorError("while constructing a Python object", mark,
515 "expected non-empty name appended to the tag", mark)
516 if '.' in name:
517 module_name, object_name = name.rsplit('.', 1)
518 else:
519 module_name = 'builtins'
520 object_name = name
521 try:
522 __import__(module_name)
523 except ImportError as exc:
524 raise ConstructorError("while constructing a Python object", mark,
525 "cannot find module %r (%s)" % (module_name, exc), mark)
526 module = sys.modules[module_name]
527 if not hasattr(module, object_name):
528 raise ConstructorError("while constructing a Python object", mark,
529 "cannot find %r in the module %r"
530 % (object_name, module.__name__), mark)
531 return getattr(module, object_name)
532
533 def construct_python_name(self, suffix, node):
534 value = self.construct_scalar(node)
535 if value:
536 raise ConstructorError("while constructing a Python name", node.start_mark,
537 "expected the empty value, but found %r" % value, node.start_mark)
538 return self.find_python_name(suffix, node.start_mark)
539
540 def construct_python_module(self, suffix, node):
541 value = self.construct_scalar(node)
542 if value:
543 raise ConstructorError("while constructing a Python module", node.start_mark,
544 "expected the empty value, but found %r" % value, node.start_mark)
545 return self.find_python_module(suffix, node.start_mark)
546
547 def make_python_instance(self, suffix, node,
548 args=None, kwds=None, newobj=False):
549 if not args:
550 args = []
551 if not kwds:
552 kwds = {}
553 cls = self.find_python_name(suffix, node.start_mark)
554 if newobj and isinstance(cls, type):
555 return cls.__new__(cls, *args, **kwds)
556 else:
557 return cls(*args, **kwds)
558
559 def set_python_instance_state(self, instance, state):
560 if hasattr(instance, '__setstate__'):
561 instance.__setstate__(state)
562 else:
563 slotstate = {}
564 if isinstance(state, tuple) and len(state) == 2:
565 state, slotstate = state
566 if hasattr(instance, '__dict__'):
567 instance.__dict__.update(state)
568 elif state:
569 slotstate.update(state)
570 for key, value in slotstate.items():
571 setattr(object, key, value)
572
573 def construct_python_object(self, suffix, node):
574 # Format:
575 # !!python/object:module.name { ... state ... }
576 instance = self.make_python_instance(suffix, node, newobj=True)
577 yield instance
578 deep = hasattr(instance, '__setstate__')
579 state = self.construct_mapping(node, deep=deep)
580 self.set_python_instance_state(instance, state)
581
582 def construct_python_object_apply(self, suffix, node, newobj=False):
583 # Format:
584 # !!python/object/apply # (or !!python/object/new)
585 # args: [ ... arguments ... ]
586 # kwds: { ... keywords ... }
587 # state: ... state ...
588 # listitems: [ ... listitems ... ]
589 # dictitems: { ... dictitems ... }
590 # or short format:
591 # !!python/object/apply [ ... arguments ... ]
592 # The difference between !!python/object/apply and !!python/object/new
593 # is how an object is created, check make_python_instance for details.
594 if isinstance(node, SequenceNode):
595 args = self.construct_sequence(node, deep=True)
596 kwds = {}
597 state = {}
598 listitems = []
599 dictitems = {}
600 else:
601 value = self.construct_mapping(node, deep=True)
602 args = value.get('args', [])
603 kwds = value.get('kwds', {})
604 state = value.get('state', {})
605 listitems = value.get('listitems', [])
606 dictitems = value.get('dictitems', {})
607 instance = self.make_python_instance(suffix, node, args, kwds, newobj)
608 if state:
609 self.set_python_instance_state(instance, state)
610 if listitems:
611 instance.extend(listitems)
612 if dictitems:
613 for key in dictitems:
614 instance[key] = dictitems[key]
615 return instance
616
617 def construct_python_object_new(self, suffix, node):
618 return self.construct_python_object_apply(suffix, node, newobj=True)
619
620 Constructor.add_constructor(
621 'tag:yaml.org,2002:python/none',
622 Constructor.construct_yaml_null)
623
624 Constructor.add_constructor(
625 'tag:yaml.org,2002:python/bool',
626 Constructor.construct_yaml_bool)
627
628 Constructor.add_constructor(
629 'tag:yaml.org,2002:python/str',
630 Constructor.construct_python_str)
631
632 Constructor.add_constructor(
633 'tag:yaml.org,2002:python/unicode',
634 Constructor.construct_python_unicode)
635
636 Constructor.add_constructor(
637 'tag:yaml.org,2002:python/bytes',
638 Constructor.construct_python_bytes)
639
640 Constructor.add_constructor(
641 'tag:yaml.org,2002:python/int',
642 Constructor.construct_yaml_int)
643
644 Constructor.add_constructor(
645 'tag:yaml.org,2002:python/long',
646 Constructor.construct_python_long)
647
648 Constructor.add_constructor(
649 'tag:yaml.org,2002:python/float',
650 Constructor.construct_yaml_float)
651
652 Constructor.add_constructor(
653 'tag:yaml.org,2002:python/complex',
654 Constructor.construct_python_complex)
655
656 Constructor.add_constructor(
657 'tag:yaml.org,2002:python/list',
658 Constructor.construct_yaml_seq)
659
660 Constructor.add_constructor(
661 'tag:yaml.org,2002:python/tuple',
662 Constructor.construct_python_tuple)
663
664 Constructor.add_constructor(
665 'tag:yaml.org,2002:python/dict',
666 Constructor.construct_yaml_map)
667
668 Constructor.add_multi_constructor(
669 'tag:yaml.org,2002:python/name:',
670 Constructor.construct_python_name)
671
672 Constructor.add_multi_constructor(
673 'tag:yaml.org,2002:python/module:',
674 Constructor.construct_python_module)
675
676 Constructor.add_multi_constructor(
677 'tag:yaml.org,2002:python/object:',
678 Constructor.construct_python_object)
679
680 Constructor.add_multi_constructor(
681 'tag:yaml.org,2002:python/object/apply:',
682 Constructor.construct_python_object_apply)
683
684 Constructor.add_multi_constructor(
685 'tag:yaml.org,2002:python/object/new:',
686 Constructor.construct_python_object_new)
687