@cryptotaxi247 / netdata-1 / commits / bef998237

bundle pyyaml

paulfantom committed Jul 16, 2016 at 16:40 UTC bef9982377df93b1d5f99b02a129c5630777284c
37 files changed +11594 -4
LICENSE.md
+5 -1
@@ -142,4 +142,8 @@ connectivity is not available.
142
143 Copyright 2014, Marc 'BlackJack' Rintsch
144 [LGPL 2.1 License](https://bitbucket.org/blackjack/pysensors)
145 -
145 +
146 +- [PyYAML](https://bitbucket.org/blackjack/pysensors)
147 +
148 + Copyright 2006, Kirill Simonov
149 + [MIT License](http://pyyaml.org)
plugins.d/python.d.plugin
+9 -3
@@ -37,11 +37,10 @@ import msg
37 try:
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
40 -
40 + PY_VERSION = 3
41 # change this hack below if we want PY_VERSION to be used in modules
42 # import builtins
43 # builtins.PY_VERSION = 3
44 - PY_VERSION = 3
44 msg.info('Using python v3')
45 except (AssertionError, ImportError):
46 try:
@@ -54,8 +53,15 @@ except (AssertionError, ImportError):
53 msg.info('Using python v2')
54 except ImportError:
55 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
56 +# try:
57 +# import yaml
58 +# except ImportError:
59 +# msg.fatal('Cannot find yaml library')
60 try:
58 - import yaml
61 + if PY_VERSION == 3:
62 + import pyyaml3 as yaml
63 + else:
64 + import pyyaml2 as yaml
65 except ImportError:
66 msg.fatal('Cannot find yaml library')
67
python.d/Makefile.am
+42
@@ -36,3 +36,45 @@ dist_pythonmodules_DATA = \
36 python_modules/msg.py \
37 python_modules/lm_sensors.py \
38 $(NULL)
39 +
40 +pythonyaml2dir=$(pythonmodulesdir)/pyyaml2
41 +dist_pythonyaml2_DATA = \
42 + python_modules/pyyaml2/__init__.py \
43 + python_modules/pyyaml2/composer.py \
44 + python_modules/pyyaml2/constructor.py \
45 + python_modules/pyyaml2/cyaml.py \
46 + python_modules/pyyaml2/dumper.py \
47 + python_modules/pyyaml2/emitter.py \
48 + python_modules/pyyaml2/error.py \
49 + python_modules/pyyaml2/events.py \
50 + python_modules/pyyaml2/loader.py \
51 + python_modules/pyyaml2/nodes.py \
52 + python_modules/pyyaml2/parser.py \
53 + python_modules/pyyaml2/reader.py \
54 + python_modules/pyyaml2/representer.py \
55 + python_modules/pyyaml2/resolver.py \
56 + python_modules/pyyaml2/scanner.py \
57 + python_modules/pyyaml2/serializer.py \
58 + python_modules/pyyaml2/tokens.py \
59 + $(NULL)
60 +
61 +pythonyaml3dir=$(pythonmodulesdir)/pyyaml3
62 +dist_pythonyaml3_DATA = \
63 + python_modules/pyyaml3/__init__.py \
64 + python_modules/pyyaml3/composer.py \
65 + python_modules/pyyaml3/constructor.py \
66 + python_modules/pyyaml3/cyaml.py \
67 + python_modules/pyyaml3/dumper.py \
68 + python_modules/pyyaml3/emitter.py \
69 + python_modules/pyyaml3/error.py \
70 + python_modules/pyyaml3/events.py \
71 + python_modules/pyyaml3/loader.py \
72 + python_modules/pyyaml3/nodes.py \
73 + python_modules/pyyaml3/parser.py \
74 + python_modules/pyyaml3/reader.py \
75 + python_modules/pyyaml3/representer.py \
76 + python_modules/pyyaml3/resolver.py \
77 + python_modules/pyyaml3/scanner.py \
78 + python_modules/pyyaml3/serializer.py \
79 + python_modules/pyyaml3/tokens.py \
80 + $(NULL)
\ No newline at end of file
python.d/python_modules/pyyaml2/__init__.py new
+315
@@ -0,0 +1,315 @@
1 +
2 +from error import *
3 +
4 +from tokens import *
5 +from events import *
6 +from nodes import *
7 +
8 +from loader import *
9 +from dumper import *
10 +
11 +__version__ = '3.11'
12 +
13 +try:
14 + from cyaml import *
15 + __with_libyaml__ = True
16 +except ImportError:
17 + __with_libyaml__ = False
18 +
19 +def scan(stream, Loader=Loader):
20 + """
21 + Scan a YAML stream and produce scanning tokens.
22 + """
23 + loader = Loader(stream)
24 + try:
25 + while loader.check_token():
26 + yield loader.get_token()
27 + finally:
28 + loader.dispose()
29 +
30 +def parse(stream, Loader=Loader):
31 + """
32 + Parse a YAML stream and produce parsing events.
33 + """
34 + loader = Loader(stream)
35 + try:
36 + while loader.check_event():
37 + yield loader.get_event()
38 + finally:
39 + loader.dispose()
40 +
41 +def compose(stream, Loader=Loader):
42 + """
43 + Parse the first YAML document in a stream
44 + and produce the corresponding representation tree.
45 + """
46 + loader = Loader(stream)
47 + try:
48 + return loader.get_single_node()
49 + finally:
50 + loader.dispose()
51 +
52 +def compose_all(stream, Loader=Loader):
53 + """
54 + Parse all YAML documents in a stream
55 + and produce corresponding representation trees.
56 + """
57 + loader = Loader(stream)
58 + try:
59 + while loader.check_node():
60 + yield loader.get_node()
61 + finally:
62 + loader.dispose()
63 +
64 +def load(stream, Loader=Loader):
65 + """
66 + Parse the first YAML document in a stream
67 + and produce the corresponding Python object.
68 + """
69 + loader = Loader(stream)
70 + try:
71 + return loader.get_single_data()
72 + finally:
73 + loader.dispose()
74 +
75 +def load_all(stream, Loader=Loader):
76 + """
77 + Parse all YAML documents in a stream
78 + and produce corresponding Python objects.
79 + """
80 + loader = Loader(stream)
81 + try:
82 + while loader.check_data():
83 + yield loader.get_data()
84 + finally:
85 + loader.dispose()
86 +
87 +def safe_load(stream):
88 + """
89 + Parse the first YAML document in a stream
90 + and produce the corresponding Python object.
91 + Resolve only basic YAML tags.
92 + """
93 + return load(stream, SafeLoader)
94 +
95 +def safe_load_all(stream):
96 + """
97 + Parse all YAML documents in a stream
98 + and produce corresponding Python objects.
99 + Resolve only basic YAML tags.
100 + """
101 + return load_all(stream, SafeLoader)
102 +
103 +def emit(events, stream=None, Dumper=Dumper,
104 + canonical=None, indent=None, width=None,
105 + allow_unicode=None, line_break=None):
106 + """
107 + Emit YAML parsing events into a stream.
108 + If stream is None, return the produced string instead.
109 + """
110 + getvalue = None
111 + if stream is None:
112 + from StringIO import StringIO
113 + stream = StringIO()
114 + getvalue = stream.getvalue
115 + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
116 + allow_unicode=allow_unicode, line_break=line_break)
117 + try:
118 + for event in events:
119 + dumper.emit(event)
120 + finally:
121 + dumper.dispose()
122 + if getvalue:
123 + return getvalue()
124 +
125 +def serialize_all(nodes, stream=None, Dumper=Dumper,
126 + canonical=None, indent=None, width=None,
127 + allow_unicode=None, line_break=None,
128 + encoding='utf-8', explicit_start=None, explicit_end=None,
129 + version=None, tags=None):
130 + """
131 + Serialize a sequence of representation trees into a YAML stream.
132 + If stream is None, return the produced string instead.
133 + """
134 + getvalue = None
135 + if stream is None:
136 + if encoding is None:
137 + from StringIO import StringIO
138 + else:
139 + from cStringIO import StringIO
140 + stream = StringIO()
141 + getvalue = stream.getvalue
142 + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
143 + allow_unicode=allow_unicode, line_break=line_break,
144 + encoding=encoding, version=version, tags=tags,
145 + explicit_start=explicit_start, explicit_end=explicit_end)
146 + try:
147 + dumper.open()
148 + for node in nodes:
149 + dumper.serialize(node)
150 + dumper.close()
151 + finally:
152 + dumper.dispose()
153 + if getvalue:
154 + return getvalue()
155 +
156 +def serialize(node, stream=None, Dumper=Dumper, **kwds):
157 + """
158 + Serialize a representation tree into a YAML stream.
159 + If stream is None, return the produced string instead.
160 + """
161 + return serialize_all([node], stream, Dumper=Dumper, **kwds)
162 +
163 +def dump_all(documents, stream=None, Dumper=Dumper,
164 + default_style=None, default_flow_style=None,
165 + canonical=None, indent=None, width=None,
166 + allow_unicode=None, line_break=None,
167 + encoding='utf-8', explicit_start=None, explicit_end=None,
168 + version=None, tags=None):
169 + """
170 + Serialize a sequence of Python objects into a YAML stream.
171 + If stream is None, return the produced string instead.
172 + """
173 + getvalue = None
174 + if stream is None:
175 + if encoding is None:
176 + from StringIO import StringIO
177 + else:
178 + from cStringIO import StringIO
179 + stream = StringIO()
180 + getvalue = stream.getvalue
181 + dumper = Dumper(stream, default_style=default_style,
182 + default_flow_style=default_flow_style,
183 + canonical=canonical, indent=indent, width=width,
184 + allow_unicode=allow_unicode, line_break=line_break,
185 + encoding=encoding, version=version, tags=tags,
186 + explicit_start=explicit_start, explicit_end=explicit_end)
187 + try:
188 + dumper.open()
189 + for data in documents:
190 + dumper.represent(data)
191 + dumper.close()
192 + finally:
193 + dumper.dispose()
194 + if getvalue:
195 + return getvalue()
196 +
197 +def dump(data, stream=None, Dumper=Dumper, **kwds):
198 + """
199 + Serialize a Python object into a YAML stream.
200 + If stream is None, return the produced string instead.
201 + """
202 + return dump_all([data], stream, Dumper=Dumper, **kwds)
203 +
204 +def safe_dump_all(documents, stream=None, **kwds):
205 + """
206 + Serialize a sequence of Python objects into a YAML stream.
207 + Produce only basic YAML tags.
208 + If stream is None, return the produced string instead.
209 + """
210 + return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
211 +
212 +def safe_dump(data, stream=None, **kwds):
213 + """
214 + Serialize a Python object into a YAML stream.
215 + Produce only basic YAML tags.
216 + If stream is None, return the produced string instead.
217 + """
218 + return dump_all([data], stream, Dumper=SafeDumper, **kwds)
219 +
220 +def add_implicit_resolver(tag, regexp, first=None,
221 + Loader=Loader, Dumper=Dumper):
222 + """
223 + Add an implicit scalar detector.
224 + If an implicit scalar value matches the given regexp,
225 + the corresponding tag is assigned to the scalar.
226 + first is a sequence of possible initial characters or None.
227 + """
228 + Loader.add_implicit_resolver(tag, regexp, first)
229 + Dumper.add_implicit_resolver(tag, regexp, first)
230 +
231 +def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper):
232 + """
233 + Add a path based resolver for the given tag.
234 + A path is a list of keys that forms a path
235 + to a node in the representation tree.
236 + Keys can be string values, integers, or None.
237 + """
238 + Loader.add_path_resolver(tag, path, kind)
239 + Dumper.add_path_resolver(tag, path, kind)
240 +
241 +def add_constructor(tag, constructor, Loader=Loader):
242 + """
243 + Add a constructor for the given tag.
244 + Constructor is a function that accepts a Loader instance
245 + and a node object and produces the corresponding Python object.
246 + """
247 + Loader.add_constructor(tag, constructor)
248 +
249 +def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader):
250 + """
251 + Add a multi-constructor for the given tag prefix.
252 + Multi-constructor is called for a node if its tag starts with tag_prefix.
253 + Multi-constructor accepts a Loader instance, a tag suffix,
254 + and a node object and produces the corresponding Python object.
255 + """
256 + Loader.add_multi_constructor(tag_prefix, multi_constructor)
257 +
258 +def add_representer(data_type, representer, Dumper=Dumper):
259 + """
260 + Add a representer for the given type.
261 + Representer is a function accepting a Dumper instance
262 + and an instance of the given data type
263 + and producing the corresponding representation node.
264 + """
265 + Dumper.add_representer(data_type, representer)
266 +
267 +def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
268 + """
269 + Add a representer for the given type.
270 + Multi-representer is a function accepting a Dumper instance
271 + and an instance of the given data type or subtype
272 + and producing the corresponding representation node.
273 + """
274 + Dumper.add_multi_representer(data_type, multi_representer)
275 +
276 +class YAMLObjectMetaclass(type):
277 + """
278 + The metaclass for YAMLObject.
279 + """
280 + def __init__(cls, name, bases, kwds):
281 + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
282 + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
283 + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
284 + cls.yaml_dumper.add_representer(cls, cls.to_yaml)
285 +
286 +class YAMLObject(object):
287 + """
288 + An object that can dump itself to a YAML stream
289 + and load itself from a YAML stream.
290 + """
291 +
292 + __metaclass__ = YAMLObjectMetaclass
293 + __slots__ = () # no direct instantiation, so allow immutable subclasses
294 +
295 + yaml_loader = Loader
296 + yaml_dumper = Dumper
297 +
298 + yaml_tag = None
299 + yaml_flow_style = None
300 +
301 + def from_yaml(cls, loader, node):
302 + """
303 + Convert a representation node to a Python object.
304 + """
305 + return loader.construct_yaml_object(node, cls)
306 + from_yaml = classmethod(from_yaml)
307 +
308 + def to_yaml(cls, dumper, data):
309 + """
310 + Convert a Python object to a representation node.
311 + """
312 + return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
313 + flow_style=cls.yaml_flow_style)
314 + to_yaml = classmethod(to_yaml)
315 +
python.d/python_modules/pyyaml2/composer.py new
+139
@@ -0,0 +1,139 @@
1 +
2 +__all__ = ['Composer', 'ComposerError']
3 +
4 +from error import MarkedYAMLError
5 +from events import *
6 +from nodes import *
7 +
8 +class ComposerError(MarkedYAMLError):
9 + pass
10 +
11 +class Composer(object):
12 +
13 + def __init__(self):
14 + self.anchors = {}
15 +
16 + def check_node(self):
17 + # Drop the STREAM-START event.
18 + if self.check_event(StreamStartEvent):
19 + self.get_event()
20 +
21 + # If there are more documents available?
22 + return not self.check_event(StreamEndEvent)
23 +
24 + def get_node(self):
25 + # Get the root node of the next document.
26 + if not self.check_event(StreamEndEvent):
27 + return self.compose_document()
28 +
29 + def get_single_node(self):
30 + # Drop the STREAM-START event.
31 + self.get_event()
32 +
33 + # Compose a document if the stream is not empty.
34 + document = None
35 + if not self.check_event(StreamEndEvent):
36 + document = self.compose_document()
37 +
38 + # Ensure that the stream contains no more documents.
39 + if not self.check_event(StreamEndEvent):
40 + event = self.get_event()
41 + raise ComposerError("expected a single document in the stream",
42 + document.start_mark, "but found another document",
43 + event.start_mark)
44 +
45 + # Drop the STREAM-END event.
46 + self.get_event()
47 +
48 + return document
49 +
50 + def compose_document(self):
51 + # Drop the DOCUMENT-START event.
52 + self.get_event()
53 +
54 + # Compose the root node.
55 + node = self.compose_node(None, None)
56 +
57 + # Drop the DOCUMENT-END event.
58 + self.get_event()
59 +
60 + self.anchors = {}
61 + return node
62 +
63 + def compose_node(self, parent, index):
64 + if self.check_event(AliasEvent):
65 + event = self.get_event()
66 + anchor = event.anchor
67 + if anchor not in self.anchors:
68 + raise ComposerError(None, None, "found undefined alias %r"
69 + % anchor.encode('utf-8'), event.start_mark)
70 + return self.anchors[anchor]
71 + event = self.peek_event()
72 + anchor = event.anchor
73 + if anchor is not None:
74 + if anchor in self.anchors:
75 + raise ComposerError("found duplicate anchor %r; first occurence"
76 + % anchor.encode('utf-8'), self.anchors[anchor].start_mark,
77 + "second occurence", event.start_mark)
78 + self.descend_resolver(parent, index)
79 + if self.check_event(ScalarEvent):
80 + node = self.compose_scalar_node(anchor)
81 + elif self.check_event(SequenceStartEvent):
82 + node = self.compose_sequence_node(anchor)
83 + elif self.check_event(MappingStartEvent):
84 + node = self.compose_mapping_node(anchor)
85 + self.ascend_resolver()
86 + return node
87 +
88 + def compose_scalar_node(self, anchor):
89 + event = self.get_event()
90 + tag = event.tag
91 + if tag is None or tag == u'!':
92 + tag = self.resolve(ScalarNode, event.value, event.implicit)
93 + node = ScalarNode(tag, event.value,
94 + event.start_mark, event.end_mark, style=event.style)
95 + if anchor is not None:
96 + self.anchors[anchor] = node
97 + return node
98 +
99 + def compose_sequence_node(self, anchor):
100 + start_event = self.get_event()
101 + tag = start_event.tag
102 + if tag is None or tag == u'!':
103 + tag = self.resolve(SequenceNode, None, start_event.implicit)
104 + node = SequenceNode(tag, [],
105 + start_event.start_mark, None,
106 + flow_style=start_event.flow_style)
107 + if anchor is not None:
108 + self.anchors[anchor] = node
109 + index = 0
110 + while not self.check_event(SequenceEndEvent):
111 + node.value.append(self.compose_node(node, index))
112 + index += 1
113 + end_event = self.get_event()
114 + node.end_mark = end_event.end_mark
115 + return node
116 +
117 + def compose_mapping_node(self, anchor):
118 + start_event = self.get_event()
119 + tag = start_event.tag
120 + if tag is None or tag == u'!':
121 + tag = self.resolve(MappingNode, None, start_event.implicit)
122 + node = MappingNode(tag, [],
123 + start_event.start_mark, None,
124 + flow_style=start_event.flow_style)
125 + if anchor is not None:
126 + self.anchors[anchor] = node
127 + while not self.check_event(MappingEndEvent):
128 + #key_event = self.peek_event()
129 + item_key = self.compose_node(node, None)
130 + #if item_key in node.value:
131 + # raise ComposerError("while composing a mapping", start_event.start_mark,
132 + # "found duplicate key", key_event.start_mark)
133 + item_value = self.compose_node(node, item_key)
134 + #node.value[item_key] = item_value
135 + node.value.append((item_key, item_value))
136 + end_event = self.get_event()
137 + node.end_mark = end_event.end_mark
138 + return node
139 +
python.d/python_modules/pyyaml2/constructor.py new
+675
@@ -0,0 +1,675 @@
1 +
2 +__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
3 + 'ConstructorError']
4 +
5 +from error import *
6 +from nodes import *
7 +
8 +import datetime
9 +
10 +import binascii, re, sys, types
11 +
12 +class ConstructorError(MarkedYAMLError):
13 + pass
14 +
15 +class BaseConstructor(object):
16 +
17 + yaml_constructors = {}
18 + yaml_multi_constructors = {}
19 +
20 + def __init__(self):
21 + self.constructed_objects = {}
22 + self.recursive_objects = {}
23 + self.state_generators = []
24 + self.deep_construct = False
25 +
26 + def check_data(self):
27 + # If there are more documents available?
28 + return self.check_node()
29 +
30 + def get_data(self):
31 + # Construct and return the next document.
32 + if self.check_node():
33 + return self.construct_document(self.get_node())
34 +
35 + def get_single_data(self):
36 + # Ensure that the stream contains a single document and construct it.
37 + node = self.get_single_node()
38 + if node is not None:
39 + return self.construct_document(node)
40 + return None
41 +
42 + def construct_document(self, node):
43 + data = self.construct_object(node)
44 + while self.state_generators:
45 + state_generators = self.state_generators
46 + self.state_generators = []
47 + for generator in state_generators:
48 + for dummy in generator:
49 + pass
50 + self.constructed_objects = {}
51 + self.recursive_objects = {}
52 + self.deep_construct = False
53 + return data
54 +
55 + def construct_object(self, node, deep=False):
56 + if node in self.constructed_objects:
57 + return self.constructed_objects[node]
58 + if deep:
59 + old_deep = self.deep_construct
60 + self.deep_construct = True
61 + if node in self.recursive_objects:
62 + raise ConstructorError(None, None,
63 + "found unconstructable recursive node", node.start_mark)
64 + self.recursive_objects[node] = None
65 + constructor = None
66 + tag_suffix = None
67 + if node.tag in self.yaml_constructors:
68 + constructor = self.yaml_constructors[node.tag]
69 + else:
70 + for tag_prefix in self.yaml_multi_constructors:
71 + if node.tag.startswith(tag_prefix):
72 + tag_suffix = node.tag[len(tag_prefix):]
73 + constructor = self.yaml_multi_constructors[tag_prefix]
74 + break
75 + else:
76 + if None in self.yaml_multi_constructors:
77 + tag_suffix = node.tag
78 + constructor = self.yaml_multi_constructors[None]
79 + elif None in self.yaml_constructors:
80 + constructor = self.yaml_constructors[None]
81 + elif isinstance(node, ScalarNode):
82 + constructor = self.__class__.construct_scalar
83 + elif isinstance(node, SequenceNode):
84 + constructor = self.__class__.construct_sequence
85 + elif isinstance(node, MappingNode):
86 + constructor = self.__class__.construct_mapping
87 + if tag_suffix is None:
88 + data = constructor(self, node)
89 + else:
90 + data = constructor(self, tag_suffix, node)
91 + if isinstance(data, types.GeneratorType):
92 + generator = data
93 + data = generator.next()
94 + if self.deep_construct:
95 + for dummy in generator:
96 + pass
97 + else:
98 + self.state_generators.append(generator)
99 + self.constructed_objects[node] = data
100 + del self.recursive_objects[node]
101 + if deep:
102 + self.deep_construct = old_deep
103 + return data
104 +
105 + def construct_scalar(self, node):
106 + if not isinstance(node, ScalarNode):
107 + raise ConstructorError(None, None,
108 + "expected a scalar node, but found %s" % node.id,
109 + node.start_mark)
110 + return node.value
111 +
112 + def construct_sequence(self, node, deep=False):
113 + if not isinstance(node, SequenceNode):
114 + raise ConstructorError(None, None,
115 + "expected a sequence node, but found %s" % node.id,
116 + node.start_mark)
117 + return [self.construct_object(child, deep=deep)
118 + for child in node.value]
119 +
120 + def construct_mapping(self, node, deep=False):
121 + if not isinstance(node, MappingNode):
122 + raise ConstructorError(None, None,
123 + "expected a mapping node, but found %s" % node.id,
124 + node.start_mark)
125 + mapping = {}
126 + for key_node, value_node in node.value:
127 + key = self.construct_object(key_node, deep=deep)
128 + try:
129 + hash(key)
130 + except TypeError, exc:
131 + raise ConstructorError("while constructing a mapping", node.start_mark,
132 + "found unacceptable key (%s)" % exc, key_node.start_mark)
133 + value = self.construct_object(value_node, deep=deep)
134 + mapping[key] = value
135 + return mapping
136 +
137 + def construct_pairs(self, node, deep=False):
138 + if not isinstance(node, MappingNode):
139 + raise ConstructorError(None, None,
140 + "expected a mapping node, but found %s" % node.id,
141 + node.start_mark)
142 + pairs = []
143 + for key_node, value_node in node.value:
144 + key = self.construct_object(key_node, deep=deep)
145 + value = self.construct_object(value_node, deep=deep)
146 + pairs.append((key, value))
147 + return pairs
148 +
149 + def add_constructor(cls, tag, constructor):
150 + if not 'yaml_constructors' in cls.__dict__:
151 + cls.yaml_constructors = cls.yaml_constructors.copy()
152 + cls.yaml_constructors[tag] = constructor
153 + add_constructor = classmethod(add_constructor)
154 +
155 + def add_multi_constructor(cls, tag_prefix, multi_constructor):
156 + if not 'yaml_multi_constructors' in cls.__dict__:
157 + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
158 + cls.yaml_multi_constructors[tag_prefix] = multi_constructor
159 + add_multi_constructor = classmethod(add_multi_constructor)
160 +
161 +class SafeConstructor(BaseConstructor):
162 +
163 + def construct_scalar(self, node):
164 + if isinstance(node, MappingNode):
165 + for key_node, value_node in node.value:
166 + if key_node.tag == u'tag:yaml.org,2002:value':
167 + return self.construct_scalar(value_node)
168 + return BaseConstructor.construct_scalar(self, node)
169 +
170 + def flatten_mapping(self, node):
171 + merge = []
172 + index = 0
173 + while index < len(node.value):
174 + key_node, value_node = node.value[index]
175 + if key_node.tag == u'tag:yaml.org,2002:merge':
176 + del node.value[index]
177 + if isinstance(value_node, MappingNode):
178 + self.flatten_mapping(value_node)
179 + merge.extend(value_node.value)
180 + elif isinstance(value_node, SequenceNode):
181 + submerge = []
182 + for subnode in value_node.value:
183 + if not isinstance(subnode, MappingNode):
184 + raise ConstructorError("while constructing a mapping",
185 + node.start_mark,
186 + "expected a mapping for merging, but found %s"
187 + % subnode.id, subnode.start_mark)
188 + self.flatten_mapping(subnode)
189 + submerge.append(subnode.value)
190 + submerge.reverse()
191 + for value in submerge:
192 + merge.extend(value)
193 + else:
194 + raise ConstructorError("while constructing a mapping", node.start_mark,
195 + "expected a mapping or list of mappings for merging, but found %s"
196 + % value_node.id, value_node.start_mark)
197 + elif key_node.tag == u'tag:yaml.org,2002:value':
198 + key_node.tag = u'tag:yaml.org,2002:str'
199 + index += 1
200 + else:
201 + index += 1
202 + if merge:
203 + node.value = merge + node.value
204 +
205 + def construct_mapping(self, node, deep=False):
206 + if isinstance(node, MappingNode):
207 + self.flatten_mapping(node)
208 + return BaseConstructor.construct_mapping(self, node, deep=deep)
209 +
210 + def construct_yaml_null(self, node):
211 + self.construct_scalar(node)
212 + return None
213 +
214 + bool_values = {
215 + u'yes': True,
216 + u'no': False,
217 + u'true': True,
218 + u'false': False,
219 + u'on': True,
220 + u'off': False,
221 + }
222 +
223 + def construct_yaml_bool(self, node):
224 + value = self.construct_scalar(node)
225 + return self.bool_values[value.lower()]
226 +
227 + def construct_yaml_int(self, node):
228 + value = str(self.construct_scalar(node))
229 + value = value.replace('_', '')
230 + sign = +1
231 + if value[0] == '-':
232 + sign = -1
233 + if value[0] in '+-':
234 + value = value[1:]
235 + if value == '0':
236 + return 0
237 + elif value.startswith('0b'):
238 + return sign*int(value[2:], 2)
239 + elif value.startswith('0x'):
240 + return sign*int(value[2:], 16)
241 + elif value[0] == '0':
242 + return sign*int(value, 8)
243 + elif ':' in value:
244 + digits = [int(part) for part in value.split(':')]
245 + digits.reverse()
246 + base = 1
247 + value = 0
248 + for digit in digits:
249 + value += digit*base
250 + base *= 60
251 + return sign*value
252 + else:
253 + return sign*int(value)
254 +
255 + inf_value = 1e300
256 + while inf_value != inf_value*inf_value:
257 + inf_value *= inf_value
258 + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
259 +
260 + def construct_yaml_float(self, node):
261 + value = str(self.construct_scalar(node))
262 + value = value.replace('_', '').lower()
263 + sign = +1
264 + if value[0] == '-':
265 + sign = -1
266 + if value[0] in '+-':
267 + value = value[1:]
268 + if value == '.inf':
269 + return sign*self.inf_value
270 + elif value == '.nan':
271 + return self.nan_value
272 + elif ':' in value:
273 + digits = [float(part) for part in value.split(':')]
274 + digits.reverse()
275 + base = 1
276 + value = 0.0
277 + for digit in digits:
278 + value += digit*base
279 + base *= 60
280 + return sign*value
281 + else:
282 + return sign*float(value)
283 +
284 + def construct_yaml_binary(self, node):
285 + value = self.construct_scalar(node)
286 + try:
287 + return str(value).decode('base64')
288 + except (binascii.Error, UnicodeEncodeError), exc:
289 + raise ConstructorError(None, None,
290 + "failed to decode base64 data: %s" % exc, node.start_mark)
291 +
292 + timestamp_regexp = re.compile(
293 + ur'''^(?P<year>[0-9][0-9][0-9][0-9])
294 + -(?P<month>[0-9][0-9]?)
295 + -(?P<day>[0-9][0-9]?)
296 + (?:(?:[Tt]|[ \t]+)
297 + (?P<hour>[0-9][0-9]?)
298 + :(?P<minute>[0-9][0-9])
299 + :(?P<second>[0-9][0-9])
300 + (?:\.(?P<fraction>[0-9]*))?
301 + (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
302 + (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
303 +
304 + def construct_yaml_timestamp(self, node):
305 + value = self.construct_scalar(node)
306 + match = self.timestamp_regexp.match(node.value)
307 + values = match.groupdict()
308 + year = int(values['year'])
309 + month = int(values['month'])
310 + day = int(values['day'])
311 + if not values['hour']:
312 + return datetime.date(year, month, day)
313 + hour = int(values['hour'])
314 + minute = int(values['minute'])
315 + second = int(values['second'])
316 + fraction = 0
317 + if values['fraction']:
318 + fraction = values['fraction'][:6]
319 + while len(fraction) < 6:
320 + fraction += '0'
321 + fraction = int(fraction)
322 + delta = None
323 + if values['tz_sign']:
324 + tz_hour = int(values['tz_hour'])
325 + tz_minute = int(values['tz_minute'] or 0)
326 + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
327 + if values['tz_sign'] == '-':
328 + delta = -delta
329 + data = datetime.datetime(year, month, day, hour, minute, second, fraction)
330 + if delta:
331 + data -= delta
332 + return data
333 +
334 + def construct_yaml_omap(self, node):
335 + # Note: we do not check for duplicate keys, because it's too
336 + # CPU-expensive.
337 + omap = []
338 + yield omap
339 + if not isinstance(node, SequenceNode):
340 + raise ConstructorError("while constructing an ordered map", node.start_mark,
341 + "expected a sequence, but found %s" % node.id, node.start_mark)
342 + for subnode in node.value:
343 + if not isinstance(subnode, MappingNode):
344 + raise ConstructorError("while constructing an ordered map", node.start_mark,
345 + "expected a mapping of length 1, but found %s" % subnode.id,
346 + subnode.start_mark)
347 + if len(subnode.value) != 1:
348 + raise ConstructorError("while constructing an ordered map", node.start_mark,
349 + "expected a single mapping item, but found %d items" % len(subnode.value),
350 + subnode.start_mark)
351 + key_node, value_node = subnode.value[0]
352 + key = self.construct_object(key_node)
353 + value = self.construct_object(value_node)
354 + omap.append((key, value))
355 +
356 + def construct_yaml_pairs(self, node):
357 + # Note: the same code as `construct_yaml_omap`.
358 + pairs = []
359 + yield pairs
360 + if not isinstance(node, SequenceNode):
361 + raise ConstructorError("while constructing pairs", node.start_mark,
362 + "expected a sequence, but found %s" % node.id, node.start_mark)
363 + for subnode in node.value:
364 + if not isinstance(subnode, MappingNode):
365 + raise ConstructorError("while constructing pairs", node.start_mark,
366 + "expected a mapping of length 1, but found %s" % subnode.id,
367 + subnode.start_mark)
368 + if len(subnode.value) != 1:
369 + raise ConstructorError("while constructing pairs", node.start_mark,
370 + "expected a single mapping item, but found %d items" % len(subnode.value),
371 + subnode.start_mark)
372 + key_node, value_node = subnode.value[0]
373 + key = self.construct_object(key_node)
374 + value = self.construct_object(value_node)
375 + pairs.append((key, value))
376 +
377 + def construct_yaml_set(self, node):
378 + data = set()
379 + yield data
380 + value = self.construct_mapping(node)
381 + data.update(value)
382 +
383 + def construct_yaml_str(self, node):
384 + value = self.construct_scalar(node)
385 + try:
386 + return value.encode('ascii')
387 + except UnicodeEncodeError:
388 + return value
389 +
390 + def construct_yaml_seq(self, node):
391 + data = []
392 + yield data
393 + data.extend(self.construct_sequence(node))
394 +
395 + def construct_yaml_map(self, node):
396 + data = {}
397 + yield data
398 + value = self.construct_mapping(node)
399 + data.update(value)
400 +
401 + def construct_yaml_object(self, node, cls):
402 + data = cls.__new__(cls)
403 + yield data
404 + if hasattr(data, '__setstate__'):
405 + state = self.construct_mapping(node, deep=True)
406 + data.__setstate__(state)
407 + else:
408 + state = self.construct_mapping(node)
409 + data.__dict__.update(state)
410 +
411 + def construct_undefined(self, node):
412 + raise ConstructorError(None, None,
413 + "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'),
414 + node.start_mark)
415 +
416 +SafeConstructor.add_constructor(
417 + u'tag:yaml.org,2002:null',
418 + SafeConstructor.construct_yaml_null)
419 +
420 +SafeConstructor.add_constructor(
421 + u'tag:yaml.org,2002:bool',
422 + SafeConstructor.construct_yaml_bool)
423 +
424 +SafeConstructor.add_constructor(
425 + u'tag:yaml.org,2002:int',
426 + SafeConstructor.construct_yaml_int)
427 +
428 +SafeConstructor.add_constructor(
429 + u'tag:yaml.org,2002:float',
430 + SafeConstructor.construct_yaml_float)
431 +
432 +SafeConstructor.add_constructor(
433 + u'tag:yaml.org,2002:binary',
434 + SafeConstructor.construct_yaml_binary)
435 +
436 +SafeConstructor.add_constructor(
437 + u'tag:yaml.org,2002:timestamp',
438 + SafeConstructor.construct_yaml_timestamp)
439 +
440 +SafeConstructor.add_constructor(
441 + u'tag:yaml.org,2002:omap',
442 + SafeConstructor.construct_yaml_omap)
443 +
444 +SafeConstructor.add_constructor(
445 + u'tag:yaml.org,2002:pairs',
446 + SafeConstructor.construct_yaml_pairs)
447 +
448 +SafeConstructor.add_constructor(
449 + u'tag:yaml.org,2002:set',
450 + SafeConstructor.construct_yaml_set)
451 +
452 +SafeConstructor.add_constructor(
453 + u'tag:yaml.org,2002:str',
454 + SafeConstructor.construct_yaml_str)
455 +
456 +SafeConstructor.add_constructor(
457 + u'tag:yaml.org,2002:seq',
458 + SafeConstructor.construct_yaml_seq)
459 +
460 +SafeConstructor.add_constructor(
461 + u'tag:yaml.org,2002:map',
462 + SafeConstructor.construct_yaml_map)
463 +
464 +SafeConstructor.add_constructor(None,
465 + SafeConstructor.construct_undefined)
466 +
467 +class Constructor(SafeConstructor):
468 +
469 + def construct_python_str(self, node):
470 + return self.construct_scalar(node).encode('utf-8')
471 +
472 + def construct_python_unicode(self, node):
473 + return self.construct_scalar(node)
474 +
475 + def construct_python_long(self, node):
476 + return long(self.construct_yaml_int(node))
477 +
478 + def construct_python_complex(self, node):
479 + return complex(self.construct_scalar(node))
480 +
481 + def construct_python_tuple(self, node):
482 + return tuple(self.construct_sequence(node))
483 +
484 + def find_python_module(self, name, mark):
485 + if not name:
486 + raise ConstructorError("while constructing a Python module", mark,
487 + "expected non-empty name appended to the tag", mark)
488 + try:
489 + __import__(name)
490 + except ImportError, exc:
491 + raise ConstructorError("while constructing a Python module", mark,
492 + "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark)
493 + return sys.modules[name]
494 +
495 + def find_python_name(self, name, mark):
496 + if not name:
497 + raise ConstructorError("while constructing a Python object", mark,
498 + "expected non-empty name appended to the tag", mark)
499 + if u'.' in name:
500 + module_name, object_name = name.rsplit('.', 1)
501 + else:
502 + module_name = '__builtin__'
503 + object_name = name
504 + try:
505 + __import__(module_name)
506 + except ImportError, exc:
507 + raise ConstructorError("while constructing a Python object", mark,
508 + "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark)
509 + module = sys.modules[module_name]
510 + if not hasattr(module, object_name):
511 + raise ConstructorError("while constructing a Python object", mark,
512 + "cannot find %r in the module %r" % (object_name.encode('utf-8'),
513 + module.__name__), mark)
514 + return getattr(module, object_name)
515 +
516 + def construct_python_name(self, suffix, node):
517 + value = self.construct_scalar(node)
518 + if value:
519 + raise ConstructorError("while constructing a Python name", node.start_mark,
520 + "expected the empty value, but found %r" % value.encode('utf-8'),
521 + node.start_mark)
522 + return self.find_python_name(suffix, node.start_mark)
523 +
524 + def construct_python_module(self, suffix, node):
525 + value = self.construct_scalar(node)
526 + if value:
527 + raise ConstructorError("while constructing a Python module", node.start_mark,
528 + "expected the empty value, but found %r" % value.encode('utf-8'),
529 + node.start_mark)
530 + return self.find_python_module(suffix, node.start_mark)
531 +
532 + class classobj: pass
533 +
534 + def make_python_instance(self, suffix, node,
535 + args=None, kwds=None, newobj=False):
536 + if not args:
537 + args = []
538 + if not kwds:
539 + kwds = {}
540 + cls = self.find_python_name(suffix, node.start_mark)
541 + if newobj and isinstance(cls, type(self.classobj)) \
542 + and not args and not kwds:
543 + instance = self.classobj()
544 + instance.__class__ = cls
545 + return instance
546 + elif newobj and isinstance(cls, type):
547 + return cls.__new__(cls, *args, **kwds)
548 + else:
549 + return cls(*args, **kwds)
550 +
551 + def set_python_instance_state(self, instance, state):
552 + if hasattr(instance, '__setstate__'):
553 + instance.__setstate__(state)
554 + else:
555 + slotstate = {}
556 + if isinstance(state, tuple) and len(state) == 2:
557 + state, slotstate = state
558 + if hasattr(instance, '__dict__'):
559 + instance.__dict__.update(state)
560 + elif state:
561 + slotstate.update(state)
562 + for key, value in slotstate.items():
563 + setattr(object, key, value)
564 +
565 + def construct_python_object(self, suffix, node):
566 + # Format:
567 + # !!python/object:module.name { ... state ... }
568 + instance = self.make_python_instance(suffix, node, newobj=True)
569 + yield instance
570 + deep = hasattr(instance, '__setstate__')
571 + state = self.construct_mapping(node, deep=deep)
572 + self.set_python_instance_state(instance, state)
573 +
574 + def construct_python_object_apply(self, suffix, node, newobj=False):
575 + # Format:
576 + # !!python/object/apply # (or !!python/object/new)
577 + # args: [ ... arguments ... ]
578 + # kwds: { ... keywords ... }
579 + # state: ... state ...
580 + # listitems: [ ... listitems ... ]
581 + # dictitems: { ... dictitems ... }
582 + # or short format:
583 + # !!python/object/apply [ ... arguments ... ]
584 + # The difference between !!python/object/apply and !!python/object/new
585 + # is how an object is created, check make_python_instance for details.
586 + if isinstance(node, SequenceNode):
587 + args = self.construct_sequence(node, deep=True)
588 + kwds = {}
589 + state = {}
590 + listitems = []
591 + dictitems = {}
592 + else:
593 + value = self.construct_mapping(node, deep=True)
594 + args = value.get('args', [])
595 + kwds = value.get('kwds', {})
596 + state = value.get('state', {})
597 + listitems = value.get('listitems', [])
598 + dictitems = value.get('dictitems', {})
599 + instance = self.make_python_instance(suffix, node, args, kwds, newobj)
600 + if state:
601 + self.set_python_instance_state(instance, state)
602 + if listitems:
603 + instance.extend(listitems)
604 + if dictitems:
605 + for key in dictitems:
606 + instance[key] = dictitems[key]
607 + return instance
608 +
609 + def construct_python_object_new(self, suffix, node):
610 + return self.construct_python_object_apply(suffix, node, newobj=True)
611 +
612 +Constructor.add_constructor(
613 + u'tag:yaml.org,2002:python/none',
614 + Constructor.construct_yaml_null)
615 +
616 +Constructor.add_constructor(
617 + u'tag:yaml.org,2002:python/bool',
618 + Constructor.construct_yaml_bool)
619 +
620 +Constructor.add_constructor(
621 + u'tag:yaml.org,2002:python/str',
622 + Constructor.construct_python_str)
623 +
624 +Constructor.add_constructor(
625 + u'tag:yaml.org,2002:python/unicode',
626 + Constructor.construct_python_unicode)
627 +
628 +Constructor.add_constructor(
629 + u'tag:yaml.org,2002:python/int',
630 + Constructor.construct_yaml_int)
631 +
632 +Constructor.add_constructor(
633 + u'tag:yaml.org,2002:python/long',
634 + Constructor.construct_python_long)
635 +
636 +Constructor.add_constructor(
637 + u'tag:yaml.org,2002:python/float',
638 + Constructor.construct_yaml_float)
639 +
640 +Constructor.add_constructor(
641 + u'tag:yaml.org,2002:python/complex',
642 + Constructor.construct_python_complex)
643 +
644 +Constructor.add_constructor(
645 + u'tag:yaml.org,2002:python/list',
646 + Constructor.construct_yaml_seq)
647 +
648 +Constructor.add_constructor(
649 + u'tag:yaml.org,2002:python/tuple',
650 + Constructor.construct_python_tuple)
651 +
652 +Constructor.add_constructor(
653 + u'tag:yaml.org,2002:python/dict',
654 + Constructor.construct_yaml_map)
655 +
656 +Constructor.add_multi_constructor(
657 + u'tag:yaml.org,2002:python/name:',
658 + Constructor.construct_python_name)
659 +
660 +Constructor.add_multi_constructor(
661 + u'tag:yaml.org,2002:python/module:',
662 + Constructor.construct_python_module)
663 +
664 +Constructor.add_multi_constructor(
665 + u'tag:yaml.org,2002:python/object:',
666 + Constructor.construct_python_object)
667 +
668 +Constructor.add_multi_constructor(
669 + u'tag:yaml.org,2002:python/object/apply:',
670 + Constructor.construct_python_object_apply)
671 +
672 +Constructor.add_multi_constructor(
673 + u'tag:yaml.org,2002:python/object/new:',
674 + Constructor.construct_python_object_new)
675 +
python.d/python_modules/pyyaml2/cyaml.py new
+85
@@ -0,0 +1,85 @@
1 +
2 +__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
3 + 'CBaseDumper', 'CSafeDumper', 'CDumper']
4 +
5 +from _yaml import CParser, CEmitter
6 +
7 +from constructor import *
8 +
9 +from serializer import *
10 +from representer import *
11 +
12 +from resolver import *
13 +
14 +class CBaseLoader(CParser, BaseConstructor, BaseResolver):
15 +
16 + def __init__(self, stream):
17 + CParser.__init__(self, stream)
18 + BaseConstructor.__init__(self)
19 + BaseResolver.__init__(self)
20 +
21 +class CSafeLoader(CParser, SafeConstructor, Resolver):
22 +
23 + def __init__(self, stream):
24 + CParser.__init__(self, stream)
25 + SafeConstructor.__init__(self)
26 + Resolver.__init__(self)
27 +
28 +class CLoader(CParser, Constructor, Resolver):
29 +
30 + def __init__(self, stream):
31 + CParser.__init__(self, stream)
32 + Constructor.__init__(self)
33 + Resolver.__init__(self)
34 +
35 +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
36 +
37 + def __init__(self, stream,
38 + default_style=None, default_flow_style=None,
39 + canonical=None, indent=None, width=None,
40 + allow_unicode=None, line_break=None,
41 + encoding=None, explicit_start=None, explicit_end=None,
42 + version=None, tags=None):
43 + CEmitter.__init__(self, stream, canonical=canonical,
44 + indent=indent, width=width, encoding=encoding,
45 + allow_unicode=allow_unicode, line_break=line_break,
46 + explicit_start=explicit_start, explicit_end=explicit_end,
47 + version=version, tags=tags)
48 + Representer.__init__(self, default_style=default_style,
49 + default_flow_style=default_flow_style)
50 + Resolver.__init__(self)
51 +
52 +class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
53 +
54 + def __init__(self, stream,
55 + default_style=None, default_flow_style=None,
56 + canonical=None, indent=None, width=None,
57 + allow_unicode=None, line_break=None,
58 + encoding=None, explicit_start=None, explicit_end=None,
59 + version=None, tags=None):
60 + CEmitter.__init__(self, stream, canonical=canonical,
61 + indent=indent, width=width, encoding=encoding,
62 + allow_unicode=allow_unicode, line_break=line_break,
63 + explicit_start=explicit_start, explicit_end=explicit_end,
64 + version=version, tags=tags)
65 + SafeRepresenter.__init__(self, default_style=default_style,
66 + default_flow_style=default_flow_style)
67 + Resolver.__init__(self)
68 +
69 +class CDumper(CEmitter, Serializer, Representer, Resolver):
70 +
71 + def __init__(self, stream,
72 + default_style=None, default_flow_style=None,
73 + canonical=None, indent=None, width=None,
74 + allow_unicode=None, line_break=None,
75 + encoding=None, explicit_start=None, explicit_end=None,
76 + version=None, tags=None):
77 + CEmitter.__init__(self, stream, canonical=canonical,
78 + indent=indent, width=width, encoding=encoding,
79 + allow_unicode=allow_unicode, line_break=line_break,
80 + explicit_start=explicit_start, explicit_end=explicit_end,
81 + version=version, tags=tags)
82 + Representer.__init__(self, default_style=default_style,
83 + default_flow_style=default_flow_style)
84 + Resolver.__init__(self)
85 +
python.d/python_modules/pyyaml2/dumper.py new
+62
@@ -0,0 +1,62 @@
1 +
2 +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper']
3 +
4 +from emitter import *
5 +from serializer import *
6 +from representer import *
7 +from resolver import *
8 +
9 +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
10 +
11 + def __init__(self, stream,
12 + default_style=None, default_flow_style=None,
13 + canonical=None, indent=None, width=None,
14 + allow_unicode=None, line_break=None,
15 + encoding=None, explicit_start=None, explicit_end=None,
16 + version=None, tags=None):
17 + Emitter.__init__(self, stream, canonical=canonical,
18 + indent=indent, width=width,
19 + allow_unicode=allow_unicode, line_break=line_break)
20 + Serializer.__init__(self, encoding=encoding,
21 + explicit_start=explicit_start, explicit_end=explicit_end,
22 + version=version, tags=tags)
23 + Representer.__init__(self, default_style=default_style,
24 + default_flow_style=default_flow_style)
25 + Resolver.__init__(self)
26 +
27 +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
28 +
29 + def __init__(self, stream,
30 + default_style=None, default_flow_style=None,
31 + canonical=None, indent=None, width=None,
32 + allow_unicode=None, line_break=None,
33 + encoding=None, explicit_start=None, explicit_end=None,
34 + version=None, tags=None):
35 + Emitter.__init__(self, stream, canonical=canonical,
36 + indent=indent, width=width,
37 + allow_unicode=allow_unicode, line_break=line_break)
38 + Serializer.__init__(self, encoding=encoding,
39 + explicit_start=explicit_start, explicit_end=explicit_end,
40 + version=version, tags=tags)
41 + SafeRepresenter.__init__(self, default_style=default_style,
42 + default_flow_style=default_flow_style)
43 + Resolver.__init__(self)
44 +
45 +class Dumper(Emitter, Serializer, Representer, Resolver):
46 +
47 + def __init__(self, stream,
48 + default_style=None, default_flow_style=None,
49 + canonical=None, indent=None, width=None,
50 + allow_unicode=None, line_break=None,
51 + encoding=None, explicit_start=None, explicit_end=None,
52 + version=None, tags=None):
53 + Emitter.__init__(self, stream, canonical=canonical,
54 + indent=indent, width=width,
55 + allow_unicode=allow_unicode, line_break=line_break)
56 + Serializer.__init__(self, encoding=encoding,
57 + explicit_start=explicit_start, explicit_end=explicit_end,
58 + version=version, tags=tags)
59 + Representer.__init__(self, default_style=default_style,
60 + default_flow_style=default_flow_style)
61 + Resolver.__init__(self)
62 +
python.d/python_modules/pyyaml2/emitter.py new
+1140
@@ -0,0 +1,1140 @@
1 +
2 +# Emitter expects events obeying the following grammar:
3 +# stream ::= STREAM-START document* STREAM-END
4 +# document ::= DOCUMENT-START node DOCUMENT-END
5 +# node ::= SCALAR | sequence | mapping
6 +# sequence ::= SEQUENCE-START node* SEQUENCE-END
7 +# mapping ::= MAPPING-START (node node)* MAPPING-END
8 +
9 +__all__ = ['Emitter', 'EmitterError']
10 +
11 +from error import YAMLError
12 +from events import *
13 +
14 +class EmitterError(YAMLError):
15 + pass
16 +
17 +class ScalarAnalysis(object):
18 + def __init__(self, scalar, empty, multiline,
19 + allow_flow_plain, allow_block_plain,
20 + allow_single_quoted, allow_double_quoted,
21 + allow_block):
22 + self.scalar = scalar
23 + self.empty = empty
24 + self.multiline = multiline
25 + self.allow_flow_plain = allow_flow_plain
26 + self.allow_block_plain = allow_block_plain
27 + self.allow_single_quoted = allow_single_quoted
28 + self.allow_double_quoted = allow_double_quoted
29 + self.allow_block = allow_block
30 +
31 +class Emitter(object):
32 +
33 + DEFAULT_TAG_PREFIXES = {
34 + u'!' : u'!',
35 + u'tag:yaml.org,2002:' : u'!!',
36 + }
37 +
38 + def __init__(self, stream, canonical=None, indent=None, width=None,
39 + allow_unicode=None, line_break=None):
40 +
41 + # The stream should have the methods `write` and possibly `flush`.
42 + self.stream = stream
43 +
44 + # Encoding can be overriden by STREAM-START.
45 + self.encoding = None
46 +
47 + # Emitter is a state machine with a stack of states to handle nested
48 + # structures.
49 + self.states = []
50 + self.state = self.expect_stream_start
51 +
52 + # Current event and the event queue.
53 + self.events = []
54 + self.event = None
55 +
56 + # The current indentation level and the stack of previous indents.
57 + self.indents = []
58 + self.indent = None
59 +
60 + # Flow level.
61 + self.flow_level = 0
62 +
63 + # Contexts.
64 + self.root_context = False
65 + self.sequence_context = False
66 + self.mapping_context = False
67 + self.simple_key_context = False
68 +
69 + # Characteristics of the last emitted character:
70 + # - current position.
71 + # - is it a whitespace?
72 + # - is it an indention character
73 + # (indentation space, '-', '?', or ':')?
74 + self.line = 0
75 + self.column = 0
76 + self.whitespace = True
77 + self.indention = True
78 +
79 + # Whether the document requires an explicit document indicator
80 + self.open_ended = False
81 +
82 + # Formatting details.
83 + self.canonical = canonical
84 + self.allow_unicode = allow_unicode
85 + self.best_indent = 2
86 + if indent and 1 < indent < 10:
87 + self.best_indent = indent
88 + self.best_width = 80
89 + if width and width > self.best_indent*2:
90 + self.best_width = width
91 + self.best_line_break = u'\n'
92 + if line_break in [u'\r', u'\n', u'\r\n']:
93 + self.best_line_break = line_break
94 +
95 + # Tag prefixes.
96 + self.tag_prefixes = None
97 +
98 + # Prepared anchor and tag.
99 + self.prepared_anchor = None
100 + self.prepared_tag = None
101 +
102 + # Scalar analysis and style.
103 + self.analysis = None
104 + self.style = None
105 +
106 + def dispose(self):
107 + # Reset the state attributes (to clear self-references)
108 + self.states = []
109 + self.state = None
110 +
111 + def emit(self, event):
112 + self.events.append(event)
113 + while not self.need_more_events():
114 + self.event = self.events.pop(0)
115 + self.state()
116 + self.event = None
117 +
118 + # In some cases, we wait for a few next events before emitting.
119 +
120 + def need_more_events(self):
121 + if not self.events:
122 + return True
123 + event = self.events[0]
124 + if isinstance(event, DocumentStartEvent):
125 + return self.need_events(1)
126 + elif isinstance(event, SequenceStartEvent):
127 + return self.need_events(2)
128 + elif isinstance(event, MappingStartEvent):
129 + return self.need_events(3)
130 + else:
131 + return False
132 +
133 + def need_events(self, count):
134 + level = 0
135 + for event in self.events[1:]:
136 + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)):
137 + level += 1
138 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)):
139 + level -= 1
140 + elif isinstance(event, StreamEndEvent):
141 + level = -1
142 + if level < 0:
143 + return False
144 + return (len(self.events) < count+1)
145 +
146 + def increase_indent(self, flow=False, indentless=False):
147 + self.indents.append(self.indent)
148 + if self.indent is None:
149 + if flow:
150 + self.indent = self.best_indent
151 + else:
152 + self.indent = 0
153 + elif not indentless:
154 + self.indent += self.best_indent
155 +
156 + # States.
157 +
158 + # Stream handlers.
159 +
160 + def expect_stream_start(self):
161 + if isinstance(self.event, StreamStartEvent):
162 + if self.event.encoding and not getattr(self.stream, 'encoding', None):
163 + self.encoding = self.event.encoding
164 + self.write_stream_start()
165 + self.state = self.expect_first_document_start
166 + else:
167 + raise EmitterError("expected StreamStartEvent, but got %s"
168 + % self.event)
169 +
170 + def expect_nothing(self):
171 + raise EmitterError("expected nothing, but got %s" % self.event)
172 +
173 + # Document handlers.
174 +
175 + def expect_first_document_start(self):
176 + return self.expect_document_start(first=True)
177 +
178 + def expect_document_start(self, first=False):
179 + if isinstance(self.event, DocumentStartEvent):
180 + if (self.event.version or self.event.tags) and self.open_ended:
181 + self.write_indicator(u'...', True)
182 + self.write_indent()
183 + if self.event.version:
184 + version_text = self.prepare_version(self.event.version)
185 + self.write_version_directive(version_text)
186 + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
187 + if self.event.tags:
188 + handles = self.event.tags.keys()
189 + handles.sort()
190 + for handle in handles:
191 + prefix = self.event.tags[handle]
192 + self.tag_prefixes[prefix] = handle
193 + handle_text = self.prepare_tag_handle(handle)
194 + prefix_text = self.prepare_tag_prefix(prefix)
195 + self.write_tag_directive(handle_text, prefix_text)
196 + implicit = (first and not self.event.explicit and not self.canonical
197 + and not self.event.version and not self.event.tags
198 + and not self.check_empty_document())
199 + if not implicit:
200 + self.write_indent()
201 + self.write_indicator(u'---', True)
202 + if self.canonical:
203 + self.write_indent()
204 + self.state = self.expect_document_root
205 + elif isinstance(self.event, StreamEndEvent):
206 + if self.open_ended:
207 + self.write_indicator(u'...', True)
208 + self.write_indent()
209 + self.write_stream_end()
210 + self.state = self.expect_nothing
211 + else:
212 + raise EmitterError("expected DocumentStartEvent, but got %s"
213 + % self.event)
214 +
215 + def expect_document_end(self):
216 + if isinstance(self.event, DocumentEndEvent):
217 + self.write_indent()
218 + if self.event.explicit:
219 + self.write_indicator(u'...', True)
220 + self.write_indent()
221 + self.flush_stream()
222 + self.state = self.expect_document_start
223 + else:
224 + raise EmitterError("expected DocumentEndEvent, but got %s"
225 + % self.event)
226 +
227 + def expect_document_root(self):
228 + self.states.append(self.expect_document_end)
229 + self.expect_node(root=True)
230 +
231 + # Node handlers.
232 +
233 + def expect_node(self, root=False, sequence=False, mapping=False,
234 + simple_key=False):
235 + self.root_context = root
236 + self.sequence_context = sequence
237 + self.mapping_context = mapping
238 + self.simple_key_context = simple_key
239 + if isinstance(self.event, AliasEvent):
240 + self.expect_alias()
241 + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)):
242 + self.process_anchor(u'&')
243 + self.process_tag()
244 + if isinstance(self.event, ScalarEvent):
245 + self.expect_scalar()
246 + elif isinstance(self.event, SequenceStartEvent):
247 + if self.flow_level or self.canonical or self.event.flow_style \
248 + or self.check_empty_sequence():
249 + self.expect_flow_sequence()
250 + else:
251 + self.expect_block_sequence()
252 + elif isinstance(self.event, MappingStartEvent):
253 + if self.flow_level or self.canonical or self.event.flow_style \
254 + or self.check_empty_mapping():
255 + self.expect_flow_mapping()
256 + else:
257 + self.expect_block_mapping()
258 + else:
259 + raise EmitterError("expected NodeEvent, but got %s" % self.event)
260 +
261 + def expect_alias(self):
262 + if self.event.anchor is None:
263 + raise EmitterError("anchor is not specified for alias")
264 + self.process_anchor(u'*')
265 + self.state = self.states.pop()
266 +
267 + def expect_scalar(self):
268 + self.increase_indent(flow=True)
269 + self.process_scalar()
270 + self.indent = self.indents.pop()
271 + self.state = self.states.pop()
272 +
273 + # Flow sequence handlers.
274 +
275 + def expect_flow_sequence(self):
276 + self.write_indicator(u'[', True, whitespace=True)
277 + self.flow_level += 1
278 + self.increase_indent(flow=True)
279 + self.state = self.expect_first_flow_sequence_item
280 +
281 + def expect_first_flow_sequence_item(self):
282 + if isinstance(self.event, SequenceEndEvent):
283 + self.indent = self.indents.pop()
284 + self.flow_level -= 1
285 + self.write_indicator(u']', False)
286 + self.state = self.states.pop()
287 + else:
288 + if self.canonical or self.column > self.best_width:
289 + self.write_indent()
290 + self.states.append(self.expect_flow_sequence_item)
291 + self.expect_node(sequence=True)
292 +
293 + def expect_flow_sequence_item(self):
294 + if isinstance(self.event, SequenceEndEvent):
295 + self.indent = self.indents.pop()
296 + self.flow_level -= 1
297 + if self.canonical:
298 + self.write_indicator(u',', False)
299 + self.write_indent()
300 + self.write_indicator(u']', False)
301 + self.state = self.states.pop()
302 + else:
303 + self.write_indicator(u',', False)
304 + if self.canonical or self.column > self.best_width:
305 + self.write_indent()
306 + self.states.append(self.expect_flow_sequence_item)
307 + self.expect_node(sequence=True)
308 +
309 + # Flow mapping handlers.
310 +
311 + def expect_flow_mapping(self):
312 + self.write_indicator(u'{', True, whitespace=True)
313 + self.flow_level += 1
314 + self.increase_indent(flow=True)
315 + self.state = self.expect_first_flow_mapping_key
316 +
317 + def expect_first_flow_mapping_key(self):
318 + if isinstance(self.event, MappingEndEvent):
319 + self.indent = self.indents.pop()
320 + self.flow_level -= 1
321 + self.write_indicator(u'}', False)
322 + self.state = self.states.pop()
323 + else:
324 + if self.canonical or self.column > self.best_width:
325 + self.write_indent()
326 + if not self.canonical and self.check_simple_key():
327 + self.states.append(self.expect_flow_mapping_simple_value)
328 + self.expect_node(mapping=True, simple_key=True)
329 + else:
330 + self.write_indicator(u'?', True)
331 + self.states.append(self.expect_flow_mapping_value)
332 + self.expect_node(mapping=True)
333 +
334 + def expect_flow_mapping_key(self):
335 + if isinstance(self.event, MappingEndEvent):
336 + self.indent = self.indents.pop()
337 + self.flow_level -= 1
338 + if self.canonical:
339 + self.write_indicator(u',', False)
340 + self.write_indent()
341 + self.write_indicator(u'}', False)
342 + self.state = self.states.pop()
343 + else:
344 + self.write_indicator(u',', False)
345 + if self.canonical or self.column > self.best_width:
346 + self.write_indent()
347 + if not self.canonical and self.check_simple_key():
348 + self.states.append(self.expect_flow_mapping_simple_value)
349 + self.expect_node(mapping=True, simple_key=True)
350 + else:
351 + self.write_indicator(u'?', True)
352 + self.states.append(self.expect_flow_mapping_value)
353 + self.expect_node(mapping=True)
354 +
355 + def expect_flow_mapping_simple_value(self):
356 + self.write_indicator(u':', False)
357 + self.states.append(self.expect_flow_mapping_key)
358 + self.expect_node(mapping=True)
359 +
360 + def expect_flow_mapping_value(self):
361 + if self.canonical or self.column > self.best_width:
362 + self.write_indent()
363 + self.write_indicator(u':', True)
364 + self.states.append(self.expect_flow_mapping_key)
365 + self.expect_node(mapping=True)
366 +
367 + # Block sequence handlers.
368 +
369 + def expect_block_sequence(self):
370 + indentless = (self.mapping_context and not self.indention)
371 + self.increase_indent(flow=False, indentless=indentless)
372 + self.state = self.expect_first_block_sequence_item
373 +
374 + def expect_first_block_sequence_item(self):
375 + return self.expect_block_sequence_item(first=True)
376 +
377 + def expect_block_sequence_item(self, first=False):
378 + if not first and isinstance(self.event, SequenceEndEvent):
379 + self.indent = self.indents.pop()
380 + self.state = self.states.pop()
381 + else:
382 + self.write_indent()
383 + self.write_indicator(u'-', True, indention=True)
384 + self.states.append(self.expect_block_sequence_item)
385 + self.expect_node(sequence=True)
386 +
387 + # Block mapping handlers.
388 +
389 + def expect_block_mapping(self):
390 + self.increase_indent(flow=False)
391 + self.state = self.expect_first_block_mapping_key
392 +
393 + def expect_first_block_mapping_key(self):
394 + return self.expect_block_mapping_key(first=True)
395 +
396 + def expect_block_mapping_key(self, first=False):
397 + if not first and isinstance(self.event, MappingEndEvent):
398 + self.indent = self.indents.pop()
399 + self.state = self.states.pop()
400 + else:
401 + self.write_indent()
402 + if self.check_simple_key():
403 + self.states.append(self.expect_block_mapping_simple_value)
404 + self.expect_node(mapping=True, simple_key=True)
405 + else:
406 + self.write_indicator(u'?', True, indention=True)
407 + self.states.append(self.expect_block_mapping_value)
408 + self.expect_node(mapping=True)
409 +
410 + def expect_block_mapping_simple_value(self):
411 + self.write_indicator(u':', False)
412 + self.states.append(self.expect_block_mapping_key)
413 + self.expect_node(mapping=True)
414 +
415 + def expect_block_mapping_value(self):
416 + self.write_indent()
417 + self.write_indicator(u':', True, indention=True)
418 + self.states.append(self.expect_block_mapping_key)
419 + self.expect_node(mapping=True)
420 +
421 + # Checkers.
422 +
423 + def check_empty_sequence(self):
424 + return (isinstance(self.event, SequenceStartEvent) and self.events
425 + and isinstance(self.events[0], SequenceEndEvent))
426 +
427 + def check_empty_mapping(self):
428 + return (isinstance(self.event, MappingStartEvent) and self.events
429 + and isinstance(self.events[0], MappingEndEvent))
430 +
431 + def check_empty_document(self):
432 + if not isinstance(self.event, DocumentStartEvent) or not self.events:
433 + return False
434 + event = self.events[0]
435 + return (isinstance(event, ScalarEvent) and event.anchor is None
436 + and event.tag is None and event.implicit and event.value == u'')
437 +
438 + def check_simple_key(self):
439 + length = 0
440 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
441 + if self.prepared_anchor is None:
442 + self.prepared_anchor = self.prepare_anchor(self.event.anchor)
443 + length += len(self.prepared_anchor)
444 + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \
445 + and self.event.tag is not None:
446 + if self.prepared_tag is None:
447 + self.prepared_tag = self.prepare_tag(self.event.tag)
448 + length += len(self.prepared_tag)
449 + if isinstance(self.event, ScalarEvent):
450 + if self.analysis is None:
451 + self.analysis = self.analyze_scalar(self.event.value)
452 + length += len(self.analysis.scalar)
453 + return (length < 128 and (isinstance(self.event, AliasEvent)
454 + or (isinstance(self.event, ScalarEvent)
455 + and not self.analysis.empty and not self.analysis.multiline)
456 + or self.check_empty_sequence() or self.check_empty_mapping()))
457 +
458 + # Anchor, Tag, and Scalar processors.
459 +
460 + def process_anchor(self, indicator):
461 + if self.event.anchor is None:
462 + self.prepared_anchor = None
463 + return
464 + if self.prepared_anchor is None:
465 + self.prepared_anchor = self.prepare_anchor(self.event.anchor)
466 + if self.prepared_anchor:
467 + self.write_indicator(indicator+self.prepared_anchor, True)
468 + self.prepared_anchor = None
469 +
470 + def process_tag(self):
471 + tag = self.event.tag
472 + if isinstance(self.event, ScalarEvent):
473 + if self.style is None:
474 + self.style = self.choose_scalar_style()
475 + if ((not self.canonical or tag is None) and
476 + ((self.style == '' and self.event.implicit[0])
477 + or (self.style != '' and self.event.implicit[1]))):
478 + self.prepared_tag = None
479 + return
480 + if self.event.implicit[0] and tag is None:
481 + tag = u'!'
482 + self.prepared_tag = None
483 + else:
484 + if (not self.canonical or tag is None) and self.event.implicit:
485 + self.prepared_tag = None
486 + return
487 + if tag is None:
488 + raise EmitterError("tag is not specified")
489 + if self.prepared_tag is None:
490 + self.prepared_tag = self.prepare_tag(tag)
491 + if self.prepared_tag:
492 + self.write_indicator(self.prepared_tag, True)
493 + self.prepared_tag = None
494 +
495 + def choose_scalar_style(self):
496 + if self.analysis is None:
497 + self.analysis = self.analyze_scalar(self.event.value)
498 + if self.event.style == '"' or self.canonical:
499 + return '"'
500 + if not self.event.style and self.event.implicit[0]:
501 + if (not (self.simple_key_context and
502 + (self.analysis.empty or self.analysis.multiline))
503 + and (self.flow_level and self.analysis.allow_flow_plain
504 + or (not self.flow_level and self.analysis.allow_block_plain))):
505 + return ''
506 + if self.event.style and self.event.style in '|>':
507 + if (not self.flow_level and not self.simple_key_context
508 + and self.analysis.allow_block):
509 + return self.event.style
510 + if not self.event.style or self.event.style == '\'':
511 + if (self.analysis.allow_single_quoted and
512 + not (self.simple_key_context and self.analysis.multiline)):
513 + return '\''
514 + return '"'
515 +
516 + def process_scalar(self):
517 + if self.analysis is None:
518 + self.analysis = self.analyze_scalar(self.event.value)
519 + if self.style is None:
520 + self.style = self.choose_scalar_style()
521 + split = (not self.simple_key_context)
522 + #if self.analysis.multiline and split \
523 + # and (not self.style or self.style in '\'\"'):
524 + # self.write_indent()
525 + if self.style == '"':
526 + self.write_double_quoted(self.analysis.scalar, split)
527 + elif self.style == '\'':
528 + self.write_single_quoted(self.analysis.scalar, split)
529 + elif self.style == '>':
530 + self.write_folded(self.analysis.scalar)
531 + elif self.style == '|':
532 + self.write_literal(self.analysis.scalar)
533 + else:
534 + self.write_plain(self.analysis.scalar, split)
535 + self.analysis = None
536 + self.style = None
537 +
538 + # Analyzers.
539 +
540 + def prepare_version(self, version):
541 + major, minor = version
542 + if major != 1:
543 + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor))
544 + return u'%d.%d' % (major, minor)
545 +
546 + def prepare_tag_handle(self, handle):
547 + if not handle:
548 + raise EmitterError("tag handle must not be empty")
549 + if handle[0] != u'!' or handle[-1] != u'!':
550 + raise EmitterError("tag handle must start and end with '!': %r"
551 + % (handle.encode('utf-8')))
552 + for ch in handle[1:-1]:
553 + if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
554 + or ch in u'-_'):
555 + raise EmitterError("invalid character %r in the tag handle: %r"
556 + % (ch.encode('utf-8'), handle.encode('utf-8')))
557 + return handle
558 +
559 + def prepare_tag_prefix(self, prefix):
560 + if not prefix:
561 + raise EmitterError("tag prefix must not be empty")
562 + chunks = []
563 + start = end = 0
564 + if prefix[0] == u'!':
565 + end = 1
566 + while end < len(prefix):
567 + ch = prefix[end]
568 + if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
569 + or ch in u'-;/?!:@&=+$,_.~*\'()[]':
570 + end += 1
571 + else:
572 + if start < end:
573 + chunks.append(prefix[start:end])
574 + start = end = end+1
575 + data = ch.encode('utf-8')
576 + for ch in data:
577 + chunks.append(u'%%%02X' % ord(ch))
578 + if start < end:
579 + chunks.append(prefix[start:end])
580 + return u''.join(chunks)
581 +
582 + def prepare_tag(self, tag):
583 + if not tag:
584 + raise EmitterError("tag must not be empty")
585 + if tag == u'!':
586 + return tag
587 + handle = None
588 + suffix = tag
589 + prefixes = self.tag_prefixes.keys()
590 + prefixes.sort()
591 + for prefix in prefixes:
592 + if tag.startswith(prefix) \
593 + and (prefix == u'!' or len(prefix) < len(tag)):
594 + handle = self.tag_prefixes[prefix]
595 + suffix = tag[len(prefix):]
596 + chunks = []
597 + start = end = 0
598 + while end < len(suffix):
599 + ch = suffix[end]
600 + if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
601 + or ch in u'-;/?:@&=+$,_.~*\'()[]' \
602 + or (ch == u'!' and handle != u'!'):
603 + end += 1
604 + else:
605 + if start < end:
606 + chunks.append(suffix[start:end])
607 + start = end = end+1
608 + data = ch.encode('utf-8')
609 + for ch in data:
610 + chunks.append(u'%%%02X' % ord(ch))
611 + if start < end:
612 + chunks.append(suffix[start:end])
613 + suffix_text = u''.join(chunks)
614 + if handle:
615 + return u'%s%s' % (handle, suffix_text)
616 + else:
617 + return u'!<%s>' % suffix_text
618 +
619 + def prepare_anchor(self, anchor):
620 + if not anchor:
621 + raise EmitterError("anchor must not be empty")
622 + for ch in anchor:
623 + if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
624 + or ch in u'-_'):
625 + raise EmitterError("invalid character %r in the anchor: %r"
626 + % (ch.encode('utf-8'), anchor.encode('utf-8')))
627 + return anchor
628 +
629 + def analyze_scalar(self, scalar):
630 +
631 + # Empty scalar is a special case.
632 + if not scalar:
633 + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
634 + allow_flow_plain=False, allow_block_plain=True,
635 + allow_single_quoted=True, allow_double_quoted=True,
636 + allow_block=False)
637 +
638 + # Indicators and special characters.
639 + block_indicators = False
640 + flow_indicators = False
641 + line_breaks = False
642 + special_characters = False
643 +
644 + # Important whitespace combinations.
645 + leading_space = False
646 + leading_break = False
647 + trailing_space = False
648 + trailing_break = False
649 + break_space = False
650 + space_break = False
651 +
652 + # Check document indicators.
653 + if scalar.startswith(u'---') or scalar.startswith(u'...'):
654 + block_indicators = True
655 + flow_indicators = True
656 +
657 + # First character or preceded by a whitespace.
658 + preceeded_by_whitespace = True
659 +
660 + # Last character or followed by a whitespace.
661 + followed_by_whitespace = (len(scalar) == 1 or
662 + scalar[1] in u'\0 \t\r\n\x85\u2028\u2029')
663 +
664 + # The previous character is a space.
665 + previous_space = False
666 +
667 + # The previous character is a break.
668 + previous_break = False
669 +
670 + index = 0
671 + while index < len(scalar):
672 + ch = scalar[index]
673 +
674 + # Check for indicators.
675 + if index == 0:
676 + # Leading indicators are special characters.
677 + if ch in u'#,[]{}&*!|>\'\"%@`':
678 + flow_indicators = True
679 + block_indicators = True
680 + if ch in u'?:':
681 + flow_indicators = True
682 + if followed_by_whitespace:
683 + block_indicators = True
684 + if ch == u'-' and followed_by_whitespace:
685 + flow_indicators = True
686 + block_indicators = True
687 + else:
688 + # Some indicators cannot appear within a scalar as well.
689 + if ch in u',?[]{}':
690 + flow_indicators = True
691 + if ch == u':':
692 + flow_indicators = True
693 + if followed_by_whitespace:
694 + block_indicators = True
695 + if ch == u'#' and preceeded_by_whitespace:
696 + flow_indicators = True
697 + block_indicators = True
698 +
699 + # Check for line breaks, special, and unicode characters.
700 + if ch in u'\n\x85\u2028\u2029':
701 + line_breaks = True
702 + if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'):
703 + if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF'
704 + or u'\uE000' <= ch <= u'\uFFFD') and ch != u'\uFEFF':
705 + unicode_characters = True
706 + if not self.allow_unicode:
707 + special_characters = True
708 + else:
709 + special_characters = True
710 +
711 + # Detect important whitespace combinations.
712 + if ch == u' ':
713 + if index == 0:
714 + leading_space = True
715 + if index == len(scalar)-1:
716 + trailing_space = True
717 + if previous_break:
718 + break_space = True
719 + previous_space = True
720 + previous_break = False
721 + elif ch in u'\n\x85\u2028\u2029':
722 + if index == 0:
723 + leading_break = True
724 + if index == len(scalar)-1:
725 + trailing_break = True
726 + if previous_space:
727 + space_break = True
728 + previous_space = False
729 + previous_break = True
730 + else:
731 + previous_space = False
732 + previous_break = False
733 +
734 + # Prepare for the next character.
735 + index += 1
736 + preceeded_by_whitespace = (ch in u'\0 \t\r\n\x85\u2028\u2029')
737 + followed_by_whitespace = (index+1 >= len(scalar) or
738 + scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029')
739 +
740 + # Let's decide what styles are allowed.
741 + allow_flow_plain = True
742 + allow_block_plain = True
743 + allow_single_quoted = True
744 + allow_double_quoted = True
745 + allow_block = True
746 +
747 + # Leading and trailing whitespaces are bad for plain scalars.
748 + if (leading_space or leading_break
749 + or trailing_space or trailing_break):
750 + allow_flow_plain = allow_block_plain = False
751 +
752 + # We do not permit trailing spaces for block scalars.
753 + if trailing_space:
754 + allow_block = False
755 +
756 + # Spaces at the beginning of a new line are only acceptable for block
757 + # scalars.
758 + if break_space:
759 + allow_flow_plain = allow_block_plain = allow_single_quoted = False
760 +
761 + # Spaces followed by breaks, as well as special character are only
762 + # allowed for double quoted scalars.
763 + if space_break or special_characters:
764 + allow_flow_plain = allow_block_plain = \
765 + allow_single_quoted = allow_block = False
766 +
767 + # Although the plain scalar writer supports breaks, we never emit
768 + # multiline plain scalars.
769 + if line_breaks:
770 + allow_flow_plain = allow_block_plain = False
771 +
772 + # Flow indicators are forbidden for flow plain scalars.
773 + if flow_indicators:
774 + allow_flow_plain = False
775 +
776 + # Block indicators are forbidden for block plain scalars.
777 + if block_indicators:
778 + allow_block_plain = False
779 +
780 + return ScalarAnalysis(scalar=scalar,
781 + empty=False, multiline=line_breaks,
782 + allow_flow_plain=allow_flow_plain,
783 + allow_block_plain=allow_block_plain,
784 + allow_single_quoted=allow_single_quoted,
785 + allow_double_quoted=allow_double_quoted,
786 + allow_block=allow_block)
787 +
788 + # Writers.
789 +
790 + def flush_stream(self):
791 + if hasattr(self.stream, 'flush'):
792 + self.stream.flush()
793 +
794 + def write_stream_start(self):
795 + # Write BOM if needed.
796 + if self.encoding and self.encoding.startswith('utf-16'):
797 + self.stream.write(u'\uFEFF'.encode(self.encoding))
798 +
799 + def write_stream_end(self):
800 + self.flush_stream()
801 +
802 + def write_indicator(self, indicator, need_whitespace,
803 + whitespace=False, indention=False):
804 + if self.whitespace or not need_whitespace:
805 + data = indicator
806 + else:
807 + data = u' '+indicator
808 + self.whitespace = whitespace
809 + self.indention = self.indention and indention
810 + self.column += len(data)
811 + self.open_ended = False
812 + if self.encoding:
813 + data = data.encode(self.encoding)
814 + self.stream.write(data)
815 +
816 + def write_indent(self):
817 + indent = self.indent or 0
818 + if not self.indention or self.column > indent \
819 + or (self.column == indent and not self.whitespace):
820 + self.write_line_break()
821 + if self.column < indent:
822 + self.whitespace = True
823 + data = u' '*(indent-self.column)
824 + self.column = indent
825 + if self.encoding:
826 + data = data.encode(self.encoding)
827 + self.stream.write(data)
828 +
829 + def write_line_break(self, data=None):
830 + if data is None:
831 + data = self.best_line_break
832 + self.whitespace = True
833 + self.indention = True
834 + self.line += 1
835 + self.column = 0
836 + if self.encoding:
837 + data = data.encode(self.encoding)
838 + self.stream.write(data)
839 +
840 + def write_version_directive(self, version_text):
841 + data = u'%%YAML %s' % version_text
842 + if self.encoding:
843 + data = data.encode(self.encoding)
844 + self.stream.write(data)
845 + self.write_line_break()
846 +
847 + def write_tag_directive(self, handle_text, prefix_text):
848 + data = u'%%TAG %s %s' % (handle_text, prefix_text)
849 + if self.encoding:
850 + data = data.encode(self.encoding)
851 + self.stream.write(data)
852 + self.write_line_break()
853 +
854 + # Scalar streams.
855 +
856 + def write_single_quoted(self, text, split=True):
857 + self.write_indicator(u'\'', True)
858 + spaces = False
859 + breaks = False
860 + start = end = 0
861 + while end <= len(text):
862 + ch = None
863 + if end < len(text):
864 + ch = text[end]
865 + if spaces:
866 + if ch is None or ch != u' ':
867 + if start+1 == end and self.column > self.best_width and split \
868 + and start != 0 and end != len(text):
869 + self.write_indent()
870 + else:
871 + data = text[start:end]
872 + self.column += len(data)
873 + if self.encoding:
874 + data = data.encode(self.encoding)
875 + self.stream.write(data)
876 + start = end
877 + elif breaks:
878 + if ch is None or ch not in u'\n\x85\u2028\u2029':
879 + if text[start] == u'\n':
880 + self.write_line_break()
881 + for br in text[start:end]:
882 + if br == u'\n':
883 + self.write_line_break()
884 + else:
885 + self.write_line_break(br)
886 + self.write_indent()
887 + start = end
888 + else:
889 + if ch is None or ch in u' \n\x85\u2028\u2029' or ch == u'\'':
890 + if start < end:
891 + data = text[start:end]
892 + self.column += len(data)
893 + if self.encoding:
894 + data = data.encode(self.encoding)
895 + self.stream.write(data)
896 + start = end
897 + if ch == u'\'':
898 + data = u'\'\''
899 + self.column += 2
900 + if self.encoding:
901 + data = data.encode(self.encoding)
902 + self.stream.write(data)
903 + start = end + 1
904 + if ch is not None:
905 + spaces = (ch == u' ')
906 + breaks = (ch in u'\n\x85\u2028\u2029')
907 + end += 1
908 + self.write_indicator(u'\'', False)
909 +
910 + ESCAPE_REPLACEMENTS = {
911 + u'\0': u'0',
912 + u'\x07': u'a',
913 + u'\x08': u'b',
914 + u'\x09': u't',
915 + u'\x0A': u'n',
916 + u'\x0B': u'v',
917 + u'\x0C': u'f',
918 + u'\x0D': u'r',
919 + u'\x1B': u'e',
920 + u'\"': u'\"',
921 + u'\\': u'\\',
922 + u'\x85': u'N',
923 + u'\xA0': u'_',
924 + u'\u2028': u'L',
925 + u'\u2029': u'P',
926 + }
927 +
928 + def write_double_quoted(self, text, split=True):
929 + self.write_indicator(u'"', True)
930 + start = end = 0
931 + while end <= len(text):
932 + ch = None
933 + if end < len(text):
934 + ch = text[end]
935 + if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \
936 + or not (u'\x20' <= ch <= u'\x7E'
937 + or (self.allow_unicode
938 + and (u'\xA0' <= ch <= u'\uD7FF'
939 + or u'\uE000' <= ch <= u'\uFFFD'))):
940 + if start < end:
941 + data = text[start:end]
942 + self.column += len(data)
943 + if self.encoding:
944 + data = data.encode(self.encoding)
945 + self.stream.write(data)
946 + start = end
947 + if ch is not None:
948 + if ch in self.ESCAPE_REPLACEMENTS:
949 + data = u'\\'+self.ESCAPE_REPLACEMENTS[ch]
950 + elif ch <= u'\xFF':
951 + data = u'\\x%02X' % ord(ch)
952 + elif ch <= u'\uFFFF':
953 + data = u'\\u%04X' % ord(ch)
954 + else:
955 + data = u'\\U%08X' % ord(ch)
956 + self.column += len(data)
957 + if self.encoding:
958 + data = data.encode(self.encoding)
959 + self.stream.write(data)
960 + start = end+1
961 + if 0 < end < len(text)-1 and (ch == u' ' or start >= end) \
962 + and self.column+(end-start) > self.best_width and split:
963 + data = text[start:end]+u'\\'
964 + if start < end:
965 + start = end
966 + self.column += len(data)
967 + if self.encoding:
968 + data = data.encode(self.encoding)
969 + self.stream.write(data)
970 + self.write_indent()
971 + self.whitespace = False
972 + self.indention = False
973 + if text[start] == u' ':
974 + data = u'\\'
975 + self.column += len(data)
976 + if self.encoding:
977 + data = data.encode(self.encoding)
978 + self.stream.write(data)
979 + end += 1
980 + self.write_indicator(u'"', False)
981 +
982 + def determine_block_hints(self, text):
983 + hints = u''
984 + if text:
985 + if text[0] in u' \n\x85\u2028\u2029':
986 + hints += unicode(self.best_indent)
987 + if text[-1] not in u'\n\x85\u2028\u2029':
988 + hints += u'-'
989 + elif len(text) == 1 or text[-2] in u'\n\x85\u2028\u2029':
990 + hints += u'+'
991 + return hints
992 +
993 + def write_folded(self, text):
994 + hints = self.determine_block_hints(text)
995 + self.write_indicator(u'>'+hints, True)
996 + if hints[-1:] == u'+':
997 + self.open_ended = True
998 + self.write_line_break()
999 + leading_space = True
1000 + spaces = False
1001 + breaks = True
1002 + start = end = 0
1003 + while end <= len(text):
1004 + ch = None
1005 + if end < len(text):
1006 + ch = text[end]
1007 + if breaks:
1008 + if ch is None or ch not in u'\n\x85\u2028\u2029':
1009 + if not leading_space and ch is not None and ch != u' ' \
1010 + and text[start] == u'\n':
1011 + self.write_line_break()
1012 + leading_space = (ch == u' ')
1013 + for br in text[start:end]:
1014 + if br == u'\n':
1015 + self.write_line_break()
1016 + else:
1017 + self.write_line_break(br)
1018 + if ch is not None:
1019 + self.write_indent()
1020 + start = end
1021 + elif spaces:
1022 + if ch != u' ':
1023 + if start+1 == end and self.column > self.best_width:
1024 + self.write_indent()
1025 + else:
1026 + data = text[start:end]
1027 + self.column += len(data)
1028 + if self.encoding:
1029 + data = data.encode(self.encoding)
1030 + self.stream.write(data)
1031 + start = end
1032 + else:
1033 + if ch is None or ch in u' \n\x85\u2028\u2029':
1034 + data = text[start:end]
1035 + self.column += len(data)
1036 + if self.encoding:
1037 + data = data.encode(self.encoding)
1038 + self.stream.write(data)
1039 + if ch is None:
1040 + self.write_line_break()
1041 + start = end
1042 + if ch is not None:
1043 + breaks = (ch in u'\n\x85\u2028\u2029')
1044 + spaces = (ch == u' ')
1045 + end += 1
1046 +
1047 + def write_literal(self, text):
1048 + hints = self.determine_block_hints(text)
1049 + self.write_indicator(u'|'+hints, True)
1050 + if hints[-1:] == u'+':
1051 + self.open_ended = True
1052 + self.write_line_break()
1053 + breaks = True
1054 + start = end = 0
1055 + while end <= len(text):
1056 + ch = None
1057 + if end < len(text):
1058 + ch = text[end]
1059 + if breaks:
1060 + if ch is None or ch not in u'\n\x85\u2028\u2029':
1061 + for br in text[start:end]:
1062 + if br == u'\n':
1063 + self.write_line_break()
1064 + else:
1065 + self.write_line_break(br)
1066 + if ch is not None:
1067 + self.write_indent()
1068 + start = end
1069 + else:
1070 + if ch is None or ch in u'\n\x85\u2028\u2029':
1071 + data = text[start:end]
1072 + if self.encoding:
1073 + data = data.encode(self.encoding)
1074 + self.stream.write(data)
1075 + if ch is None:
1076 + self.write_line_break()
1077 + start = end
1078 + if ch is not None:
1079 + breaks = (ch in u'\n\x85\u2028\u2029')
1080 + end += 1
1081 +
1082 + def write_plain(self, text, split=True):
1083 + if self.root_context:
1084 + self.open_ended = True
1085 + if not text:
1086 + return
1087 + if not self.whitespace:
1088 + data = u' '
1089 + self.column += len(data)
1090 + if self.encoding:
1091 + data = data.encode(self.encoding)
1092 + self.stream.write(data)
1093 + self.whitespace = False
1094 + self.indention = False
1095 + spaces = False
1096 + breaks = False
1097 + start = end = 0
1098 + while end <= len(text):
1099 + ch = None
1100 + if end < len(text):
1101 + ch = text[end]
1102 + if spaces:
1103 + if ch != u' ':
1104 + if start+1 == end and self.column > self.best_width and split:
1105 + self.write_indent()
1106 + self.whitespace = False
1107 + self.indention = False
1108 + else:
1109 + data = text[start:end]
1110 + self.column += len(data)
1111 + if self.encoding:
1112 + data = data.encode(self.encoding)
1113 + self.stream.write(data)
1114 + start = end
1115 + elif breaks:
1116 + if ch not in u'\n\x85\u2028\u2029':
1117 + if text[start] == u'\n':
1118 + self.write_line_break()
1119 + for br in text[start:end]:
1120 + if br == u'\n':
1121 + self.write_line_break()
1122 + else:
1123 + self.write_line_break(br)
1124 + self.write_indent()
1125 + self.whitespace = False
1126 + self.indention = False
1127 + start = end
1128 + else:
1129 + if ch is None or ch in u' \n\x85\u2028\u2029':
1130 + data = text[start:end]
1131 + self.column += len(data)
1132 + if self.encoding:
1133 + data = data.encode(self.encoding)
1134 + self.stream.write(data)
1135 + start = end
1136 + if ch is not None:
1137 + spaces = (ch == u' ')
1138 + breaks = (ch in u'\n\x85\u2028\u2029')
1139 + end += 1
1140 +
python.d/python_modules/pyyaml2/error.py new
+75
@@ -0,0 +1,75 @@
1 +
2 +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError']
3 +
4 +class Mark(object):
5 +
6 + def __init__(self, name, index, line, column, buffer, pointer):
7 + self.name = name
8 + self.index = index
9 + self.line = line
10 + self.column = column
11 + self.buffer = buffer
12 + self.pointer = pointer
13 +
14 + def get_snippet(self, indent=4, max_length=75):
15 + if self.buffer is None:
16 + return None
17 + head = ''
18 + start = self.pointer
19 + while start > 0 and self.buffer[start-1] not in u'\0\r\n\x85\u2028\u2029':
20 + start -= 1
21 + if self.pointer-start > max_length/2-1:
22 + head = ' ... '
23 + start += 5
24 + break
25 + tail = ''
26 + end = self.pointer
27 + while end < len(self.buffer) and self.buffer[end] not in u'\0\r\n\x85\u2028\u2029':
28 + end += 1
29 + if end-self.pointer > max_length/2-1:
30 + tail = ' ... '
31 + end -= 5
32 + break
33 + snippet = self.buffer[start:end].encode('utf-8')
34 + return ' '*indent + head + snippet + tail + '\n' \
35 + + ' '*(indent+self.pointer-start+len(head)) + '^'
36 +
37 + def __str__(self):
38 + snippet = self.get_snippet()
39 + where = " in \"%s\", line %d, column %d" \
40 + % (self.name, self.line+1, self.column+1)
41 + if snippet is not None:
42 + where += ":\n"+snippet
43 + return where
44 +
45 +class YAMLError(Exception):
46 + pass
47 +
48 +class MarkedYAMLError(YAMLError):
49 +
50 + def __init__(self, context=None, context_mark=None,
51 + problem=None, problem_mark=None, note=None):
52 + self.context = context
53 + self.context_mark = context_mark
54 + self.problem = problem
55 + self.problem_mark = problem_mark
56 + self.note = note
57 +
58 + def __str__(self):
59 + lines = []
60 + if self.context is not None:
61 + lines.append(self.context)
62 + if self.context_mark is not None \
63 + and (self.problem is None or self.problem_mark is None
64 + or self.context_mark.name != self.problem_mark.name
65 + or self.context_mark.line != self.problem_mark.line
66 + or self.context_mark.column != self.problem_mark.column):
67 + lines.append(str(self.context_mark))
68 + if self.problem is not None:
69 + lines.append(self.problem)
70 + if self.problem_mark is not None:
71 + lines.append(str(self.problem_mark))
72 + if self.note is not None:
73 + lines.append(self.note)
74 + return '\n'.join(lines)
75 +
python.d/python_modules/pyyaml2/events.py new
+86
@@ -0,0 +1,86 @@
1 +
2 +# Abstract classes.
3 +
4 +class Event(object):
5 + def __init__(self, start_mark=None, end_mark=None):
6 + self.start_mark = start_mark
7 + self.end_mark = end_mark
8 + def __repr__(self):
9 + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value']
10 + if hasattr(self, key)]
11 + arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
12 + for key in attributes])
13 + return '%s(%s)' % (self.__class__.__name__, arguments)
14 +
15 +class NodeEvent(Event):
16 + def __init__(self, anchor, start_mark=None, end_mark=None):
17 + self.anchor = anchor
18 + self.start_mark = start_mark
19 + self.end_mark = end_mark
20 +
21 +class CollectionStartEvent(NodeEvent):
22 + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None,
23 + flow_style=None):
24 + self.anchor = anchor
25 + self.tag = tag
26 + self.implicit = implicit
27 + self.start_mark = start_mark
28 + self.end_mark = end_mark
29 + self.flow_style = flow_style
30 +
31 +class CollectionEndEvent(Event):
32 + pass
33 +
34 +# Implementations.
35 +
36 +class StreamStartEvent(Event):
37 + def __init__(self, start_mark=None, end_mark=None, encoding=None):
38 + self.start_mark = start_mark
39 + self.end_mark = end_mark
40 + self.encoding = encoding
41 +
42 +class StreamEndEvent(Event):
43 + pass
44 +
45 +class DocumentStartEvent(Event):
46 + def __init__(self, start_mark=None, end_mark=None,
47 + explicit=None, version=None, tags=None):
48 + self.start_mark = start_mark
49 + self.end_mark = end_mark
50 + self.explicit = explicit
51 + self.version = version
52 + self.tags = tags
53 +
54 +class DocumentEndEvent(Event):
55 + def __init__(self, start_mark=None, end_mark=None,
56 + explicit=None):
57 + self.start_mark = start_mark
58 + self.end_mark = end_mark
59 + self.explicit = explicit
60 +
61 +class AliasEvent(NodeEvent):
62 + pass
63 +
64 +class ScalarEvent(NodeEvent):
65 + def __init__(self, anchor, tag, implicit, value,
66 + start_mark=None, end_mark=None, style=None):
67 + self.anchor = anchor
68 + self.tag = tag
69 + self.implicit = implicit
70 + self.value = value
71 + self.start_mark = start_mark
72 + self.end_mark = end_mark
73 + self.style = style
74 +
75 +class SequenceStartEvent(CollectionStartEvent):
76 + pass
77 +
78 +class SequenceEndEvent(CollectionEndEvent):
79 + pass
80 +
81 +class MappingStartEvent(CollectionStartEvent):
82 + pass
83 +
84 +class MappingEndEvent(CollectionEndEvent):
85 + pass
86 +
python.d/python_modules/pyyaml2/loader.py new
+40
@@ -0,0 +1,40 @@
1 +
2 +__all__ = ['BaseLoader', 'SafeLoader', 'Loader']
3 +
4 +from reader import *
5 +from scanner import *
6 +from parser import *
7 +from composer import *
8 +from constructor import *
9 +from resolver import *
10 +
11 +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):
12 +
13 + def __init__(self, stream):
14 + Reader.__init__(self, stream)
15 + Scanner.__init__(self)
16 + Parser.__init__(self)
17 + Composer.__init__(self)
18 + BaseConstructor.__init__(self)
19 + BaseResolver.__init__(self)
20 +
21 +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver):
22 +
23 + def __init__(self, stream):
24 + Reader.__init__(self, stream)
25 + Scanner.__init__(self)
26 + Parser.__init__(self)
27 + Composer.__init__(self)
28 + SafeConstructor.__init__(self)
29 + Resolver.__init__(self)
30 +
31 +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
32 +
33 + def __init__(self, stream):
34 + Reader.__init__(self, stream)
35 + Scanner.__init__(self)
36 + Parser.__init__(self)
37 + Composer.__init__(self)
38 + Constructor.__init__(self)
39 + Resolver.__init__(self)
40 +
python.d/python_modules/pyyaml2/nodes.py new
+49
@@ -0,0 +1,49 @@
1 +
2 +class Node(object):
3 + def __init__(self, tag, value, start_mark, end_mark):
4 + self.tag = tag
5 + self.value = value
6 + self.start_mark = start_mark
7 + self.end_mark = end_mark
8 + def __repr__(self):
9 + value = self.value
10 + #if isinstance(value, list):
11 + # if len(value) == 0:
12 + # value = '<empty>'
13 + # elif len(value) == 1:
14 + # value = '<1 item>'
15 + # else:
16 + # value = '<%d items>' % len(value)
17 + #else:
18 + # if len(value) > 75:
19 + # value = repr(value[:70]+u' ... ')
20 + # else:
21 + # value = repr(value)
22 + value = repr(value)
23 + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
24 +
25 +class ScalarNode(Node):
26 + id = 'scalar'
27 + def __init__(self, tag, value,
28 + start_mark=None, end_mark=None, style=None):
29 + self.tag = tag
30 + self.value = value
31 + self.start_mark = start_mark
32 + self.end_mark = end_mark
33 + self.style = style
34 +
35 +class CollectionNode(Node):
36 + def __init__(self, tag, value,
37 + start_mark=None, end_mark=None, flow_style=None):
38 + self.tag = tag
39 + self.value = value
40 + self.start_mark = start_mark
41 + self.end_mark = end_mark
42 + self.flow_style = flow_style
43 +
44 +class SequenceNode(CollectionNode):
45 + id = 'sequence'
46 +
47 +class MappingNode(CollectionNode):
48 + id = 'mapping'
49 +
python.d/python_modules/pyyaml2/parser.py new
+589
@@ -0,0 +1,589 @@
1 +
2 +# The following YAML grammar is LL(1) and is parsed by a recursive descent
3 +# parser.
4 +#
5 +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
6 +# implicit_document ::= block_node DOCUMENT-END*
7 +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
8 +# block_node_or_indentless_sequence ::=
9 +# ALIAS
10 +# | properties (block_content | indentless_block_sequence)?
11 +# | block_content
12 +# | indentless_block_sequence
13 +# block_node ::= ALIAS
14 +# | properties block_content?
15 +# | block_content
16 +# flow_node ::= ALIAS
17 +# | properties flow_content?
18 +# | flow_content
19 +# properties ::= TAG ANCHOR? | ANCHOR TAG?
20 +# block_content ::= block_collection | flow_collection | SCALAR
21 +# flow_content ::= flow_collection | SCALAR
22 +# block_collection ::= block_sequence | block_mapping
23 +# flow_collection ::= flow_sequence | flow_mapping
24 +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
25 +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
26 +# block_mapping ::= BLOCK-MAPPING_START
27 +# ((KEY block_node_or_indentless_sequence?)?
28 +# (VALUE block_node_or_indentless_sequence?)?)*
29 +# BLOCK-END
30 +# flow_sequence ::= FLOW-SEQUENCE-START
31 +# (flow_sequence_entry FLOW-ENTRY)*
32 +# flow_sequence_entry?
33 +# FLOW-SEQUENCE-END
34 +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
35 +# flow_mapping ::= FLOW-MAPPING-START
36 +# (flow_mapping_entry FLOW-ENTRY)*
37 +# flow_mapping_entry?
38 +# FLOW-MAPPING-END
39 +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
40 +#
41 +# FIRST sets:
42 +#
43 +# stream: { STREAM-START }
44 +# explicit_document: { DIRECTIVE DOCUMENT-START }
45 +# implicit_document: FIRST(block_node)
46 +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
47 +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
48 +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
49 +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
50 +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
51 +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
52 +# block_sequence: { BLOCK-SEQUENCE-START }
53 +# block_mapping: { BLOCK-MAPPING-START }
54 +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
55 +# indentless_sequence: { ENTRY }
56 +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
57 +# flow_sequence: { FLOW-SEQUENCE-START }
58 +# flow_mapping: { FLOW-MAPPING-START }
59 +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
60 +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
61 +
62 +__all__ = ['Parser', 'ParserError']
63 +
64 +from error import MarkedYAMLError
65 +from tokens import *
66 +from events import *
67 +from scanner import *
68 +
69 +class ParserError(MarkedYAMLError):
70 + pass
71 +
72 +class Parser(object):
73 + # Since writing a recursive-descendant parser is a straightforward task, we
74 + # do not give many comments here.
75 +
76 + DEFAULT_TAGS = {
77 + u'!': u'!',
78 + u'!!': u'tag:yaml.org,2002:',
79 + }
80 +
81 + def __init__(self):
82 + self.current_event = None
83 + self.yaml_version = None
84 + self.tag_handles = {}
85 + self.states = []
86 + self.marks = []
87 + self.state = self.parse_stream_start
88 +
89 + def dispose(self):
90 + # Reset the state attributes (to clear self-references)
91 + self.states = []
92 + self.state = None
93 +
94 + def check_event(self, *choices):
95 + # Check the type of the next event.
96 + if self.current_event is None:
97 + if self.state:
98 + self.current_event = self.state()
99 + if self.current_event is not None:
100 + if not choices:
101 + return True
102 + for choice in choices:
103 + if isinstance(self.current_event, choice):
104 + return True
105 + return False
106 +
107 + def peek_event(self):
108 + # Get the next event.
109 + if self.current_event is None:
110 + if self.state:
111 + self.current_event = self.state()
112 + return self.current_event
113 +
114 + def get_event(self):
115 + # Get the next event and proceed further.
116 + if self.current_event is None:
117 + if self.state:
118 + self.current_event = self.state()
119 + value = self.current_event
120 + self.current_event = None
121 + return value
122 +
123 + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
124 + # implicit_document ::= block_node DOCUMENT-END*
125 + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
126 +
127 + def parse_stream_start(self):
128 +
129 + # Parse the stream start.
130 + token = self.get_token()
131 + event = StreamStartEvent(token.start_mark, token.end_mark,
132 + encoding=token.encoding)
133 +
134 + # Prepare the next state.
135 + self.state = self.parse_implicit_document_start
136 +
137 + return event
138 +
139 + def parse_implicit_document_start(self):
140 +
141 + # Parse an implicit document.
142 + if not self.check_token(DirectiveToken, DocumentStartToken,
143 + StreamEndToken):
144 + self.tag_handles = self.DEFAULT_TAGS
145 + token = self.peek_token()
146 + start_mark = end_mark = token.start_mark
147 + event = DocumentStartEvent(start_mark, end_mark,
148 + explicit=False)
149 +
150 + # Prepare the next state.
151 + self.states.append(self.parse_document_end)
152 + self.state = self.parse_block_node
153 +
154 + return event
155 +
156 + else:
157 + return self.parse_document_start()
158 +
159 + def parse_document_start(self):
160 +
161 + # Parse any extra document end indicators.
162 + while self.check_token(DocumentEndToken):
163 + self.get_token()
164 +
165 + # Parse an explicit document.
166 + if not self.check_token(StreamEndToken):
167 + token = self.peek_token()
168 + start_mark = token.start_mark
169 + version, tags = self.process_directives()
170 + if not self.check_token(DocumentStartToken):
171 + raise ParserError(None, None,
172 + "expected '<document start>', but found %r"
173 + % self.peek_token().id,
174 + self.peek_token().start_mark)
175 + token = self.get_token()
176 + end_mark = token.end_mark
177 + event = DocumentStartEvent(start_mark, end_mark,
178 + explicit=True, version=version, tags=tags)
179 + self.states.append(self.parse_document_end)
180 + self.state = self.parse_document_content
181 + else:
182 + # Parse the end of the stream.
183 + token = self.get_token()
184 + event = StreamEndEvent(token.start_mark, token.end_mark)
185 + assert not self.states
186 + assert not self.marks
187 + self.state = None
188 + return event
189 +
190 + def parse_document_end(self):
191 +
192 + # Parse the document end.
193 + token = self.peek_token()
194 + start_mark = end_mark = token.start_mark
195 + explicit = False
196 + if self.check_token(DocumentEndToken):
197 + token = self.get_token()
198 + end_mark = token.end_mark
199 + explicit = True
200 + event = DocumentEndEvent(start_mark, end_mark,
201 + explicit=explicit)
202 +
203 + # Prepare the next state.
204 + self.state = self.parse_document_start
205 +
206 + return event
207 +
208 + def parse_document_content(self):
209 + if self.check_token(DirectiveToken,
210 + DocumentStartToken, DocumentEndToken, StreamEndToken):
211 + event = self.process_empty_scalar(self.peek_token().start_mark)
212 + self.state = self.states.pop()
213 + return event
214 + else:
215 + return self.parse_block_node()
216 +
217 + def process_directives(self):
218 + self.yaml_version = None
219 + self.tag_handles = {}
220 + while self.check_token(DirectiveToken):
221 + token = self.get_token()
222 + if token.name == u'YAML':
223 + if self.yaml_version is not None:
224 + raise ParserError(None, None,
225 + "found duplicate YAML directive", token.start_mark)
226 + major, minor = token.value
227 + if major != 1:
228 + raise ParserError(None, None,
229 + "found incompatible YAML document (version 1.* is required)",
230 + token.start_mark)
231 + self.yaml_version = token.value
232 + elif token.name == u'TAG':
233 + handle, prefix = token.value
234 + if handle in self.tag_handles:
235 + raise ParserError(None, None,
236 + "duplicate tag handle %r" % handle.encode('utf-8'),
237 + token.start_mark)
238 + self.tag_handles[handle] = prefix
239 + if self.tag_handles:
240 + value = self.yaml_version, self.tag_handles.copy()
241 + else:
242 + value = self.yaml_version, None
243 + for key in self.DEFAULT_TAGS:
244 + if key not in self.tag_handles:
245 + self.tag_handles[key] = self.DEFAULT_TAGS[key]
246 + return value
247 +
248 + # block_node_or_indentless_sequence ::= ALIAS
249 + # | properties (block_content | indentless_block_sequence)?
250 + # | block_content
251 + # | indentless_block_sequence
252 + # block_node ::= ALIAS
253 + # | properties block_content?
254 + # | block_content
255 + # flow_node ::= ALIAS
256 + # | properties flow_content?
257 + # | flow_content
258 + # properties ::= TAG ANCHOR? | ANCHOR TAG?
259 + # block_content ::= block_collection | flow_collection | SCALAR
260 + # flow_content ::= flow_collection | SCALAR
261 + # block_collection ::= block_sequence | block_mapping
262 + # flow_collection ::= flow_sequence | flow_mapping
263 +
264 + def parse_block_node(self):
265 + return self.parse_node(block=True)
266 +
267 + def parse_flow_node(self):
268 + return self.parse_node()
269 +
270 + def parse_block_node_or_indentless_sequence(self):
271 + return self.parse_node(block=True, indentless_sequence=True)
272 +
273 + def parse_node(self, block=False, indentless_sequence=False):
274 + if self.check_token(AliasToken):
275 + token = self.get_token()
276 + event = AliasEvent(token.value, token.start_mark, token.end_mark)
277 + self.state = self.states.pop()
278 + else:
279 + anchor = None
280 + tag = None
281 + start_mark = end_mark = tag_mark = None
282 + if self.check_token(AnchorToken):
283 + token = self.get_token()
284 + start_mark = token.start_mark
285 + end_mark = token.end_mark
286 + anchor = token.value
287 + if self.check_token(TagToken):
288 + token = self.get_token()
289 + tag_mark = token.start_mark
290 + end_mark = token.end_mark
291 + tag = token.value
292 + elif self.check_token(TagToken):
293 + token = self.get_token()
294 + start_mark = tag_mark = token.start_mark
295 + end_mark = token.end_mark
296 + tag = token.value
297 + if self.check_token(AnchorToken):
298 + token = self.get_token()
299 + end_mark = token.end_mark
300 + anchor = token.value
301 + if tag is not None:
302 + handle, suffix = tag
303 + if handle is not None:
304 + if handle not in self.tag_handles:
305 + raise ParserError("while parsing a node", start_mark,
306 + "found undefined tag handle %r" % handle.encode('utf-8'),
307 + tag_mark)
308 + tag = self.tag_handles[handle]+suffix
309 + else:
310 + tag = suffix
311 + #if tag == u'!':
312 + # raise ParserError("while parsing a node", start_mark,
313 + # "found non-specific tag '!'", tag_mark,
314 + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
315 + if start_mark is None:
316 + start_mark = end_mark = self.peek_token().start_mark
317 + event = None
318 + implicit = (tag is None or tag == u'!')
319 + if indentless_sequence and self.check_token(BlockEntryToken):
320 + end_mark = self.peek_token().end_mark
321 + event = SequenceStartEvent(anchor, tag, implicit,
322 + start_mark, end_mark)
323 + self.state = self.parse_indentless_sequence_entry
324 + else:
325 + if self.check_token(ScalarToken):
326 + token = self.get_token()
327 + end_mark = token.end_mark
328 + if (token.plain and tag is None) or tag == u'!':
329 + implicit = (True, False)
330 + elif tag is None:
331 + implicit = (False, True)
332 + else:
333 + implicit = (False, False)
334 + event = ScalarEvent(anchor, tag, implicit, token.value,
335 + start_mark, end_mark, style=token.style)
336 + self.state = self.states.pop()
337 + elif self.check_token(FlowSequenceStartToken):
338 + end_mark = self.peek_token().end_mark
339 + event = SequenceStartEvent(anchor, tag, implicit,
340 + start_mark, end_mark, flow_style=True)
341 + self.state = self.parse_flow_sequence_first_entry
342 + elif self.check_token(FlowMappingStartToken):
343 + end_mark = self.peek_token().end_mark
344 + event = MappingStartEvent(anchor, tag, implicit,
345 + start_mark, end_mark, flow_style=True)
346 + self.state = self.parse_flow_mapping_first_key
347 + elif block and self.check_token(BlockSequenceStartToken):
348 + end_mark = self.peek_token().start_mark
349 + event = SequenceStartEvent(anchor, tag, implicit,
350 + start_mark, end_mark, flow_style=False)
351 + self.state = self.parse_block_sequence_first_entry
352 + elif block and self.check_token(BlockMappingStartToken):
353 + end_mark = self.peek_token().start_mark
354 + event = MappingStartEvent(anchor, tag, implicit,
355 + start_mark, end_mark, flow_style=False)
356 + self.state = self.parse_block_mapping_first_key
357 + elif anchor is not None or tag is not None:
358 + # Empty scalars are allowed even if a tag or an anchor is
359 + # specified.
360 + event = ScalarEvent(anchor, tag, (implicit, False), u'',
361 + start_mark, end_mark)
362 + self.state = self.states.pop()
363 + else:
364 + if block:
365 + node = 'block'
366 + else:
367 + node = 'flow'
368 + token = self.peek_token()
369 + raise ParserError("while parsing a %s node" % node, start_mark,
370 + "expected the node content, but found %r" % token.id,
371 + token.start_mark)
372 + return event
373 +
374 + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
375 +
376 + def parse_block_sequence_first_entry(self):
377 + token = self.get_token()
378 + self.marks.append(token.start_mark)
379 + return self.parse_block_sequence_entry()
380 +
381 + def parse_block_sequence_entry(self):
382 + if self.check_token(BlockEntryToken):
383 + token = self.get_token()
384 + if not self.check_token(BlockEntryToken, BlockEndToken):
385 + self.states.append(self.parse_block_sequence_entry)
386 + return self.parse_block_node()
387 + else:
388 + self.state = self.parse_block_sequence_entry
389 + return self.process_empty_scalar(token.end_mark)
390 + if not self.check_token(BlockEndToken):
391 + token = self.peek_token()
392 + raise ParserError("while parsing a block collection", self.marks[-1],
393 + "expected <block end>, but found %r" % token.id, token.start_mark)
394 + token = self.get_token()
395 + event = SequenceEndEvent(token.start_mark, token.end_mark)
396 + self.state = self.states.pop()
397 + self.marks.pop()
398 + return event
399 +
400 + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
401 +
402 + def parse_indentless_sequence_entry(self):
403 + if self.check_token(BlockEntryToken):
404 + token = self.get_token()
405 + if not self.check_token(BlockEntryToken,
406 + KeyToken, ValueToken, BlockEndToken):
407 + self.states.append(self.parse_indentless_sequence_entry)
408 + return self.parse_block_node()
409 + else:
410 + self.state = self.parse_indentless_sequence_entry
411 + return self.process_empty_scalar(token.end_mark)
412 + token = self.peek_token()
413 + event = SequenceEndEvent(token.start_mark, token.start_mark)
414 + self.state = self.states.pop()
415 + return event
416 +
417 + # block_mapping ::= BLOCK-MAPPING_START
418 + # ((KEY block_node_or_indentless_sequence?)?
419 + # (VALUE block_node_or_indentless_sequence?)?)*
420 + # BLOCK-END
421 +
422 + def parse_block_mapping_first_key(self):
423 + token = self.get_token()
424 + self.marks.append(token.start_mark)
425 + return self.parse_block_mapping_key()
426 +
427 + def parse_block_mapping_key(self):
428 + if self.check_token(KeyToken):
429 + token = self.get_token()
430 + if not self.check_token(KeyToken, ValueToken, BlockEndToken):
431 + self.states.append(self.parse_block_mapping_value)
432 + return self.parse_block_node_or_indentless_sequence()
433 + else:
434 + self.state = self.parse_block_mapping_value
435 + return self.process_empty_scalar(token.end_mark)
436 + if not self.check_token(BlockEndToken):
437 + token = self.peek_token()
438 + raise ParserError("while parsing a block mapping", self.marks[-1],
439 + "expected <block end>, but found %r" % token.id, token.start_mark)
440 + token = self.get_token()
441 + event = MappingEndEvent(token.start_mark, token.end_mark)
442 + self.state = self.states.pop()
443 + self.marks.pop()
444 + return event
445 +
446 + def parse_block_mapping_value(self):
447 + if self.check_token(ValueToken):
448 + token = self.get_token()
449 + if not self.check_token(KeyToken, ValueToken, BlockEndToken):
450 + self.states.append(self.parse_block_mapping_key)
451 + return self.parse_block_node_or_indentless_sequence()
452 + else:
453 + self.state = self.parse_block_mapping_key
454 + return self.process_empty_scalar(token.end_mark)
455 + else:
456 + self.state = self.parse_block_mapping_key
457 + token = self.peek_token()
458 + return self.process_empty_scalar(token.start_mark)
459 +
460 + # flow_sequence ::= FLOW-SEQUENCE-START
461 + # (flow_sequence_entry FLOW-ENTRY)*
462 + # flow_sequence_entry?
463 + # FLOW-SEQUENCE-END
464 + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
465 + #
466 + # Note that while production rules for both flow_sequence_entry and
467 + # flow_mapping_entry are equal, their interpretations are different.
468 + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
469 + # generate an inline mapping (set syntax).
470 +
471 + def parse_flow_sequence_first_entry(self):
472 + token = self.get_token()
473 + self.marks.append(token.start_mark)
474 + return self.parse_flow_sequence_entry(first=True)
475 +
476 + def parse_flow_sequence_entry(self, first=False):
477 + if not self.check_token(FlowSequenceEndToken):
478 + if not first:
479 + if self.check_token(FlowEntryToken):
480 + self.get_token()
481 + else:
482 + token = self.peek_token()
483 + raise ParserError("while parsing a flow sequence", self.marks[-1],
484 + "expected ',' or ']', but got %r" % token.id, token.start_mark)
485 +
486 + if self.check_token(KeyToken):
487 + token = self.peek_token()
488 + event = MappingStartEvent(None, None, True,
489 + token.start_mark, token.end_mark,
490 + flow_style=True)
491 + self.state = self.parse_flow_sequence_entry_mapping_key
492 + return event
493 + elif not self.check_token(FlowSequenceEndToken):
494 + self.states.append(self.parse_flow_sequence_entry)
495 + return self.parse_flow_node()
496 + token = self.get_token()
497 + event = SequenceEndEvent(token.start_mark, token.end_mark)
498 + self.state = self.states.pop()
499 + self.marks.pop()
500 + return event
501 +
502 + def parse_flow_sequence_entry_mapping_key(self):
503 + token = self.get_token()
504 + if not self.check_token(ValueToken,
505 + FlowEntryToken, FlowSequenceEndToken):
506 + self.states.append(self.parse_flow_sequence_entry_mapping_value)
507 + return self.parse_flow_node()
508 + else:
509 + self.state = self.parse_flow_sequence_entry_mapping_value
510 + return self.process_empty_scalar(token.end_mark)
511 +
512 + def parse_flow_sequence_entry_mapping_value(self):
513 + if self.check_token(ValueToken):
514 + token = self.get_token()
515 + if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
516 + self.states.append(self.parse_flow_sequence_entry_mapping_end)
517 + return self.parse_flow_node()
518 + else:
519 + self.state = self.parse_flow_sequence_entry_mapping_end
520 + return self.process_empty_scalar(token.end_mark)
521 + else:
522 + self.state = self.parse_flow_sequence_entry_mapping_end
523 + token = self.peek_token()
524 + return self.process_empty_scalar(token.start_mark)
525 +
526 + def parse_flow_sequence_entry_mapping_end(self):
527 + self.state = self.parse_flow_sequence_entry
528 + token = self.peek_token()
529 + return MappingEndEvent(token.start_mark, token.start_mark)
530 +
531 + # flow_mapping ::= FLOW-MAPPING-START
532 + # (flow_mapping_entry FLOW-ENTRY)*
533 + # flow_mapping_entry?
534 + # FLOW-MAPPING-END
535 + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
536 +
537 + def parse_flow_mapping_first_key(self):
538 + token = self.get_token()
539 + self.marks.append(token.start_mark)
540 + return self.parse_flow_mapping_key(first=True)
541 +
542 + def parse_flow_mapping_key(self, first=False):
543 + if not self.check_token(FlowMappingEndToken):
544 + if not first:
545 + if self.check_token(FlowEntryToken):
546 + self.get_token()
547 + else:
548 + token = self.peek_token()
549 + raise ParserError("while parsing a flow mapping", self.marks[-1],
550 + "expected ',' or '}', but got %r" % token.id, token.start_mark)
551 + if self.check_token(KeyToken):
552 + token = self.get_token()
553 + if not self.check_token(ValueToken,
554 + FlowEntryToken, FlowMappingEndToken):
555 + self.states.append(self.parse_flow_mapping_value)
556 + return self.parse_flow_node()
557 + else:
558 + self.state = self.parse_flow_mapping_value
559 + return self.process_empty_scalar(token.end_mark)
560 + elif not self.check_token(FlowMappingEndToken):
561 + self.states.append(self.parse_flow_mapping_empty_value)
562 + return self.parse_flow_node()
563 + token = self.get_token()
564 + event = MappingEndEvent(token.start_mark, token.end_mark)
565 + self.state = self.states.pop()
566 + self.marks.pop()
567 + return event
568 +
569 + def parse_flow_mapping_value(self):
570 + if self.check_token(ValueToken):
571 + token = self.get_token()
572 + if not self.check_token(FlowEntryToken, FlowMappingEndToken):
573 + self.states.append(self.parse_flow_mapping_key)
574 + return self.parse_flow_node()
575 + else:
576 + self.state = self.parse_flow_mapping_key
577 + return self.process_empty_scalar(token.end_mark)
578 + else:
579 + self.state = self.parse_flow_mapping_key
580 + token = self.peek_token()
581 + return self.process_empty_scalar(token.start_mark)
582 +
583 + def parse_flow_mapping_empty_value(self):
584 + self.state = self.parse_flow_mapping_key
585 + return self.process_empty_scalar(self.peek_token().start_mark)
586 +
587 + def process_empty_scalar(self, mark):
588 + return ScalarEvent(None, None, (True, False), u'', mark, mark)
589 +
python.d/python_modules/pyyaml2/reader.py new
+190
@@ -0,0 +1,190 @@
1 +# This module contains abstractions for the input stream. You don't have to
2 +# looks further, there are no pretty code.
3 +#
4 +# We define two classes here.
5 +#
6 +# Mark(source, line, column)
7 +# It's just a record and its only use is producing nice error messages.
8 +# Parser does not use it for any other purposes.
9 +#
10 +# Reader(source, data)
11 +# Reader determines the encoding of `data` and converts it to unicode.
12 +# Reader provides the following methods and attributes:
13 +# reader.peek(length=1) - return the next `length` characters
14 +# reader.forward(length=1) - move the current position to `length` characters.
15 +# reader.index - the number of the current character.
16 +# reader.line, stream.column - the line and the column of the current character.
17 +
18 +__all__ = ['Reader', 'ReaderError']
19 +
20 +from error import YAMLError, Mark
21 +
22 +import codecs, re
23 +
24 +class ReaderError(YAMLError):
25 +
26 + def __init__(self, name, position, character, encoding, reason):
27 + self.name = name
28 + self.character = character
29 + self.position = position
30 + self.encoding = encoding
31 + self.reason = reason
32 +
33 + def __str__(self):
34 + if isinstance(self.character, str):
35 + return "'%s' codec can't decode byte #x%02x: %s\n" \
36 + " in \"%s\", position %d" \
37 + % (self.encoding, ord(self.character), self.reason,
38 + self.name, self.position)
39 + else:
40 + return "unacceptable character #x%04x: %s\n" \
41 + " in \"%s\", position %d" \
42 + % (self.character, self.reason,
43 + self.name, self.position)
44 +
45 +class Reader(object):
46 + # Reader:
47 + # - determines the data encoding and converts it to unicode,
48 + # - checks if characters are in allowed range,
49 + # - adds '\0' to the end.
50 +
51 + # Reader accepts
52 + # - a `str` object,
53 + # - a `unicode` object,
54 + # - a file-like object with its `read` method returning `str`,
55 + # - a file-like object with its `read` method returning `unicode`.
56 +
57 + # Yeah, it's ugly and slow.
58 +
59 + def __init__(self, stream):
60 + self.name = None
61 + self.stream = None
62 + self.stream_pointer = 0
63 + self.eof = True
64 + self.buffer = u''
65 + self.pointer = 0
66 + self.raw_buffer = None
67 + self.raw_decode = None
68 + self.encoding = None
69 + self.index = 0
70 + self.line = 0
71 + self.column = 0
72 + if isinstance(stream, unicode):
73 + self.name = "<unicode string>"
74 + self.check_printable(stream)
75 + self.buffer = stream+u'\0'
76 + elif isinstance(stream, str):
77 + self.name = "<string>"
78 + self.raw_buffer = stream
79 + self.determine_encoding()
80 + else:
81 + self.stream = stream
82 + self.name = getattr(stream, 'name', "<file>")
83 + self.eof = False
84 + self.raw_buffer = ''
85 + self.determine_encoding()
86 +
87 + def peek(self, index=0):
88 + try:
89 + return self.buffer[self.pointer+index]
90 + except IndexError:
91 + self.update(index+1)
92 + return self.buffer[self.pointer+index]
93 +
94 + def prefix(self, length=1):
95 + if self.pointer+length >= len(self.buffer):
96 + self.update(length)
97 + return self.buffer[self.pointer:self.pointer+length]
98 +
99 + def forward(self, length=1):
100 + if self.pointer+length+1 >= len(self.buffer):
101 + self.update(length+1)
102 + while length:
103 + ch = self.buffer[self.pointer]
104 + self.pointer += 1
105 + self.index += 1
106 + if ch in u'\n\x85\u2028\u2029' \
107 + or (ch == u'\r' and self.buffer[self.pointer] != u'\n'):
108 + self.line += 1
109 + self.column = 0
110 + elif ch != u'\uFEFF':
111 + self.column += 1
112 + length -= 1
113 +
114 + def get_mark(self):
115 + if self.stream is None:
116 + return Mark(self.name, self.index, self.line, self.column,
117 + self.buffer, self.pointer)
118 + else:
119 + return Mark(self.name, self.index, self.line, self.column,
120 + None, None)
121 +
122 + def determine_encoding(self):
123 + while not self.eof and len(self.raw_buffer) < 2:
124 + self.update_raw()
125 + if not isinstance(self.raw_buffer, unicode):
126 + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
127 + self.raw_decode = codecs.utf_16_le_decode
128 + self.encoding = 'utf-16-le'
129 + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
130 + self.raw_decode = codecs.utf_16_be_decode
131 + self.encoding = 'utf-16-be'
132 + else:
133 + self.raw_decode = codecs.utf_8_decode
134 + self.encoding = 'utf-8'
135 + self.update(1)
136 +
137 + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')
138 + def check_printable(self, data):
139 + match = self.NON_PRINTABLE.search(data)
140 + if match:
141 + character = match.group()
142 + position = self.index+(len(self.buffer)-self.pointer)+match.start()
143 + raise ReaderError(self.name, position, ord(character),
144 + 'unicode', "special characters are not allowed")
145 +
146 + def update(self, length):
147 + if self.raw_buffer is None:
148 + return
149 + self.buffer = self.buffer[self.pointer:]
150 + self.pointer = 0
151 + while len(self.buffer) < length:
152 + if not self.eof:
153 + self.update_raw()
154 + if self.raw_decode is not None:
155 + try:
156 + data, converted = self.raw_decode(self.raw_buffer,
157 + 'strict', self.eof)
158 + except UnicodeDecodeError, exc:
159 + character = exc.object[exc.start]
160 + if self.stream is not None:
161 + position = self.stream_pointer-len(self.raw_buffer)+exc.start
162 + else:
163 + position = exc.start
164 + raise ReaderError(self.name, position, character,
165 + exc.encoding, exc.reason)
166 + else:
167 + data = self.raw_buffer
168 + converted = len(data)
169 + self.check_printable(data)
170 + self.buffer += data
171 + self.raw_buffer = self.raw_buffer[converted:]
172 + if self.eof:
173 + self.buffer += u'\0'
174 + self.raw_buffer = None
175 + break
176 +
177 + def update_raw(self, size=1024):
178 + data = self.stream.read(size)
179 + if data:
180 + self.raw_buffer += data
181 + self.stream_pointer += len(data)
182 + else:
183 + self.eof = True
184 +
185 +#try:
186 +# import psyco
187 +# psyco.bind(Reader)
188 +#except ImportError:
189 +# pass
190 +
python.d/python_modules/pyyaml2/representer.py new
+484
@@ -0,0 +1,484 @@
1 +
2 +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer',
3 + 'RepresenterError']
4 +
5 +from error import *
6 +from nodes import *
7 +
8 +import datetime
9 +
10 +import sys, copy_reg, types
11 +
12 +class RepresenterError(YAMLError):
13 + pass
14 +
15 +class BaseRepresenter(object):
16 +
17 + yaml_representers = {}
18 + yaml_multi_representers = {}
19 +
20 + def __init__(self, default_style=None, default_flow_style=None):
21 + self.default_style = default_style
22 + self.default_flow_style = default_flow_style
23 + self.represented_objects = {}
24 + self.object_keeper = []
25 + self.alias_key = None
26 +
27 + def represent(self, data):
28 + node = self.represent_data(data)
29 + self.serialize(node)
30 + self.represented_objects = {}
31 + self.object_keeper = []
32 + self.alias_key = None
33 +
34 + def get_classobj_bases(self, cls):
35 + bases = [cls]
36 + for base in cls.__bases__:
37 + bases.extend(self.get_classobj_bases(base))
38 + return bases
39 +
40 + def represent_data(self, data):
41 + if self.ignore_aliases(data):
42 + self.alias_key = None
43 + else:
44 + self.alias_key = id(data)
45 + if self.alias_key is not None:
46 + if self.alias_key in self.represented_objects:
47 + node = self.represented_objects[self.alias_key]
48 + #if node is None:
49 + # raise RepresenterError("recursive objects are not allowed: %r" % data)
50 + return node
51 + #self.represented_objects[alias_key] = None
52 + self.object_keeper.append(data)
53 + data_types = type(data).__mro__
54 + if type(data) is types.InstanceType:
55 + data_types = self.get_classobj_bases(data.__class__)+list(data_types)
56 + if data_types[0] in self.yaml_representers:
57 + node = self.yaml_representers[data_types[0]](self, data)
58 + else:
59 + for data_type in data_types:
60 + if data_type in self.yaml_multi_representers:
61 + node = self.yaml_multi_representers[data_type](self, data)
62 + break
63 + else:
64 + if None in self.yaml_multi_representers:
65 + node = self.yaml_multi_representers[None](self, data)
66 + elif None in self.yaml_representers:
67 + node = self.yaml_representers[None](self, data)
68 + else:
69 + node = ScalarNode(None, unicode(data))
70 + #if alias_key is not None:
71 + # self.represented_objects[alias_key] = node
72 + return node
73 +
74 + def add_representer(cls, data_type, representer):
75 + if not 'yaml_representers' in cls.__dict__:
76 + cls.yaml_representers = cls.yaml_representers.copy()
77 + cls.yaml_representers[data_type] = representer
78 + add_representer = classmethod(add_representer)
79 +
80 + def add_multi_representer(cls, data_type, representer):
81 + if not 'yaml_multi_representers' in cls.__dict__:
82 + cls.yaml_multi_representers = cls.yaml_multi_representers.copy()
83 + cls.yaml_multi_representers[data_type] = representer
84 + add_multi_representer = classmethod(add_multi_representer)
85 +
86 + def represent_scalar(self, tag, value, style=None):
87 + if style is None:
88 + style = self.default_style
89 + node = ScalarNode(tag, value, style=style)
90 + if self.alias_key is not None:
91 + self.represented_objects[self.alias_key] = node
92 + return node
93 +
94 + def represent_sequence(self, tag, sequence, flow_style=None):
95 + value = []
96 + node = SequenceNode(tag, value, flow_style=flow_style)
97 + if self.alias_key is not None:
98 + self.represented_objects[self.alias_key] = node
99 + best_style = True
100 + for item in sequence:
101 + node_item = self.represent_data(item)
102 + if not (isinstance(node_item, ScalarNode) and not node_item.style):
103 + best_style = False
104 + value.append(node_item)
105 + if flow_style is None:
106 + if self.default_flow_style is not None:
107 + node.flow_style = self.default_flow_style
108 + else:
109 + node.flow_style = best_style
110 + return node
111 +
112 + def represent_mapping(self, tag, mapping, flow_style=None):
113 + value = []
114 + node = MappingNode(tag, value, flow_style=flow_style)
115 + if self.alias_key is not None:
116 + self.represented_objects[self.alias_key] = node
117 + best_style = True
118 + if hasattr(mapping, 'items'):
119 + mapping = mapping.items()
120 + mapping.sort()
121 + for item_key, item_value in mapping:
122 + node_key = self.represent_data(item_key)
123 + node_value = self.represent_data(item_value)
124 + if not (isinstance(node_key, ScalarNode) and not node_key.style):
125 + best_style = False
126 + if not (isinstance(node_value, ScalarNode) and not node_value.style):
127 + best_style = False
128 + value.append((node_key, node_value))
129 + if flow_style is None:
130 + if self.default_flow_style is not None:
131 + node.flow_style = self.default_flow_style
132 + else:
133 + node.flow_style = best_style
134 + return node
135 +
136 + def ignore_aliases(self, data):
137 + return False
138 +
139 +class SafeRepresenter(BaseRepresenter):
140 +
141 + def ignore_aliases(self, data):
142 + if data in [None, ()]:
143 + return True
144 + if isinstance(data, (str, unicode, bool, int, float)):
145 + return True
146 +
147 + def represent_none(self, data):
148 + return self.represent_scalar(u'tag:yaml.org,2002:null',
149 + u'null')
150 +
151 + def represent_str(self, data):
152 + tag = None
153 + style = None
154 + try:
155 + data = unicode(data, 'ascii')
156 + tag = u'tag:yaml.org,2002:str'
157 + except UnicodeDecodeError:
158 + try:
159 + data = unicode(data, 'utf-8')
160 + tag = u'tag:yaml.org,2002:str'
161 + except UnicodeDecodeError:
162 + data = data.encode('base64')
163 + tag = u'tag:yaml.org,2002:binary'
164 + style = '|'
165 + return self.represent_scalar(tag, data, style=style)
166 +
167 + def represent_unicode(self, data):
168 + return self.represent_scalar(u'tag:yaml.org,2002:str', data)
169 +
170 + def represent_bool(self, data):
171 + if data:
172 + value = u'true'
173 + else:
174 + value = u'false'
175 + return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
176 +
177 + def represent_int(self, data):
178 + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data))
179 +
180 + def represent_long(self, data):
181 + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data))
182 +
183 + inf_value = 1e300
184 + while repr(inf_value) != repr(inf_value*inf_value):
185 + inf_value *= inf_value
186 +
187 + def represent_float(self, data):
188 + if data != data or (data == 0.0 and data == 1.0):
189 + value = u'.nan'
190 + elif data == self.inf_value:
191 + value = u'.inf'
192 + elif data == -self.inf_value:
193 + value = u'-.inf'
194 + else:
195 + value = unicode(repr(data)).lower()
196 + # Note that in some cases `repr(data)` represents a float number
197 + # without the decimal parts. For instance:
198 + # >>> repr(1e17)
199 + # '1e17'
200 + # Unfortunately, this is not a valid float representation according
201 + # to the definition of the `!!float` tag. We fix this by adding
202 + # '.0' before the 'e' symbol.
203 + if u'.' not in value and u'e' in value:
204 + value = value.replace(u'e', u'.0e', 1)
205 + return self.represent_scalar(u'tag:yaml.org,2002:float', value)
206 +
207 + def represent_list(self, data):
208 + #pairs = (len(data) > 0 and isinstance(data, list))
209 + #if pairs:
210 + # for item in data:
211 + # if not isinstance(item, tuple) or len(item) != 2:
212 + # pairs = False
213 + # break
214 + #if not pairs:
215 + return self.represent_sequence(u'tag:yaml.org,2002:seq', data)
216 + #value = []
217 + #for item_key, item_value in data:
218 + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
219 + # [(item_key, item_value)]))
220 + #return SequenceNode(u'tag:yaml.org,2002:pairs', value)
221 +
222 + def represent_dict(self, data):
223 + return self.represent_mapping(u'tag:yaml.org,2002:map', data)
224 +
225 + def represent_set(self, data):
226 + value = {}
227 + for key in data:
228 + value[key] = None
229 + return self.represent_mapping(u'tag:yaml.org,2002:set', value)
230 +
231 + def represent_date(self, data):
232 + value = unicode(data.isoformat())
233 + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value)
234 +
235 + def represent_datetime(self, data):
236 + value = unicode(data.isoformat(' '))
237 + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value)
238 +
239 + def represent_yaml_object(self, tag, data, cls, flow_style=None):
240 + if hasattr(data, '__getstate__'):
241 + state = data.__getstate__()
242 + else:
243 + state = data.__dict__.copy()
244 + return self.represent_mapping(tag, state, flow_style=flow_style)
245 +
246 + def represent_undefined(self, data):
247 + raise RepresenterError("cannot represent an object: %s" % data)
248 +
249 +SafeRepresenter.add_representer(type(None),
250 + SafeRepresenter.represent_none)
251 +
252 +SafeRepresenter.add_representer(str,
253 + SafeRepresenter.represent_str)
254 +
255 +SafeRepresenter.add_representer(unicode,
256 + SafeRepresenter.represent_unicode)
257 +
258 +SafeRepresenter.add_representer(bool,
259 + SafeRepresenter.represent_bool)
260 +
261 +SafeRepresenter.add_representer(int,
262 + SafeRepresenter.represent_int)
263 +
264 +SafeRepresenter.add_representer(long,
265 + SafeRepresenter.represent_long)
266 +
267 +SafeRepresenter.add_representer(float,
268 + SafeRepresenter.represent_float)
269 +
270 +SafeRepresenter.add_representer(list,
271 + SafeRepresenter.represent_list)
272 +
273 +SafeRepresenter.add_representer(tuple,
274 + SafeRepresenter.represent_list)
275 +
276 +SafeRepresenter.add_representer(dict,
277 + SafeRepresenter.represent_dict)
278 +
279 +SafeRepresenter.add_representer(set,
280 + SafeRepresenter.represent_set)
281 +
282 +SafeRepresenter.add_representer(datetime.date,
283 + SafeRepresenter.represent_date)
284 +
285 +SafeRepresenter.add_representer(datetime.datetime,
286 + SafeRepresenter.represent_datetime)
287 +
288 +SafeRepresenter.add_representer(None,
289 + SafeRepresenter.represent_undefined)
290 +
291 +class Representer(SafeRepresenter):
292 +
293 + def represent_str(self, data):
294 + tag = None
295 + style = None
296 + try:
297 + data = unicode(data, 'ascii')
298 + tag = u'tag:yaml.org,2002:str'
299 + except UnicodeDecodeError:
300 + try:
301 + data = unicode(data, 'utf-8')
302 + tag = u'tag:yaml.org,2002:python/str'
303 + except UnicodeDecodeError:
304 + data = data.encode('base64')
305 + tag = u'tag:yaml.org,2002:binary'
306 + style = '|'
307 + return self.represent_scalar(tag, data, style=style)
308 +
309 + def represent_unicode(self, data):
310 + tag = None
311 + try:
312 + data.encode('ascii')
313 + tag = u'tag:yaml.org,2002:python/unicode'
314 + except UnicodeEncodeError:
315 + tag = u'tag:yaml.org,2002:str'
316 + return self.represent_scalar(tag, data)
317 +
318 + def represent_long(self, data):
319 + tag = u'tag:yaml.org,2002:int'
320 + if int(data) is not data:
321 + tag = u'tag:yaml.org,2002:python/long'
322 + return self.represent_scalar(tag, unicode(data))
323 +
324 + def represent_complex(self, data):
325 + if data.imag == 0.0:
326 + data = u'%r' % data.real
327 + elif data.real == 0.0:
328 + data = u'%rj' % data.imag
329 + elif data.imag > 0:
330 + data = u'%r+%rj' % (data.real, data.imag)
331 + else:
332 + data = u'%r%rj' % (data.real, data.imag)
333 + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
334 +
335 + def represent_tuple(self, data):
336 + return self.represent_sequence(u'tag:yaml.org,2002:python/tuple', data)
337 +
338 + def represent_name(self, data):
339 + name = u'%s.%s' % (data.__module__, data.__name__)
340 + return self.represent_scalar(u'tag:yaml.org,2002:python/name:'+name, u'')
341 +
342 + def represent_module(self, data):
343 + return self.represent_scalar(
344 + u'tag:yaml.org,2002:python/module:'+data.__name__, u'')
345 +
346 + def represent_instance(self, data):
347 + # For instances of classic classes, we use __getinitargs__ and
348 + # __getstate__ to serialize the data.
349 +
350 + # If data.__getinitargs__ exists, the object must be reconstructed by
351 + # calling cls(**args), where args is a tuple returned by
352 + # __getinitargs__. Otherwise, the cls.__init__ method should never be
353 + # called and the class instance is created by instantiating a trivial
354 + # class and assigning to the instance's __class__ variable.
355 +
356 + # If data.__getstate__ exists, it returns the state of the object.
357 + # Otherwise, the state of the object is data.__dict__.
358 +
359 + # We produce either a !!python/object or !!python/object/new node.
360 + # If data.__getinitargs__ does not exist and state is a dictionary, we
361 + # produce a !!python/object node . Otherwise we produce a
362 + # !!python/object/new node.
363 +
364 + cls = data.__class__
365 + class_name = u'%s.%s' % (cls.__module__, cls.__name__)
366 + args = None
367 + state = None
368 + if hasattr(data, '__getinitargs__'):
369 + args = list(data.__getinitargs__())
370 + if hasattr(data, '__getstate__'):
371 + state = data.__getstate__()
372 + else:
373 + state = data.__dict__
374 + if args is None and isinstance(state, dict):
375 + return self.represent_mapping(
376 + u'tag:yaml.org,2002:python/object:'+class_name, state)
377 + if isinstance(state, dict) and not state:
378 + return self.represent_sequence(
379 + u'tag:yaml.org,2002:python/object/new:'+class_name, args)
380 + value = {}
381 + if args:
382 + value['args'] = args
383 + value['state'] = state
384 + return self.represent_mapping(
385 + u'tag:yaml.org,2002:python/object/new:'+class_name, value)
386 +
387 + def represent_object(self, data):
388 + # We use __reduce__ API to save the data. data.__reduce__ returns
389 + # a tuple of length 2-5:
390 + # (function, args, state, listitems, dictitems)
391 +
392 + # For reconstructing, we calls function(*args), then set its state,
393 + # listitems, and dictitems if they are not None.
394 +
395 + # A special case is when function.__name__ == '__newobj__'. In this
396 + # case we create the object with args[0].__new__(*args).
397 +
398 + # Another special case is when __reduce__ returns a string - we don't
399 + # support it.
400 +
401 + # We produce a !!python/object, !!python/object/new or
402 + # !!python/object/apply node.
403 +
404 + cls = type(data)
405 + if cls in copy_reg.dispatch_table:
406 + reduce = copy_reg.dispatch_table[cls](data)
407 + elif hasattr(data, '__reduce_ex__'):
408 + reduce = data.__reduce_ex__(2)
409 + elif hasattr(data, '__reduce__'):
410 + reduce = data.__reduce__()
411 + else:
412 + raise RepresenterError("cannot represent object: %r" % data)
413 + reduce = (list(reduce)+[None]*5)[:5]
414 + function, args, state, listitems, dictitems = reduce
415 + args = list(args)
416 + if state is None:
417 + state = {}
418 + if listitems is not None:
419 + listitems = list(listitems)
420 + if dictitems is not None:
421 + dictitems = dict(dictitems)
422 + if function.__name__ == '__newobj__':
423 + function = args[0]
424 + args = args[1:]
425 + tag = u'tag:yaml.org,2002:python/object/new:'
426 + newobj = True
427 + else:
428 + tag = u'tag:yaml.org,2002:python/object/apply:'
429 + newobj = False
430 + function_name = u'%s.%s' % (function.__module__, function.__name__)
431 + if not args and not listitems and not dictitems \
432 + and isinstance(state, dict) and newobj:
433 + return self.represent_mapping(
434 + u'tag:yaml.org,2002:python/object:'+function_name, state)
435 + if not listitems and not dictitems \
436 + and isinstance(state, dict) and not state:
437 + return self.represent_sequence(tag+function_name, args)
438 + value = {}
439 + if args:
440 + value['args'] = args
441 + if state or not isinstance(state, dict):
442 + value['state'] = state
443 + if listitems:
444 + value['listitems'] = listitems
445 + if dictitems:
446 + value['dictitems'] = dictitems
447 + return self.represent_mapping(tag+function_name, value)
448 +
449 +Representer.add_representer(str,
450 + Representer.represent_str)
451 +
452 +Representer.add_representer(unicode,
453 + Representer.represent_unicode)
454 +
455 +Representer.add_representer(long,
456 + Representer.represent_long)
457 +
458 +Representer.add_representer(complex,
459 + Representer.represent_complex)
460 +
461 +Representer.add_representer(tuple,
462 + Representer.represent_tuple)
463 +
464 +Representer.add_representer(type,
465 + Representer.represent_name)
466 +
467 +Representer.add_representer(types.ClassType,
468 + Representer.represent_name)
469 +
470 +Representer.add_representer(types.FunctionType,
471 + Representer.represent_name)
472 +
473 +Representer.add_representer(types.BuiltinFunctionType,
474 + Representer.represent_name)
475 +
476 +Representer.add_representer(types.ModuleType,
477 + Representer.represent_module)
478 +
479 +Representer.add_multi_representer(types.InstanceType,
480 + Representer.represent_instance)
481 +
482 +Representer.add_multi_representer(object,
483 + Representer.represent_object)
484 +
python.d/python_modules/pyyaml2/resolver.py new
+224
@@ -0,0 +1,224 @@
1 +
2 +__all__ = ['BaseResolver', 'Resolver']
3 +
4 +from error import *
5 +from nodes import *
6 +
7 +import re
8 +
9 +class ResolverError(YAMLError):
10 + pass
11 +
12 +class BaseResolver(object):
13 +
14 + DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str'
15 + DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq'
16 + DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map'
17 +
18 + yaml_implicit_resolvers = {}
19 + yaml_path_resolvers = {}
20 +
21 + def __init__(self):
22 + self.resolver_exact_paths = []
23 + self.resolver_prefix_paths = []
24 +
25 + def add_implicit_resolver(cls, tag, regexp, first):
26 + if not 'yaml_implicit_resolvers' in cls.__dict__:
27 + cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
28 + if first is None:
29 + first = [None]
30 + for ch in first:
31 + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
32 + add_implicit_resolver = classmethod(add_implicit_resolver)
33 +
34 + def add_path_resolver(cls, tag, path, kind=None):
35 + # Note: `add_path_resolver` is experimental. The API could be changed.
36 + # `new_path` is a pattern that is matched against the path from the
37 + # root to the node that is being considered. `node_path` elements are
38 + # tuples `(node_check, index_check)`. `node_check` is a node class:
39 + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None`
40 + # matches any kind of a node. `index_check` could be `None`, a boolean
41 + # value, a string value, or a number. `None` and `False` match against
42 + # any _value_ of sequence and mapping nodes. `True` matches against
43 + # any _key_ of a mapping node. A string `index_check` matches against
44 + # a mapping value that corresponds to a scalar key which content is
45 + # equal to the `index_check` value. An integer `index_check` matches
46 + # against a sequence value with the index equal to `index_check`.
47 + if not 'yaml_path_resolvers' in cls.__dict__:
48 + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
49 + new_path = []
50 + for element in path:
51 + if isinstance(element, (list, tuple)):
52 + if len(element) == 2:
53 + node_check, index_check = element
54 + elif len(element) == 1:
55 + node_check = element[0]
56 + index_check = True
57 + else:
58 + raise ResolverError("Invalid path element: %s" % element)
59 + else:
60 + node_check = None
61 + index_check = element
62 + if node_check is str:
63 + node_check = ScalarNode
64 + elif node_check is list:
65 + node_check = SequenceNode
66 + elif node_check is dict:
67 + node_check = MappingNode
68 + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \
69 + and not isinstance(node_check, basestring) \
70 + and node_check is not None:
71 + raise ResolverError("Invalid node checker: %s" % node_check)
72 + if not isinstance(index_check, (basestring, int)) \
73 + and index_check is not None:
74 + raise ResolverError("Invalid index checker: %s" % index_check)
75 + new_path.append((node_check, index_check))
76 + if kind is str:
77 + kind = ScalarNode
78 + elif kind is list:
79 + kind = SequenceNode
80 + elif kind is dict:
81 + kind = MappingNode
82 + elif kind not in [ScalarNode, SequenceNode, MappingNode] \
83 + and kind is not None:
84 + raise ResolverError("Invalid node kind: %s" % kind)
85 + cls.yaml_path_resolvers[tuple(new_path), kind] = tag
86 + add_path_resolver = classmethod(add_path_resolver)
87 +
88 + def descend_resolver(self, current_node, current_index):
89 + if not self.yaml_path_resolvers:
90 + return
91 + exact_paths = {}
92 + prefix_paths = []
93 + if current_node:
94 + depth = len(self.resolver_prefix_paths)
95 + for path, kind in self.resolver_prefix_paths[-1]:
96 + if self.check_resolver_prefix(depth, path, kind,
97 + current_node, current_index):
98 + if len(path) > depth:
99 + prefix_paths.append((path, kind))
100 + else:
101 + exact_paths[kind] = self.yaml_path_resolvers[path, kind]
102 + else:
103 + for path, kind in self.yaml_path_resolvers:
104 + if not path:
105 + exact_paths[kind] = self.yaml_path_resolvers[path, kind]
106 + else:
107 + prefix_paths.append((path, kind))
108 + self.resolver_exact_paths.append(exact_paths)
109 + self.resolver_prefix_paths.append(prefix_paths)
110 +
111 + def ascend_resolver(self):
112 + if not self.yaml_path_resolvers:
113 + return
114 + self.resolver_exact_paths.pop()
115 + self.resolver_prefix_paths.pop()
116 +
117 + def check_resolver_prefix(self, depth, path, kind,
118 + current_node, current_index):
119 + node_check, index_check = path[depth-1]
120 + if isinstance(node_check, basestring):
121 + if current_node.tag != node_check:
122 + return
123 + elif node_check is not None:
124 + if not isinstance(current_node, node_check):
125 + return
126 + if index_check is True and current_index is not None:
127 + return
128 + if (index_check is False or index_check is None) \
129 + and current_index is None:
130 + return
131 + if isinstance(index_check, basestring):
132 + if not (isinstance(current_index, ScalarNode)
133 + and index_check == current_index.value):
134 + return
135 + elif isinstance(index_check, int) and not isinstance(index_check, bool):
136 + if index_check != current_index:
137 + return
138 + return True
139 +
140 + def resolve(self, kind, value, implicit):
141 + if kind is ScalarNode and implicit[0]:
142 + if value == u'':
143 + resolvers = self.yaml_implicit_resolvers.get(u'', [])
144 + else:
145 + resolvers = self.yaml_implicit_resolvers.get(value[0], [])
146 + resolvers += self.yaml_implicit_resolvers.get(None, [])
147 + for tag, regexp in resolvers:
148 + if regexp.match(value):
149 + return tag
150 + implicit = implicit[1]
151 + if self.yaml_path_resolvers:
152 + exact_paths = self.resolver_exact_paths[-1]
153 + if kind in exact_paths:
154 + return exact_paths[kind]
155 + if None in exact_paths:
156 + return exact_paths[None]
157 + if kind is ScalarNode:
158 + return self.DEFAULT_SCALAR_TAG
159 + elif kind is SequenceNode:
160 + return self.DEFAULT_SEQUENCE_TAG
161 + elif kind is MappingNode:
162 + return self.DEFAULT_MAPPING_TAG
163 +
164 +class Resolver(BaseResolver):
165 + pass
166 +
167 +Resolver.add_implicit_resolver(
168 + u'tag:yaml.org,2002:bool',
169 + re.compile(ur'''^(?:yes|Yes|YES|no|No|NO
170 + |true|True|TRUE|false|False|FALSE
171 + |on|On|ON|off|Off|OFF)$''', re.X),
172 + list(u'yYnNtTfFoO'))
173 +
174 +Resolver.add_implicit_resolver(
175 + u'tag:yaml.org,2002:float',
176 + re.compile(ur'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?
177 + |\.[0-9_]+(?:[eE][-+][0-9]+)?
178 + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
179 + |[-+]?\.(?:inf|Inf|INF)
180 + |\.(?:nan|NaN|NAN))$''', re.X),
181 + list(u'-+0123456789.'))
182 +
183 +Resolver.add_implicit_resolver(
184 + u'tag:yaml.org,2002:int',
185 + re.compile(ur'''^(?:[-+]?0b[0-1_]+
186 + |[-+]?0[0-7_]+
187 + |[-+]?(?:0|[1-9][0-9_]*)
188 + |[-+]?0x[0-9a-fA-F_]+
189 + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
190 + list(u'-+0123456789'))
191 +
192 +Resolver.add_implicit_resolver(
193 + u'tag:yaml.org,2002:merge',
194 + re.compile(ur'^(?:<<)$'),
195 + [u'<'])
196 +
197 +Resolver.add_implicit_resolver(
198 + u'tag:yaml.org,2002:null',
199 + re.compile(ur'''^(?: ~
200 + |null|Null|NULL
201 + | )$''', re.X),
202 + [u'~', u'n', u'N', u''])
203 +
204 +Resolver.add_implicit_resolver(
205 + u'tag:yaml.org,2002:timestamp',
206 + re.compile(ur'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
207 + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
208 + (?:[Tt]|[ \t]+)[0-9][0-9]?
209 + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)?
210 + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
211 + list(u'0123456789'))
212 +
213 +Resolver.add_implicit_resolver(
214 + u'tag:yaml.org,2002:value',
215 + re.compile(ur'^(?:=)$'),
216 + [u'='])
217 +
218 +# The following resolver is only for documentation purposes. It cannot work
219 +# because plain scalars cannot start with '!', '&', or '*'.
220 +Resolver.add_implicit_resolver(
221 + u'tag:yaml.org,2002:yaml',
222 + re.compile(ur'^(?:!|&|\*)$'),
223 + list(u'!&*'))
224 +
python.d/python_modules/pyyaml2/scanner.py new
+1457
@@ -0,0 +1,1457 @@
1 +
2 +# Scanner produces tokens of the following types:
3 +# STREAM-START
4 +# STREAM-END
5 +# DIRECTIVE(name, value)
6 +# DOCUMENT-START
7 +# DOCUMENT-END
8 +# BLOCK-SEQUENCE-START
9 +# BLOCK-MAPPING-START
10 +# BLOCK-END
11 +# FLOW-SEQUENCE-START
12 +# FLOW-MAPPING-START
13 +# FLOW-SEQUENCE-END
14 +# FLOW-MAPPING-END
15 +# BLOCK-ENTRY
16 +# FLOW-ENTRY
17 +# KEY
18 +# VALUE
19 +# ALIAS(value)
20 +# ANCHOR(value)
21 +# TAG(value)
22 +# SCALAR(value, plain, style)
23 +#
24 +# Read comments in the Scanner code for more details.
25 +#
26 +
27 +__all__ = ['Scanner', 'ScannerError']
28 +
29 +from error import MarkedYAMLError
30 +from tokens import *
31 +
32 +class ScannerError(MarkedYAMLError):
33 + pass
34 +
35 +class SimpleKey(object):
36 + # See below simple keys treatment.
37 +
38 + def __init__(self, token_number, required, index, line, column, mark):
39 + self.token_number = token_number
40 + self.required = required
41 + self.index = index
42 + self.line = line
43 + self.column = column
44 + self.mark = mark
45 +
46 +class Scanner(object):
47 +
48 + def __init__(self):
49 + """Initialize the scanner."""
50 + # It is assumed that Scanner and Reader will have a common descendant.
51 + # Reader do the dirty work of checking for BOM and converting the
52 + # input data to Unicode. It also adds NUL to the end.
53 + #
54 + # Reader supports the following methods
55 + # self.peek(i=0) # peek the next i-th character
56 + # self.prefix(l=1) # peek the next l characters
57 + # self.forward(l=1) # read the next l characters and move the pointer.
58 +
59 + # Had we reached the end of the stream?
60 + self.done = False
61 +
62 + # The number of unclosed '{' and '['. `flow_level == 0` means block
63 + # context.
64 + self.flow_level = 0
65 +
66 + # List of processed tokens that are not yet emitted.
67 + self.tokens = []
68 +
69 + # Add the STREAM-START token.
70 + self.fetch_stream_start()
71 +
72 + # Number of tokens that were emitted through the `get_token` method.
73 + self.tokens_taken = 0
74 +
75 + # The current indentation level.
76 + self.indent = -1
77 +
78 + # Past indentation levels.
79 + self.indents = []
80 +
81 + # Variables related to simple keys treatment.
82 +
83 + # A simple key is a key that is not denoted by the '?' indicator.
84 + # Example of simple keys:
85 + # ---
86 + # block simple key: value
87 + # ? not a simple key:
88 + # : { flow simple key: value }
89 + # We emit the KEY token before all keys, so when we find a potential
90 + # simple key, we try to locate the corresponding ':' indicator.
91 + # Simple keys should be limited to a single line and 1024 characters.
92 +
93 + # Can a simple key start at the current position? A simple key may
94 + # start:
95 + # - at the beginning of the line, not counting indentation spaces
96 + # (in block context),
97 + # - after '{', '[', ',' (in the flow context),
98 + # - after '?', ':', '-' (in the block context).
99 + # In the block context, this flag also signifies if a block collection
100 + # may start at the current position.
101 + self.allow_simple_key = True
102 +
103 + # Keep track of possible simple keys. This is a dictionary. The key
104 + # is `flow_level`; there can be no more that one possible simple key
105 + # for each level. The value is a SimpleKey record:
106 + # (token_number, required, index, line, column, mark)
107 + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
108 + # '[', or '{' tokens.
109 + self.possible_simple_keys = {}
110 +
111 + # Public methods.
112 +
113 + def check_token(self, *choices):
114 + # Check if the next token is one of the given types.
115 + while self.need_more_tokens():
116 + self.fetch_more_tokens()
117 + if self.tokens:
118 + if not choices:
119 + return True
120 + for choice in choices:
121 + if isinstance(self.tokens[0], choice):
122 + return True
123 + return False
124 +
125 + def peek_token(self):
126 + # Return the next token, but do not delete if from the queue.
127 + while self.need_more_tokens():
128 + self.fetch_more_tokens()
129 + if self.tokens:
130 + return self.tokens[0]
131 +
132 + def get_token(self):
133 + # Return the next token.
134 + while self.need_more_tokens():
135 + self.fetch_more_tokens()
136 + if self.tokens:
137 + self.tokens_taken += 1
138 + return self.tokens.pop(0)
139 +
140 + # Private methods.
141 +
142 + def need_more_tokens(self):
143 + if self.done:
144 + return False
145 + if not self.tokens:
146 + return True
147 + # The current token may be a potential simple key, so we
148 + # need to look further.
149 + self.stale_possible_simple_keys()
150 + if self.next_possible_simple_key() == self.tokens_taken:
151 + return True
152 +
153 + def fetch_more_tokens(self):
154 +
155 + # Eat whitespaces and comments until we reach the next token.
156 + self.scan_to_next_token()
157 +
158 + # Remove obsolete possible simple keys.
159 + self.stale_possible_simple_keys()
160 +
161 + # Compare the current indentation and column. It may add some tokens
162 + # and decrease the current indentation level.
163 + self.unwind_indent(self.column)
164 +
165 + # Peek the next character.
166 + ch = self.peek()
167 +
168 + # Is it the end of stream?
169 + if ch == u'\0':
170 + return self.fetch_stream_end()
171 +
172 + # Is it a directive?
173 + if ch == u'%' and self.check_directive():
174 + return self.fetch_directive()
175 +
176 + # Is it the document start?
177 + if ch == u'-' and self.check_document_start():
178 + return self.fetch_document_start()
179 +
180 + # Is it the document end?
181 + if ch == u'.' and self.check_document_end():
182 + return self.fetch_document_end()
183 +
184 + # TODO: support for BOM within a stream.
185 + #if ch == u'\uFEFF':
186 + # return self.fetch_bom() <-- issue BOMToken
187 +
188 + # Note: the order of the following checks is NOT significant.
189 +
190 + # Is it the flow sequence start indicator?
191 + if ch == u'[':
192 + return self.fetch_flow_sequence_start()
193 +
194 + # Is it the flow mapping start indicator?
195 + if ch == u'{':
196 + return self.fetch_flow_mapping_start()
197 +
198 + # Is it the flow sequence end indicator?
199 + if ch == u']':
200 + return self.fetch_flow_sequence_end()
201 +
202 + # Is it the flow mapping end indicator?
203 + if ch == u'}':
204 + return self.fetch_flow_mapping_end()
205 +
206 + # Is it the flow entry indicator?
207 + if ch == u',':
208 + return self.fetch_flow_entry()
209 +
210 + # Is it the block entry indicator?
211 + if ch == u'-' and self.check_block_entry():
212 + return self.fetch_block_entry()
213 +
214 + # Is it the key indicator?
215 + if ch == u'?' and self.check_key():
216 + return self.fetch_key()
217 +
218 + # Is it the value indicator?
219 + if ch == u':' and self.check_value():
220 + return self.fetch_value()
221 +
222 + # Is it an alias?
223 + if ch == u'*':
224 + return self.fetch_alias()
225 +
226 + # Is it an anchor?
227 + if ch == u'&':
228 + return self.fetch_anchor()
229 +
230 + # Is it a tag?
231 + if ch == u'!':
232 + return self.fetch_tag()
233 +
234 + # Is it a literal scalar?
235 + if ch == u'|' and not self.flow_level:
236 + return self.fetch_literal()
237 +
238 + # Is it a folded scalar?
239 + if ch == u'>' and not self.flow_level:
240 + return self.fetch_folded()
241 +
242 + # Is it a single quoted scalar?
243 + if ch == u'\'':
244 + return self.fetch_single()
245 +
246 + # Is it a double quoted scalar?
247 + if ch == u'\"':
248 + return self.fetch_double()
249 +
250 + # It must be a plain scalar then.
251 + if self.check_plain():
252 + return self.fetch_plain()
253 +
254 + # No? It's an error. Let's produce a nice error message.
255 + raise ScannerError("while scanning for the next token", None,
256 + "found character %r that cannot start any token"
257 + % ch.encode('utf-8'), self.get_mark())
258 +
259 + # Simple keys treatment.
260 +
261 + def next_possible_simple_key(self):
262 + # Return the number of the nearest possible simple key. Actually we
263 + # don't need to loop through the whole dictionary. We may replace it
264 + # with the following code:
265 + # if not self.possible_simple_keys:
266 + # return None
267 + # return self.possible_simple_keys[
268 + # min(self.possible_simple_keys.keys())].token_number
269 + min_token_number = None
270 + for level in self.possible_simple_keys:
271 + key = self.possible_simple_keys[level]
272 + if min_token_number is None or key.token_number < min_token_number:
273 + min_token_number = key.token_number
274 + return min_token_number
275 +
276 + def stale_possible_simple_keys(self):
277 + # Remove entries that are no longer possible simple keys. According to
278 + # the YAML specification, simple keys
279 + # - should be limited to a single line,
280 + # - should be no longer than 1024 characters.
281 + # Disabling this procedure will allow simple keys of any length and
282 + # height (may cause problems if indentation is broken though).
283 + for level in self.possible_simple_keys.keys():
284 + key = self.possible_simple_keys[level]
285 + if key.line != self.line \
286 + or self.index-key.index > 1024:
287 + if key.required:
288 + raise ScannerError("while scanning a simple key", key.mark,
289 + "could not found expected ':'", self.get_mark())
290 + del self.possible_simple_keys[level]
291 +
292 + def save_possible_simple_key(self):
293 + # The next token may start a simple key. We check if it's possible
294 + # and save its position. This function is called for
295 + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
296 +
297 + # Check if a simple key is required at the current position.
298 + required = not self.flow_level and self.indent == self.column
299 +
300 + # A simple key is required only if it is the first token in the current
301 + # line. Therefore it is always allowed.
302 + assert self.allow_simple_key or not required
303 +
304 + # The next token might be a simple key. Let's save it's number and
305 + # position.
306 + if self.allow_simple_key:
307 + self.remove_possible_simple_key()
308 + token_number = self.tokens_taken+len(self.tokens)
309 + key = SimpleKey(token_number, required,
310 + self.index, self.line, self.column, self.get_mark())
311 + self.possible_simple_keys[self.flow_level] = key
312 +
313 + def remove_possible_simple_key(self):
314 + # Remove the saved possible key position at the current flow level.
315 + if self.flow_level in self.possible_simple_keys:
316 + key = self.possible_simple_keys[self.flow_level]
317 +
318 + if key.required:
319 + raise ScannerError("while scanning a simple key", key.mark,
320 + "could not found expected ':'", self.get_mark())
321 +
322 + del self.possible_simple_keys[self.flow_level]
323 +
324 + # Indentation functions.
325 +
326 + def unwind_indent(self, column):
327 +
328 + ## In flow context, tokens should respect indentation.
329 + ## Actually the condition should be `self.indent >= column` according to
330 + ## the spec. But this condition will prohibit intuitively correct
331 + ## constructions such as
332 + ## key : {
333 + ## }
334 + #if self.flow_level and self.indent > column:
335 + # raise ScannerError(None, None,
336 + # "invalid intendation or unclosed '[' or '{'",
337 + # self.get_mark())
338 +
339 + # In the flow context, indentation is ignored. We make the scanner less
340 + # restrictive then specification requires.
341 + if self.flow_level:
342 + return
343 +
344 + # In block context, we may need to issue the BLOCK-END tokens.
345 + while self.indent > column:
346 + mark = self.get_mark()
347 + self.indent = self.indents.pop()
348 + self.tokens.append(BlockEndToken(mark, mark))
349 +
350 + def add_indent(self, column):
351 + # Check if we need to increase indentation.
352 + if self.indent < column:
353 + self.indents.append(self.indent)
354 + self.indent = column
355 + return True
356 + return False
357 +
358 + # Fetchers.
359 +
360 + def fetch_stream_start(self):
361 + # We always add STREAM-START as the first token and STREAM-END as the
362 + # last token.
363 +
364 + # Read the token.
365 + mark = self.get_mark()
366 +
367 + # Add STREAM-START.
368 + self.tokens.append(StreamStartToken(mark, mark,
369 + encoding=self.encoding))
370 +
371 +
372 + def fetch_stream_end(self):
373 +
374 + # Set the current intendation to -1.
375 + self.unwind_indent(-1)
376 +
377 + # Reset simple keys.
378 + self.remove_possible_simple_key()
379 + self.allow_simple_key = False
380 + self.possible_simple_keys = {}
381 +
382 + # Read the token.
383 + mark = self.get_mark()
384 +
385 + # Add STREAM-END.
386 + self.tokens.append(StreamEndToken(mark, mark))
387 +
388 + # The steam is finished.
389 + self.done = True
390 +
391 + def fetch_directive(self):
392 +
393 + # Set the current intendation to -1.
394 + self.unwind_indent(-1)
395 +
396 + # Reset simple keys.
397 + self.remove_possible_simple_key()
398 + self.allow_simple_key = False
399 +
400 + # Scan and add DIRECTIVE.
401 + self.tokens.append(self.scan_directive())
402 +
403 + def fetch_document_start(self):
404 + self.fetch_document_indicator(DocumentStartToken)
405 +
406 + def fetch_document_end(self):
407 + self.fetch_document_indicator(DocumentEndToken)
408 +
409 + def fetch_document_indicator(self, TokenClass):
410 +
411 + # Set the current intendation to -1.
412 + self.unwind_indent(-1)
413 +
414 + # Reset simple keys. Note that there could not be a block collection
415 + # after '---'.
416 + self.remove_possible_simple_key()
417 + self.allow_simple_key = False
418 +
419 + # Add DOCUMENT-START or DOCUMENT-END.
420 + start_mark = self.get_mark()
421 + self.forward(3)
422 + end_mark = self.get_mark()
423 + self.tokens.append(TokenClass(start_mark, end_mark))
424 +
425 + def fetch_flow_sequence_start(self):
426 + self.fetch_flow_collection_start(FlowSequenceStartToken)
427 +
428 + def fetch_flow_mapping_start(self):
429 + self.fetch_flow_collection_start(FlowMappingStartToken)
430 +
431 + def fetch_flow_collection_start(self, TokenClass):
432 +
433 + # '[' and '{' may start a simple key.
434 + self.save_possible_simple_key()
435 +
436 + # Increase the flow level.
437 + self.flow_level += 1
438 +
439 + # Simple keys are allowed after '[' and '{'.
440 + self.allow_simple_key = True
441 +
442 + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
443 + start_mark = self.get_mark()
444 + self.forward()
445 + end_mark = self.get_mark()
446 + self.tokens.append(TokenClass(start_mark, end_mark))
447 +
448 + def fetch_flow_sequence_end(self):
449 + self.fetch_flow_collection_end(FlowSequenceEndToken)
450 +
451 + def fetch_flow_mapping_end(self):
452 + self.fetch_flow_collection_end(FlowMappingEndToken)
453 +
454 + def fetch_flow_collection_end(self, TokenClass):
455 +
456 + # Reset possible simple key on the current level.
457 + self.remove_possible_simple_key()
458 +
459 + # Decrease the flow level.
460 + self.flow_level -= 1
461 +
462 + # No simple keys after ']' or '}'.
463 + self.allow_simple_key = False
464 +
465 + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
466 + start_mark = self.get_mark()
467 + self.forward()
468 + end_mark = self.get_mark()
469 + self.tokens.append(TokenClass(start_mark, end_mark))
470 +
471 + def fetch_flow_entry(self):
472 +
473 + # Simple keys are allowed after ','.
474 + self.allow_simple_key = True
475 +
476 + # Reset possible simple key on the current level.
477 + self.remove_possible_simple_key()
478 +
479 + # Add FLOW-ENTRY.
480 + start_mark = self.get_mark()
481 + self.forward()
482 + end_mark = self.get_mark()
483 + self.tokens.append(FlowEntryToken(start_mark, end_mark))
484 +
485 + def fetch_block_entry(self):
486 +
487 + # Block context needs additional checks.
488 + if not self.flow_level:
489 +
490 + # Are we allowed to start a new entry?
491 + if not self.allow_simple_key:
492 + raise ScannerError(None, None,
493 + "sequence entries are not allowed here",
494 + self.get_mark())
495 +
496 + # We may need to add BLOCK-SEQUENCE-START.
497 + if self.add_indent(self.column):
498 + mark = self.get_mark()
499 + self.tokens.append(BlockSequenceStartToken(mark, mark))
500 +
501 + # It's an error for the block entry to occur in the flow context,
502 + # but we let the parser detect this.
503 + else:
504 + pass
505 +
506 + # Simple keys are allowed after '-'.
507 + self.allow_simple_key = True
508 +
509 + # Reset possible simple key on the current level.
510 + self.remove_possible_simple_key()
511 +
512 + # Add BLOCK-ENTRY.
513 + start_mark = self.get_mark()
514 + self.forward()
515 + end_mark = self.get_mark()
516 + self.tokens.append(BlockEntryToken(start_mark, end_mark))
517 +
518 + def fetch_key(self):
519 +
520 + # Block context needs additional checks.
521 + if not self.flow_level:
522 +
523 + # Are we allowed to start a key (not nessesary a simple)?
524 + if not self.allow_simple_key:
525 + raise ScannerError(None, None,
526 + "mapping keys are not allowed here",
527 + self.get_mark())
528 +
529 + # We may need to add BLOCK-MAPPING-START.
530 + if self.add_indent(self.column):
531 + mark = self.get_mark()
532 + self.tokens.append(BlockMappingStartToken(mark, mark))
533 +
534 + # Simple keys are allowed after '?' in the block context.
535 + self.allow_simple_key = not self.flow_level
536 +
537 + # Reset possible simple key on the current level.
538 + self.remove_possible_simple_key()
539 +
540 + # Add KEY.
541 + start_mark = self.get_mark()
542 + self.forward()
543 + end_mark = self.get_mark()
544 + self.tokens.append(KeyToken(start_mark, end_mark))
545 +
546 + def fetch_value(self):
547 +
548 + # Do we determine a simple key?
549 + if self.flow_level in self.possible_simple_keys:
550 +
551 + # Add KEY.
552 + key = self.possible_simple_keys[self.flow_level]
553 + del self.possible_simple_keys[self.flow_level]
554 + self.tokens.insert(key.token_number-self.tokens_taken,
555 + KeyToken(key.mark, key.mark))
556 +
557 + # If this key starts a new block mapping, we need to add
558 + # BLOCK-MAPPING-START.
559 + if not self.flow_level:
560 + if self.add_indent(key.column):
561 + self.tokens.insert(key.token_number-self.tokens_taken,
562 + BlockMappingStartToken(key.mark, key.mark))
563 +
564 + # There cannot be two simple keys one after another.
565 + self.allow_simple_key = False
566 +
567 + # It must be a part of a complex key.
568 + else:
569 +
570 + # Block context needs additional checks.
571 + # (Do we really need them? They will be catched by the parser
572 + # anyway.)
573 + if not self.flow_level:
574 +
575 + # We are allowed to start a complex value if and only if
576 + # we can start a simple key.
577 + if not self.allow_simple_key:
578 + raise ScannerError(None, None,
579 + "mapping values are not allowed here",
580 + self.get_mark())
581 +
582 + # If this value starts a new block mapping, we need to add
583 + # BLOCK-MAPPING-START. It will be detected as an error later by
584 + # the parser.
585 + if not self.flow_level:
586 + if self.add_indent(self.column):
587 + mark = self.get_mark()
588 + self.tokens.append(BlockMappingStartToken(mark, mark))
589 +
590 + # Simple keys are allowed after ':' in the block context.
591 + self.allow_simple_key = not self.flow_level
592 +
593 + # Reset possible simple key on the current level.
594 + self.remove_possible_simple_key()
595 +
596 + # Add VALUE.
597 + start_mark = self.get_mark()
598 + self.forward()
599 + end_mark = self.get_mark()
600 + self.tokens.append(ValueToken(start_mark, end_mark))
601 +
602 + def fetch_alias(self):
603 +
604 + # ALIAS could be a simple key.
605 + self.save_possible_simple_key()
606 +
607 + # No simple keys after ALIAS.
608 + self.allow_simple_key = False
609 +
610 + # Scan and add ALIAS.
611 + self.tokens.append(self.scan_anchor(AliasToken))
612 +
613 + def fetch_anchor(self):
614 +
615 + # ANCHOR could start a simple key.
616 + self.save_possible_simple_key()
617 +
618 + # No simple keys after ANCHOR.
619 + self.allow_simple_key = False
620 +
621 + # Scan and add ANCHOR.
622 + self.tokens.append(self.scan_anchor(AnchorToken))
623 +
624 + def fetch_tag(self):
625 +
626 + # TAG could start a simple key.
627 + self.save_possible_simple_key()
628 +
629 + # No simple keys after TAG.
630 + self.allow_simple_key = False
631 +
632 + # Scan and add TAG.
633 + self.tokens.append(self.scan_tag())
634 +
635 + def fetch_literal(self):
636 + self.fetch_block_scalar(style='|')
637 +
638 + def fetch_folded(self):
639 + self.fetch_block_scalar(style='>')
640 +
641 + def fetch_block_scalar(self, style):
642 +
643 + # A simple key may follow a block scalar.
644 + self.allow_simple_key = True
645 +
646 + # Reset possible simple key on the current level.
647 + self.remove_possible_simple_key()
648 +
649 + # Scan and add SCALAR.
650 + self.tokens.append(self.scan_block_scalar(style))
651 +
652 + def fetch_single(self):
653 + self.fetch_flow_scalar(style='\'')
654 +
655 + def fetch_double(self):
656 + self.fetch_flow_scalar(style='"')
657 +
658 + def fetch_flow_scalar(self, style):
659 +
660 + # A flow scalar could be a simple key.
661 + self.save_possible_simple_key()
662 +
663 + # No simple keys after flow scalars.
664 + self.allow_simple_key = False
665 +
666 + # Scan and add SCALAR.
667 + self.tokens.append(self.scan_flow_scalar(style))
668 +
669 + def fetch_plain(self):
670 +
671 + # A plain scalar could be a simple key.
672 + self.save_possible_simple_key()
673 +
674 + # No simple keys after plain scalars. But note that `scan_plain` will
675 + # change this flag if the scan is finished at the beginning of the
676 + # line.
677 + self.allow_simple_key = False
678 +
679 + # Scan and add SCALAR. May change `allow_simple_key`.
680 + self.tokens.append(self.scan_plain())
681 +
682 + # Checkers.
683 +
684 + def check_directive(self):
685 +
686 + # DIRECTIVE: ^ '%' ...
687 + # The '%' indicator is already checked.
688 + if self.column == 0:
689 + return True
690 +
691 + def check_document_start(self):
692 +
693 + # DOCUMENT-START: ^ '---' (' '|'\n')
694 + if self.column == 0:
695 + if self.prefix(3) == u'---' \
696 + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
697 + return True
698 +
699 + def check_document_end(self):
700 +
701 + # DOCUMENT-END: ^ '...' (' '|'\n')
702 + if self.column == 0:
703 + if self.prefix(3) == u'...' \
704 + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
705 + return True
706 +
707 + def check_block_entry(self):
708 +
709 + # BLOCK-ENTRY: '-' (' '|'\n')
710 + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
711 +
712 + def check_key(self):
713 +
714 + # KEY(flow context): '?'
715 + if self.flow_level:
716 + return True
717 +
718 + # KEY(block context): '?' (' '|'\n')
719 + else:
720 + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
721 +
722 + def check_value(self):
723 +
724 + # VALUE(flow context): ':'
725 + if self.flow_level:
726 + return True
727 +
728 + # VALUE(block context): ':' (' '|'\n')
729 + else:
730 + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
731 +
732 + def check_plain(self):
733 +
734 + # A plain scalar may start with any non-space character except:
735 + # '-', '?', ':', ',', '[', ']', '{', '}',
736 + # '#', '&', '*', '!', '|', '>', '\'', '\"',
737 + # '%', '@', '`'.
738 + #
739 + # It may also start with
740 + # '-', '?', ':'
741 + # if it is followed by a non-space character.
742 + #
743 + # Note that we limit the last rule to the block context (except the
744 + # '-' character) because we want the flow context to be space
745 + # independent.
746 + ch = self.peek()
747 + return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \
748 + or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
749 + and (ch == u'-' or (not self.flow_level and ch in u'?:')))
750 +
751 + # Scanners.
752 +
753 + def scan_to_next_token(self):
754 + # We ignore spaces, line breaks and comments.
755 + # If we find a line break in the block context, we set the flag
756 + # `allow_simple_key` on.
757 + # The byte order mark is stripped if it's the first character in the
758 + # stream. We do not yet support BOM inside the stream as the
759 + # specification requires. Any such mark will be considered as a part
760 + # of the document.
761 + #
762 + # TODO: We need to make tab handling rules more sane. A good rule is
763 + # Tabs cannot precede tokens
764 + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
765 + # KEY(block), VALUE(block), BLOCK-ENTRY
766 + # So the checking code is
767 + # if <TAB>:
768 + # self.allow_simple_keys = False
769 + # We also need to add the check for `allow_simple_keys == True` to
770 + # `unwind_indent` before issuing BLOCK-END.
771 + # Scanners for block, flow, and plain scalars need to be modified.
772 +
773 + if self.index == 0 and self.peek() == u'\uFEFF':
774 + self.forward()
775 + found = False
776 + while not found:
777 + while self.peek() == u' ':
778 + self.forward()
779 + if self.peek() == u'#':
780 + while self.peek() not in u'\0\r\n\x85\u2028\u2029':
781 + self.forward()
782 + if self.scan_line_break():
783 + if not self.flow_level:
784 + self.allow_simple_key = True
785 + else:
786 + found = True
787 +
788 + def scan_directive(self):
789 + # See the specification for details.
790 + start_mark = self.get_mark()
791 + self.forward()
792 + name = self.scan_directive_name(start_mark)
793 + value = None
794 + if name == u'YAML':
795 + value = self.scan_yaml_directive_value(start_mark)
796 + end_mark = self.get_mark()
797 + elif name == u'TAG':
798 + value = self.scan_tag_directive_value(start_mark)
799 + end_mark = self.get_mark()
800 + else:
801 + end_mark = self.get_mark()
802 + while self.peek() not in u'\0\r\n\x85\u2028\u2029':
803 + self.forward()
804 + self.scan_directive_ignored_line(start_mark)
805 + return DirectiveToken(name, value, start_mark, end_mark)
806 +
807 + def scan_directive_name(self, start_mark):
808 + # See the specification for details.
809 + length = 0
810 + ch = self.peek(length)
811 + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
812 + or ch in u'-_':
813 + length += 1
814 + ch = self.peek(length)
815 + if not length:
816 + raise ScannerError("while scanning a directive", start_mark,
817 + "expected alphabetic or numeric character, but found %r"
818 + % ch.encode('utf-8'), self.get_mark())
819 + value = self.prefix(length)
820 + self.forward(length)
821 + ch = self.peek()
822 + if ch not in u'\0 \r\n\x85\u2028\u2029':
823 + raise ScannerError("while scanning a directive", start_mark,
824 + "expected alphabetic or numeric character, but found %r"
825 + % ch.encode('utf-8'), self.get_mark())
826 + return value
827 +
828 + def scan_yaml_directive_value(self, start_mark):
829 + # See the specification for details.
830 + while self.peek() == u' ':
831 + self.forward()
832 + major = self.scan_yaml_directive_number(start_mark)
833 + if self.peek() != '.':
834 + raise ScannerError("while scanning a directive", start_mark,
835 + "expected a digit or '.', but found %r"
836 + % self.peek().encode('utf-8'),
837 + self.get_mark())
838 + self.forward()
839 + minor = self.scan_yaml_directive_number(start_mark)
840 + if self.peek() not in u'\0 \r\n\x85\u2028\u2029':
841 + raise ScannerError("while scanning a directive", start_mark,
842 + "expected a digit or ' ', but found %r"
843 + % self.peek().encode('utf-8'),
844 + self.get_mark())
845 + return (major, minor)
846 +
847 + def scan_yaml_directive_number(self, start_mark):
848 + # See the specification for details.
849 + ch = self.peek()
850 + if not (u'0' <= ch <= u'9'):
851 + raise ScannerError("while scanning a directive", start_mark,
852 + "expected a digit, but found %r" % ch.encode('utf-8'),
853 + self.get_mark())
854 + length = 0
855 + while u'0' <= self.peek(length) <= u'9':
856 + length += 1
857 + value = int(self.prefix(length))
858 + self.forward(length)
859 + return value
860 +
861 + def scan_tag_directive_value(self, start_mark):
862 + # See the specification for details.
863 + while self.peek() == u' ':
864 + self.forward()
865 + handle = self.scan_tag_directive_handle(start_mark)
866 + while self.peek() == u' ':
867 + self.forward()
868 + prefix = self.scan_tag_directive_prefix(start_mark)
869 + return (handle, prefix)
870 +
871 + def scan_tag_directive_handle(self, start_mark):
872 + # See the specification for details.
873 + value = self.scan_tag_handle('directive', start_mark)
874 + ch = self.peek()
875 + if ch != u' ':
876 + raise ScannerError("while scanning a directive", start_mark,
877 + "expected ' ', but found %r" % ch.encode('utf-8'),
878 + self.get_mark())
879 + return value
880 +
881 + def scan_tag_directive_prefix(self, start_mark):
882 + # See the specification for details.
883 + value = self.scan_tag_uri('directive', start_mark)
884 + ch = self.peek()
885 + if ch not in u'\0 \r\n\x85\u2028\u2029':
886 + raise ScannerError("while scanning a directive", start_mark,
887 + "expected ' ', but found %r" % ch.encode('utf-8'),
888 + self.get_mark())
889 + return value
890 +
891 + def scan_directive_ignored_line(self, start_mark):
892 + # See the specification for details.
893 + while self.peek() == u' ':
894 + self.forward()
895 + if self.peek() == u'#':
896 + while self.peek() not in u'\0\r\n\x85\u2028\u2029':
897 + self.forward()
898 + ch = self.peek()
899 + if ch not in u'\0\r\n\x85\u2028\u2029':
900 + raise ScannerError("while scanning a directive", start_mark,
901 + "expected a comment or a line break, but found %r"
902 + % ch.encode('utf-8'), self.get_mark())
903 + self.scan_line_break()
904 +
905 + def scan_anchor(self, TokenClass):
906 + # The specification does not restrict characters for anchors and
907 + # aliases. This may lead to problems, for instance, the document:
908 + # [ *alias, value ]
909 + # can be interpteted in two ways, as
910 + # [ "value" ]
911 + # and
912 + # [ *alias , "value" ]
913 + # Therefore we restrict aliases to numbers and ASCII letters.
914 + start_mark = self.get_mark()
915 + indicator = self.peek()
916 + if indicator == u'*':
917 + name = 'alias'
918 + else:
919 + name = 'anchor'
920 + self.forward()
921 + length = 0
922 + ch = self.peek(length)
923 + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
924 + or ch in u'-_':
925 + length += 1
926 + ch = self.peek(length)
927 + if not length:
928 + raise ScannerError("while scanning an %s" % name, start_mark,
929 + "expected alphabetic or numeric character, but found %r"
930 + % ch.encode('utf-8'), self.get_mark())
931 + value = self.prefix(length)
932 + self.forward(length)
933 + ch = self.peek()
934 + if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
935 + raise ScannerError("while scanning an %s" % name, start_mark,
936 + "expected alphabetic or numeric character, but found %r"
937 + % ch.encode('utf-8'), self.get_mark())
938 + end_mark = self.get_mark()
939 + return TokenClass(value, start_mark, end_mark)
940 +
941 + def scan_tag(self):
942 + # See the specification for details.
943 + start_mark = self.get_mark()
944 + ch = self.peek(1)
945 + if ch == u'<':
946 + handle = None
947 + self.forward(2)
948 + suffix = self.scan_tag_uri('tag', start_mark)
949 + if self.peek() != u'>':
950 + raise ScannerError("while parsing a tag", start_mark,
951 + "expected '>', but found %r" % self.peek().encode('utf-8'),
952 + self.get_mark())
953 + self.forward()
954 + elif ch in u'\0 \t\r\n\x85\u2028\u2029':
955 + handle = None
956 + suffix = u'!'
957 + self.forward()
958 + else:
959 + length = 1
960 + use_handle = False
961 + while ch not in u'\0 \r\n\x85\u2028\u2029':
962 + if ch == u'!':
963 + use_handle = True
964 + break
965 + length += 1
966 + ch = self.peek(length)
967 + handle = u'!'
968 + if use_handle:
969 + handle = self.scan_tag_handle('tag', start_mark)
970 + else:
971 + handle = u'!'
972 + self.forward()
973 + suffix = self.scan_tag_uri('tag', start_mark)
974 + ch = self.peek()
975 + if ch not in u'\0 \r\n\x85\u2028\u2029':
976 + raise ScannerError("while scanning a tag", start_mark,
977 + "expected ' ', but found %r" % ch.encode('utf-8'),
978 + self.get_mark())
979 + value = (handle, suffix)
980 + end_mark = self.get_mark()
981 + return TagToken(value, start_mark, end_mark)
982 +
983 + def scan_block_scalar(self, style):
984 + # See the specification for details.
985 +
986 + if style == '>':
987 + folded = True
988 + else:
989 + folded = False
990 +
991 + chunks = []
992 + start_mark = self.get_mark()
993 +
994 + # Scan the header.
995 + self.forward()
996 + chomping, increment = self.scan_block_scalar_indicators(start_mark)
997 + self.scan_block_scalar_ignored_line(start_mark)
998 +
999 + # Determine the indentation level and go to the first non-empty line.
1000 + min_indent = self.indent+1
1001 + if min_indent < 1:
1002 + min_indent = 1
1003 + if increment is None:
1004 + breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
1005 + indent = max(min_indent, max_indent)
1006 + else:
1007 + indent = min_indent+increment-1
1008 + breaks, end_mark = self.scan_block_scalar_breaks(indent)
1009 + line_break = u''
1010 +
1011 + # Scan the inner part of the block scalar.
1012 + while self.column == indent and self.peek() != u'\0':
1013 + chunks.extend(breaks)
1014 + leading_non_space = self.peek() not in u' \t'
1015 + length = 0
1016 + while self.peek(length) not in u'\0\r\n\x85\u2028\u2029':
1017 + length += 1
1018 + chunks.append(self.prefix(length))
1019 + self.forward(length)
1020 + line_break = self.scan_line_break()
1021 + breaks, end_mark = self.scan_block_scalar_breaks(indent)
1022 + if self.column == indent and self.peek() != u'\0':
1023 +
1024 + # Unfortunately, folding rules are ambiguous.
1025 + #
1026 + # This is the folding according to the specification:
1027 +
1028 + if folded and line_break == u'\n' \
1029 + and leading_non_space and self.peek() not in u' \t':
1030 + if not breaks:
1031 + chunks.append(u' ')
1032 + else:
1033 + chunks.append(line_break)
1034 +
1035 + # This is Clark Evans's interpretation (also in the spec
1036 + # examples):
1037 + #
1038 + #if folded and line_break == u'\n':
1039 + # if not breaks:
1040 + # if self.peek() not in ' \t':
1041 + # chunks.append(u' ')
1042 + # else:
1043 + # chunks.append(line_break)
1044 + #else:
1045 + # chunks.append(line_break)
1046 + else:
1047 + break
1048 +
1049 + # Chomp the tail.
1050 + if chomping is not False:
1051 + chunks.append(line_break)
1052 + if chomping is True:
1053 + chunks.extend(breaks)
1054 +
1055 + # We are done.
1056 + return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
1057 + style)
1058 +
1059 + def scan_block_scalar_indicators(self, start_mark):
1060 + # See the specification for details.
1061 + chomping = None
1062 + increment = None
1063 + ch = self.peek()
1064 + if ch in u'+-':
1065 + if ch == '+':
1066 + chomping = True
1067 + else:
1068 + chomping = False
1069 + self.forward()
1070 + ch = self.peek()
1071 + if ch in u'0123456789':
1072 + increment = int(ch)
1073 + if increment == 0:
1074 + raise ScannerError("while scanning a block scalar", start_mark,
1075 + "expected indentation indicator in the range 1-9, but found 0",
1076 + self.get_mark())
1077 + self.forward()
1078 + elif ch in u'0123456789':
1079 + increment = int(ch)
1080 + if increment == 0:
1081 + raise ScannerError("while scanning a block scalar", start_mark,
1082 + "expected indentation indicator in the range 1-9, but found 0",
1083 + self.get_mark())
1084 + self.forward()
1085 + ch = self.peek()
1086 + if ch in u'+-':
1087 + if ch == '+':
1088 + chomping = True
1089 + else:
1090 + chomping = False
1091 + self.forward()
1092 + ch = self.peek()
1093 + if ch not in u'\0 \r\n\x85\u2028\u2029':
1094 + raise ScannerError("while scanning a block scalar", start_mark,
1095 + "expected chomping or indentation indicators, but found %r"
1096 + % ch.encode('utf-8'), self.get_mark())
1097 + return chomping, increment
1098 +
1099 + def scan_block_scalar_ignored_line(self, start_mark):
1100 + # See the specification for details.
1101 + while self.peek() == u' ':
1102 + self.forward()
1103 + if self.peek() == u'#':
1104 + while self.peek() not in u'\0\r\n\x85\u2028\u2029':
1105 + self.forward()
1106 + ch = self.peek()
1107 + if ch not in u'\0\r\n\x85\u2028\u2029':
1108 + raise ScannerError("while scanning a block scalar", start_mark,
1109 + "expected a comment or a line break, but found %r"
1110 + % ch.encode('utf-8'), self.get_mark())
1111 + self.scan_line_break()
1112 +
1113 + def scan_block_scalar_indentation(self):
1114 + # See the specification for details.
1115 + chunks = []
1116 + max_indent = 0
1117 + end_mark = self.get_mark()
1118 + while self.peek() in u' \r\n\x85\u2028\u2029':
1119 + if self.peek() != u' ':
1120 + chunks.append(self.scan_line_break())
1121 + end_mark = self.get_mark()
1122 + else:
1123 + self.forward()
1124 + if self.column > max_indent:
1125 + max_indent = self.column
1126 + return chunks, max_indent, end_mark
1127 +
1128 + def scan_block_scalar_breaks(self, indent):
1129 + # See the specification for details.
1130 + chunks = []
1131 + end_mark = self.get_mark()
1132 + while self.column < indent and self.peek() == u' ':
1133 + self.forward()
1134 + while self.peek() in u'\r\n\x85\u2028\u2029':
1135 + chunks.append(self.scan_line_break())
1136 + end_mark = self.get_mark()
1137 + while self.column < indent and self.peek() == u' ':
1138 + self.forward()
1139 + return chunks, end_mark
1140 +
1141 + def scan_flow_scalar(self, style):
1142 + # See the specification for details.
1143 + # Note that we loose indentation rules for quoted scalars. Quoted
1144 + # scalars don't need to adhere indentation because " and ' clearly
1145 + # mark the beginning and the end of them. Therefore we are less
1146 + # restrictive then the specification requires. We only need to check
1147 + # that document separators are not included in scalars.
1148 + if style == '"':
1149 + double = True
1150 + else:
1151 + double = False
1152 + chunks = []
1153 + start_mark = self.get_mark()
1154 + quote = self.peek()
1155 + self.forward()
1156 + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1157 + while self.peek() != quote:
1158 + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
1159 + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1160 + self.forward()
1161 + end_mark = self.get_mark()
1162 + return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
1163 + style)
1164 +
1165 + ESCAPE_REPLACEMENTS = {
1166 + u'0': u'\0',
1167 + u'a': u'\x07',
1168 + u'b': u'\x08',
1169 + u't': u'\x09',
1170 + u'\t': u'\x09',
1171 + u'n': u'\x0A',
1172 + u'v': u'\x0B',
1173 + u'f': u'\x0C',
1174 + u'r': u'\x0D',
1175 + u'e': u'\x1B',
1176 + u' ': u'\x20',
1177 + u'\"': u'\"',
1178 + u'\\': u'\\',
1179 + u'N': u'\x85',
1180 + u'_': u'\xA0',
1181 + u'L': u'\u2028',
1182 + u'P': u'\u2029',
1183 + }
1184 +
1185 + ESCAPE_CODES = {
1186 + u'x': 2,
1187 + u'u': 4,
1188 + u'U': 8,
1189 + }
1190 +
1191 + def scan_flow_scalar_non_spaces(self, double, start_mark):
1192 + # See the specification for details.
1193 + chunks = []
1194 + while True:
1195 + length = 0
1196 + while self.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
1197 + length += 1
1198 + if length:
1199 + chunks.append(self.prefix(length))
1200 + self.forward(length)
1201 + ch = self.peek()
1202 + if not double and ch == u'\'' and self.peek(1) == u'\'':
1203 + chunks.append(u'\'')
1204 + self.forward(2)
1205 + elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
1206 + chunks.append(ch)
1207 + self.forward()
1208 + elif double and ch == u'\\':
1209 + self.forward()
1210 + ch = self.peek()
1211 + if ch in self.ESCAPE_REPLACEMENTS:
1212 + chunks.append(self.ESCAPE_REPLACEMENTS[ch])
1213 + self.forward()
1214 + elif ch in self.ESCAPE_CODES:
1215 + length = self.ESCAPE_CODES[ch]
1216 + self.forward()
1217 + for k in range(length):
1218 + if self.peek(k) not in u'0123456789ABCDEFabcdef':
1219 + raise ScannerError("while scanning a double-quoted scalar", start_mark,
1220 + "expected escape sequence of %d hexdecimal numbers, but found %r" %
1221 + (length, self.peek(k).encode('utf-8')), self.get_mark())
1222 + code = int(self.prefix(length), 16)
1223 + chunks.append(unichr(code))
1224 + self.forward(length)
1225 + elif ch in u'\r\n\x85\u2028\u2029':
1226 + self.scan_line_break()
1227 + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
1228 + else:
1229 + raise ScannerError("while scanning a double-quoted scalar", start_mark,
1230 + "found unknown escape character %r" % ch.encode('utf-8'), self.get_mark())
1231 + else:
1232 + return chunks
1233 +
1234 + def scan_flow_scalar_spaces(self, double, start_mark):
1235 + # See the specification for details.
1236 + chunks = []
1237 + length = 0
1238 + while self.peek(length) in u' \t':
1239 + length += 1
1240 + whitespaces = self.prefix(length)
1241 + self.forward(length)
1242 + ch = self.peek()
1243 + if ch == u'\0':
1244 + raise ScannerError("while scanning a quoted scalar", start_mark,
1245 + "found unexpected end of stream", self.get_mark())
1246 + elif ch in u'\r\n\x85\u2028\u2029':
1247 + line_break = self.scan_line_break()
1248 + breaks = self.scan_flow_scalar_breaks(double, start_mark)
1249 + if line_break != u'\n':
1250 + chunks.append(line_break)
1251 + elif not breaks:
1252 + chunks.append(u' ')
1253 + chunks.extend(breaks)
1254 + else:
1255 + chunks.append(whitespaces)
1256 + return chunks
1257 +
1258 + def scan_flow_scalar_breaks(self, double, start_mark):
1259 + # See the specification for details.
1260 + chunks = []
1261 + while True:
1262 + # Instead of checking indentation, we check for document
1263 + # separators.
1264 + prefix = self.prefix(3)
1265 + if (prefix == u'---' or prefix == u'...') \
1266 + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1267 + raise ScannerError("while scanning a quoted scalar", start_mark,
1268 + "found unexpected document separator", self.get_mark())
1269 + while self.peek() in u' \t':
1270 + self.forward()
1271 + if self.peek() in u'\r\n\x85\u2028\u2029':
1272 + chunks.append(self.scan_line_break())
1273 + else:
1274 + return chunks
1275 +
1276 + def scan_plain(self):
1277 + # See the specification for details.
1278 + # We add an additional restriction for the flow context:
1279 + # plain scalars in the flow context cannot contain ',', ':' and '?'.
1280 + # We also keep track of the `allow_simple_key` flag here.
1281 + # Indentation rules are loosed for the flow context.
1282 + chunks = []
1283 + start_mark = self.get_mark()
1284 + end_mark = start_mark
1285 + indent = self.indent+1
1286 + # We allow zero indentation for scalars, but then we need to check for
1287 + # document separators at the beginning of the line.
1288 + #if indent == 0:
1289 + # indent = 1
1290 + spaces = []
1291 + while True:
1292 + length = 0
1293 + if self.peek() == u'#':
1294 + break
1295 + while True:
1296 + ch = self.peek(length)
1297 + if ch in u'\0 \t\r\n\x85\u2028\u2029' \
1298 + or (not self.flow_level and ch == u':' and
1299 + self.peek(length+1) in u'\0 \t\r\n\x85\u2028\u2029') \
1300 + or (self.flow_level and ch in u',:?[]{}'):
1301 + break
1302 + length += 1
1303 + # It's not clear what we should do with ':' in the flow context.
1304 + if (self.flow_level and ch == u':'
1305 + and self.peek(length+1) not in u'\0 \t\r\n\x85\u2028\u2029,[]{}'):
1306 + self.forward(length)
1307 + raise ScannerError("while scanning a plain scalar", start_mark,
1308 + "found unexpected ':'", self.get_mark(),
1309 + "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
1310 + if length == 0:
1311 + break
1312 + self.allow_simple_key = False
1313 + chunks.extend(spaces)
1314 + chunks.append(self.prefix(length))
1315 + self.forward(length)
1316 + end_mark = self.get_mark()
1317 + spaces = self.scan_plain_spaces(indent, start_mark)
1318 + if not spaces or self.peek() == u'#' \
1319 + or (not self.flow_level and self.column < indent):
1320 + break
1321 + return ScalarToken(u''.join(chunks), True, start_mark, end_mark)
1322 +
1323 + def scan_plain_spaces(self, indent, start_mark):
1324 + # See the specification for details.
1325 + # The specification is really confusing about tabs in plain scalars.
1326 + # We just forbid them completely. Do not use tabs in YAML!
1327 + chunks = []
1328 + length = 0
1329 + while self.peek(length) in u' ':
1330 + length += 1
1331 + whitespaces = self.prefix(length)
1332 + self.forward(length)
1333 + ch = self.peek()
1334 + if ch in u'\r\n\x85\u2028\u2029':
1335 + line_break = self.scan_line_break()
1336 + self.allow_simple_key = True
1337 + prefix = self.prefix(3)
1338 + if (prefix == u'---' or prefix == u'...') \
1339 + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1340 + return
1341 + breaks = []
1342 + while self.peek() in u' \r\n\x85\u2028\u2029':
1343 + if self.peek() == ' ':
1344 + self.forward()
1345 + else:
1346 + breaks.append(self.scan_line_break())
1347 + prefix = self.prefix(3)
1348 + if (prefix == u'---' or prefix == u'...') \
1349 + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1350 + return
1351 + if line_break != u'\n':
1352 + chunks.append(line_break)
1353 + elif not breaks:
1354 + chunks.append(u' ')
1355 + chunks.extend(breaks)
1356 + elif whitespaces:
1357 + chunks.append(whitespaces)
1358 + return chunks
1359 +
1360 + def scan_tag_handle(self, name, start_mark):
1361 + # See the specification for details.
1362 + # For some strange reasons, the specification does not allow '_' in
1363 + # tag handles. I have allowed it anyway.
1364 + ch = self.peek()
1365 + if ch != u'!':
1366 + raise ScannerError("while scanning a %s" % name, start_mark,
1367 + "expected '!', but found %r" % ch.encode('utf-8'),
1368 + self.get_mark())
1369 + length = 1
1370 + ch = self.peek(length)
1371 + if ch != u' ':
1372 + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
1373 + or ch in u'-_':
1374 + length += 1
1375 + ch = self.peek(length)
1376 + if ch != u'!':
1377 + self.forward(length)
1378 + raise ScannerError("while scanning a %s" % name, start_mark,
1379 + "expected '!', but found %r" % ch.encode('utf-8'),
1380 + self.get_mark())
1381 + length += 1
1382 + value = self.prefix(length)
1383 + self.forward(length)
1384 + return value
1385 +
1386 + def scan_tag_uri(self, name, start_mark):
1387 + # See the specification for details.
1388 + # Note: we do not check if URI is well-formed.
1389 + chunks = []
1390 + length = 0
1391 + ch = self.peek(length)
1392 + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
1393 + or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
1394 + if ch == u'%':
1395 + chunks.append(self.prefix(length))
1396 + self.forward(length)
1397 + length = 0
1398 + chunks.append(self.scan_uri_escapes(name, start_mark))
1399 + else:
1400 + length += 1
1401 + ch = self.peek(length)
1402 + if length:
1403 + chunks.append(self.prefix(length))
1404 + self.forward(length)
1405 + length = 0
1406 + if not chunks:
1407 + raise ScannerError("while parsing a %s" % name, start_mark,
1408 + "expected URI, but found %r" % ch.encode('utf-8'),
1409 + self.get_mark())
1410 + return u''.join(chunks)
1411 +
1412 + def scan_uri_escapes(self, name, start_mark):
1413 + # See the specification for details.
1414 + bytes = []
1415 + mark = self.get_mark()
1416 + while self.peek() == u'%':
1417 + self.forward()
1418 + for k in range(2):
1419 + if self.peek(k) not in u'0123456789ABCDEFabcdef':
1420 + raise ScannerError("while scanning a %s" % name, start_mark,
1421 + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
1422 + (self.peek(k).encode('utf-8')), self.get_mark())
1423 + bytes.append(chr(int(self.prefix(2), 16)))
1424 + self.forward(2)
1425 + try:
1426 + value = unicode(''.join(bytes), 'utf-8')
1427 + except UnicodeDecodeError, exc:
1428 + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
1429 + return value
1430 +
1431 + def scan_line_break(self):
1432 + # Transforms:
1433 + # '\r\n' : '\n'
1434 + # '\r' : '\n'
1435 + # '\n' : '\n'
1436 + # '\x85' : '\n'
1437 + # '\u2028' : '\u2028'
1438 + # '\u2029 : '\u2029'
1439 + # default : ''
1440 + ch = self.peek()
1441 + if ch in u'\r\n\x85':
1442 + if self.prefix(2) == u'\r\n':
1443 + self.forward(2)
1444 + else:
1445 + self.forward()
1446 + return u'\n'
1447 + elif ch in u'\u2028\u2029':
1448 + self.forward()
1449 + return ch
1450 + return u''
1451 +
1452 +#try:
1453 +# import psyco
1454 +# psyco.bind(Scanner)
1455 +#except ImportError:
1456 +# pass
1457 +
python.d/python_modules/pyyaml2/serializer.py new
+111
@@ -0,0 +1,111 @@
1 +
2 +__all__ = ['Serializer', 'SerializerError']
3 +
4 +from error import YAMLError
5 +from events import *
6 +from nodes import *
7 +
8 +class SerializerError(YAMLError):
9 + pass
10 +
11 +class Serializer(object):
12 +
13 + ANCHOR_TEMPLATE = u'id%03d'
14 +
15 + def __init__(self, encoding=None,
16 + explicit_start=None, explicit_end=None, version=None, tags=None):
17 + self.use_encoding = encoding
18 + self.use_explicit_start = explicit_start
19 + self.use_explicit_end = explicit_end
20 + self.use_version = version
21 + self.use_tags = tags
22 + self.serialized_nodes = {}
23 + self.anchors = {}
24 + self.last_anchor_id = 0
25 + self.closed = None
26 +
27 + def open(self):
28 + if self.closed is None:
29 + self.emit(StreamStartEvent(encoding=self.use_encoding))
30 + self.closed = False
31 + elif self.closed:
32 + raise SerializerError("serializer is closed")
33 + else:
34 + raise SerializerError("serializer is already opened")
35 +
36 + def close(self):
37 + if self.closed is None:
38 + raise SerializerError("serializer is not opened")
39 + elif not self.closed:
40 + self.emit(StreamEndEvent())
41 + self.closed = True
42 +
43 + #def __del__(self):
44 + # self.close()
45 +
46 + def serialize(self, node):
47 + if self.closed is None:
48 + raise SerializerError("serializer is not opened")
49 + elif self.closed:
50 + raise SerializerError("serializer is closed")
51 + self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
52 + version=self.use_version, tags=self.use_tags))
53 + self.anchor_node(node)
54 + self.serialize_node(node, None, None)
55 + self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
56 + self.serialized_nodes = {}
57 + self.anchors = {}
58 + self.last_anchor_id = 0
59 +
60 + def anchor_node(self, node):
61 + if node in self.anchors:
62 + if self.anchors[node] is None:
63 + self.anchors[node] = self.generate_anchor(node)
64 + else:
65 + self.anchors[node] = None
66 + if isinstance(node, SequenceNode):
67 + for item in node.value:
68 + self.anchor_node(item)
69 + elif isinstance(node, MappingNode):
70 + for key, value in node.value:
71 + self.anchor_node(key)
72 + self.anchor_node(value)
73 +
74 + def generate_anchor(self, node):
75 + self.last_anchor_id += 1
76 + return self.ANCHOR_TEMPLATE % self.last_anchor_id
77 +
78 + def serialize_node(self, node, parent, index):
79 + alias = self.anchors[node]
80 + if node in self.serialized_nodes:
81 + self.emit(AliasEvent(alias))
82 + else:
83 + self.serialized_nodes[node] = True
84 + self.descend_resolver(parent, index)
85 + if isinstance(node, ScalarNode):
86 + detected_tag = self.resolve(ScalarNode, node.value, (True, False))
87 + default_tag = self.resolve(ScalarNode, node.value, (False, True))
88 + implicit = (node.tag == detected_tag), (node.tag == default_tag)
89 + self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
90 + style=node.style))
91 + elif isinstance(node, SequenceNode):
92 + implicit = (node.tag
93 + == self.resolve(SequenceNode, node.value, True))
94 + self.emit(SequenceStartEvent(alias, node.tag, implicit,
95 + flow_style=node.flow_style))
96 + index = 0
97 + for item in node.value:
98 + self.serialize_node(item, node, index)
99 + index += 1
100 + self.emit(SequenceEndEvent())
101 + elif isinstance(node, MappingNode):
102 + implicit = (node.tag
103 + == self.resolve(MappingNode, node.value, True))
104 + self.emit(MappingStartEvent(alias, node.tag, implicit,
105 + flow_style=node.flow_style))
106 + for key, value in node.value:
107 + self.serialize_node(key, node, None)
108 + self.serialize_node(value, node, key)
109 + self.emit(MappingEndEvent())
110 + self.ascend_resolver()
111 +
python.d/python_modules/pyyaml2/tokens.py new
+104
@@ -0,0 +1,104 @@
1 +
2 +class Token(object):
3 + def __init__(self, start_mark, end_mark):
4 + self.start_mark = start_mark
5 + self.end_mark = end_mark
6 + def __repr__(self):
7 + attributes = [key for key in self.__dict__
8 + if not key.endswith('_mark')]
9 + attributes.sort()
10 + arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
11 + for key in attributes])
12 + return '%s(%s)' % (self.__class__.__name__, arguments)
13 +
14 +#class BOMToken(Token):
15 +# id = '<byte order mark>'
16 +
17 +class DirectiveToken(Token):
18 + id = '<directive>'
19 + def __init__(self, name, value, start_mark, end_mark):
20 + self.name = name
21 + self.value = value
22 + self.start_mark = start_mark
23 + self.end_mark = end_mark
24 +
25 +class DocumentStartToken(Token):
26 + id = '<document start>'
27 +
28 +class DocumentEndToken(Token):
29 + id = '<document end>'
30 +
31 +class StreamStartToken(Token):
32 + id = '<stream start>'
33 + def __init__(self, start_mark=None, end_mark=None,
34 + encoding=None):
35 + self.start_mark = start_mark
36 + self.end_mark = end_mark
37 + self.encoding = encoding
38 +
39 +class StreamEndToken(Token):
40 + id = '<stream end>'
41 +
42 +class BlockSequenceStartToken(Token):
43 + id = '<block sequence start>'
44 +
45 +class BlockMappingStartToken(Token):
46 + id = '<block mapping start>'
47 +
48 +class BlockEndToken(Token):
49 + id = '<block end>'
50 +
51 +class FlowSequenceStartToken(Token):
52 + id = '['
53 +
54 +class FlowMappingStartToken(Token):
55 + id = '{'
56 +
57 +class FlowSequenceEndToken(Token):
58 + id = ']'
59 +
60 +class FlowMappingEndToken(Token):
61 + id = '}'
62 +
63 +class KeyToken(Token):
64 + id = '?'
65 +
66 +class ValueToken(Token):
67 + id = ':'
68 +
69 +class BlockEntryToken(Token):
70 + id = '-'
71 +
72 +class FlowEntryToken(Token):
73 + id = ','
74 +
75 +class AliasToken(Token):
76 + id = '<alias>'
77 + def __init__(self, value, start_mark, end_mark):
78 + self.value = value
79 + self.start_mark = start_mark
80 + self.end_mark = end_mark
81 +
82 +class AnchorToken(Token):
83 + id = '<anchor>'
84 + def __init__(self, value, start_mark, end_mark):
85 + self.value = value
86 + self.start_mark = start_mark
87 + self.end_mark = end_mark
88 +
89 +class TagToken(Token):
90 + id = '<tag>'
91 + def __init__(self, value, start_mark, end_mark):
92 + self.value = value
93 + self.start_mark = start_mark
94 + self.end_mark = end_mark
95 +
96 +class ScalarToken(Token):
97 + id = '<scalar>'
98 + def __init__(self, value, plain, start_mark, end_mark, style=None):
99 + self.value = value
100 + self.plain = plain
101 + self.start_mark = start_mark
102 + self.end_mark = end_mark
103 + self.style = style
104 +
python.d/python_modules/pyyaml3/__init__.py new
+312
@@ -0,0 +1,312 @@
1 +
2 +from .error import *
3 +
4 +from .tokens import *
5 +from .events import *
6 +from .nodes import *
7 +
8 +from .loader import *
9 +from .dumper import *
10 +
11 +__version__ = '3.11'
12 +try:
13 + from .cyaml import *
14 + __with_libyaml__ = True
15 +except ImportError:
16 + __with_libyaml__ = False
17 +
18 +import io
19 +
20 +def scan(stream, Loader=Loader):
21 + """
22 + Scan a YAML stream and produce scanning tokens.
23 + """
24 + loader = Loader(stream)
25 + try:
26 + while loader.check_token():
27 + yield loader.get_token()
28 + finally:
29 + loader.dispose()
30 +
31 +def parse(stream, Loader=Loader):
32 + """
33 + Parse a YAML stream and produce parsing events.
34 + """
35 + loader = Loader(stream)
36 + try:
37 + while loader.check_event():
38 + yield loader.get_event()
39 + finally:
40 + loader.dispose()
41 +
42 +def compose(stream, Loader=Loader):
43 + """
44 + Parse the first YAML document in a stream
45 + and produce the corresponding representation tree.
46 + """
47 + loader = Loader(stream)
48 + try:
49 + return loader.get_single_node()
50 + finally:
51 + loader.dispose()
52 +
53 +def compose_all(stream, Loader=Loader):
54 + """
55 + Parse all YAML documents in a stream
56 + and produce corresponding representation trees.
57 + """
58 + loader = Loader(stream)
59 + try:
60 + while loader.check_node():
61 + yield loader.get_node()
62 + finally:
63 + loader.dispose()
64 +
65 +def load(stream, Loader=Loader):
66 + """
67 + Parse the first YAML document in a stream
68 + and produce the corresponding Python object.
69 + """
70 + loader = Loader(stream)
71 + try:
72 + return loader.get_single_data()
73 + finally:
74 + loader.dispose()
75 +
76 +def load_all(stream, Loader=Loader):
77 + """
78 + Parse all YAML documents in a stream
79 + and produce corresponding Python objects.
80 + """
81 + loader = Loader(stream)
82 + try:
83 + while loader.check_data():
84 + yield loader.get_data()
85 + finally:
86 + loader.dispose()
87 +
88 +def safe_load(stream):
89 + """
90 + Parse the first YAML document in a stream
91 + and produce the corresponding Python object.
92 + Resolve only basic YAML tags.
93 + """
94 + return load(stream, SafeLoader)
95 +
96 +def safe_load_all(stream):
97 + """
98 + Parse all YAML documents in a stream
99 + and produce corresponding Python objects.
100 + Resolve only basic YAML tags.
101 + """
102 + return load_all(stream, SafeLoader)
103 +
104 +def emit(events, stream=None, Dumper=Dumper,
105 + canonical=None, indent=None, width=None,
106 + allow_unicode=None, line_break=None):
107 + """
108 + Emit YAML parsing events into a stream.
109 + If stream is None, return the produced string instead.
110 + """
111 + getvalue = None
112 + if stream is None:
113 + stream = io.StringIO()
114 + getvalue = stream.getvalue
115 + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
116 + allow_unicode=allow_unicode, line_break=line_break)
117 + try:
118 + for event in events:
119 + dumper.emit(event)
120 + finally:
121 + dumper.dispose()
122 + if getvalue:
123 + return getvalue()
124 +
125 +def serialize_all(nodes, stream=None, Dumper=Dumper,
126 + canonical=None, indent=None, width=None,
127 + allow_unicode=None, line_break=None,
128 + encoding=None, explicit_start=None, explicit_end=None,
129 + version=None, tags=None):
130 + """
131 + Serialize a sequence of representation trees into a YAML stream.
132 + If stream is None, return the produced string instead.
133 + """
134 + getvalue = None
135 + if stream is None:
136 + if encoding is None:
137 + stream = io.StringIO()
138 + else:
139 + stream = io.BytesIO()
140 + getvalue = stream.getvalue
141 + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
142 + allow_unicode=allow_unicode, line_break=line_break,
143 + encoding=encoding, version=version, tags=tags,
144 + explicit_start=explicit_start, explicit_end=explicit_end)
145 + try:
146 + dumper.open()
147 + for node in nodes:
148 + dumper.serialize(node)
149 + dumper.close()
150 + finally:
151 + dumper.dispose()
152 + if getvalue:
153 + return getvalue()
154 +
155 +def serialize(node, stream=None, Dumper=Dumper, **kwds):
156 + """
157 + Serialize a representation tree into a YAML stream.
158 + If stream is None, return the produced string instead.
159 + """
160 + return serialize_all([node], stream, Dumper=Dumper, **kwds)
161 +
162 +def dump_all(documents, stream=None, Dumper=Dumper,
163 + default_style=None, default_flow_style=None,
164 + canonical=None, indent=None, width=None,
165 + allow_unicode=None, line_break=None,
166 + encoding=None, explicit_start=None, explicit_end=None,
167 + version=None, tags=None):
168 + """
169 + Serialize a sequence of Python objects into a YAML stream.
170 + If stream is None, return the produced string instead.
171 + """
172 + getvalue = None
173 + if stream is None:
174 + if encoding is None:
175 + stream = io.StringIO()
176 + else:
177 + stream = io.BytesIO()
178 + getvalue = stream.getvalue
179 + dumper = Dumper(stream, default_style=default_style,
180 + default_flow_style=default_flow_style,
181 + canonical=canonical, indent=indent, width=width,
182 + allow_unicode=allow_unicode, line_break=line_break,
183 + encoding=encoding, version=version, tags=tags,
184 + explicit_start=explicit_start, explicit_end=explicit_end)
185 + try:
186 + dumper.open()
187 + for data in documents:
188 + dumper.represent(data)
189 + dumper.close()
190 + finally:
191 + dumper.dispose()
192 + if getvalue:
193 + return getvalue()
194 +
195 +def dump(data, stream=None, Dumper=Dumper, **kwds):
196 + """
197 + Serialize a Python object into a YAML stream.
198 + If stream is None, return the produced string instead.
199 + """
200 + return dump_all([data], stream, Dumper=Dumper, **kwds)
201 +
202 +def safe_dump_all(documents, stream=None, **kwds):
203 + """
204 + Serialize a sequence of Python objects into a YAML stream.
205 + Produce only basic YAML tags.
206 + If stream is None, return the produced string instead.
207 + """
208 + return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
209 +
210 +def safe_dump(data, stream=None, **kwds):
211 + """
212 + Serialize a Python object into a YAML stream.
213 + Produce only basic YAML tags.
214 + If stream is None, return the produced string instead.
215 + """
216 + return dump_all([data], stream, Dumper=SafeDumper, **kwds)
217 +
218 +def add_implicit_resolver(tag, regexp, first=None,
219 + Loader=Loader, Dumper=Dumper):
220 + """
221 + Add an implicit scalar detector.
222 + If an implicit scalar value matches the given regexp,
223 + the corresponding tag is assigned to the scalar.
224 + first is a sequence of possible initial characters or None.
225 + """
226 + Loader.add_implicit_resolver(tag, regexp, first)
227 + Dumper.add_implicit_resolver(tag, regexp, first)
228 +
229 +def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper):
230 + """
231 + Add a path based resolver for the given tag.
232 + A path is a list of keys that forms a path
233 + to a node in the representation tree.
234 + Keys can be string values, integers, or None.
235 + """
236 + Loader.add_path_resolver(tag, path, kind)
237 + Dumper.add_path_resolver(tag, path, kind)
238 +
239 +def add_constructor(tag, constructor, Loader=Loader):
240 + """
241 + Add a constructor for the given tag.
242 + Constructor is a function that accepts a Loader instance
243 + and a node object and produces the corresponding Python object.
244 + """
245 + Loader.add_constructor(tag, constructor)
246 +
247 +def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader):
248 + """
249 + Add a multi-constructor for the given tag prefix.
250 + Multi-constructor is called for a node if its tag starts with tag_prefix.
251 + Multi-constructor accepts a Loader instance, a tag suffix,
252 + and a node object and produces the corresponding Python object.
253 + """
254 + Loader.add_multi_constructor(tag_prefix, multi_constructor)
255 +
256 +def add_representer(data_type, representer, Dumper=Dumper):
257 + """
258 + Add a representer for the given type.
259 + Representer is a function accepting a Dumper instance
260 + and an instance of the given data type
261 + and producing the corresponding representation node.
262 + """
263 + Dumper.add_representer(data_type, representer)
264 +
265 +def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
266 + """
267 + Add a representer for the given type.
268 + Multi-representer is a function accepting a Dumper instance
269 + and an instance of the given data type or subtype
270 + and producing the corresponding representation node.
271 + """
272 + Dumper.add_multi_representer(data_type, multi_representer)
273 +
274 +class YAMLObjectMetaclass(type):
275 + """
276 + The metaclass for YAMLObject.
277 + """
278 + def __init__(cls, name, bases, kwds):
279 + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
280 + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
281 + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
282 + cls.yaml_dumper.add_representer(cls, cls.to_yaml)
283 +
284 +class YAMLObject(metaclass=YAMLObjectMetaclass):
285 + """
286 + An object that can dump itself to a YAML stream
287 + and load itself from a YAML stream.
288 + """
289 +
290 + __slots__ = () # no direct instantiation, so allow immutable subclasses
291 +
292 + yaml_loader = Loader
293 + yaml_dumper = Dumper
294 +
295 + yaml_tag = None
296 + yaml_flow_style = None
297 +
298 + @classmethod
299 + def from_yaml(cls, loader, node):
300 + """
301 + Convert a representation node to a Python object.
302 + """
303 + return loader.construct_yaml_object(node, cls)
304 +
305 + @classmethod
306 + def to_yaml(cls, dumper, data):
307 + """
308 + Convert a Python object to a representation node.
309 + """
310 + return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
311 + flow_style=cls.yaml_flow_style)
312 +
python.d/python_modules/pyyaml3/composer.py new
+139
@@ -0,0 +1,139 @@
1 +
2 +__all__ = ['Composer', 'ComposerError']
3 +
4 +from .error import MarkedYAMLError
5 +from .events import *
6 +from .nodes import *
7 +
8 +class ComposerError(MarkedYAMLError):
9 + pass
10 +
11 +class Composer:
12 +
13 + def __init__(self):
14 + self.anchors = {}
15 +
16 + def check_node(self):
17 + # Drop the STREAM-START event.
18 + if self.check_event(StreamStartEvent):
19 + self.get_event()
20 +
21 + # If there are more documents available?
22 + return not self.check_event(StreamEndEvent)
23 +
24 + def get_node(self):
25 + # Get the root node of the next document.
26 + if not self.check_event(StreamEndEvent):
27 + return self.compose_document()
28 +
29 + def get_single_node(self):
30 + # Drop the STREAM-START event.
31 + self.get_event()
32 +
33 + # Compose a document if the stream is not empty.
34 + document = None
35 + if not self.check_event(StreamEndEvent):
36 + document = self.compose_document()
37 +
38 + # Ensure that the stream contains no more documents.
39 + if not self.check_event(StreamEndEvent):
40 + event = self.get_event()
41 + raise ComposerError("expected a single document in the stream",
42 + document.start_mark, "but found another document",
43 + event.start_mark)
44 +
45 + # Drop the STREAM-END event.
46 + self.get_event()
47 +
48 + return document
49 +
50 + def compose_document(self):
51 + # Drop the DOCUMENT-START event.
52 + self.get_event()
53 +
54 + # Compose the root node.
55 + node = self.compose_node(None, None)
56 +
57 + # Drop the DOCUMENT-END event.
58 + self.get_event()
59 +
60 + self.anchors = {}
61 + return node
62 +
63 + def compose_node(self, parent, index):
64 + if self.check_event(AliasEvent):
65 + event = self.get_event()
66 + anchor = event.anchor
67 + if anchor not in self.anchors:
68 + raise ComposerError(None, None, "found undefined alias %r"
69 + % anchor, event.start_mark)
70 + return self.anchors[anchor]
71 + event = self.peek_event()
72 + anchor = event.anchor
73 + if anchor is not None:
74 + if anchor in self.anchors:
75 + raise ComposerError("found duplicate anchor %r; first occurence"
76 + % anchor, self.anchors[anchor].start_mark,
77 + "second occurence", event.start_mark)
78 + self.descend_resolver(parent, index)
79 + if self.check_event(ScalarEvent):
80 + node = self.compose_scalar_node(anchor)
81 + elif self.check_event(SequenceStartEvent):
82 + node = self.compose_sequence_node(anchor)
83 + elif self.check_event(MappingStartEvent):
84 + node = self.compose_mapping_node(anchor)
85 + self.ascend_resolver()
86 + return node
87 +
88 + def compose_scalar_node(self, anchor):
89 + event = self.get_event()
90 + tag = event.tag
91 + if tag is None or tag == '!':
92 + tag = self.resolve(ScalarNode, event.value, event.implicit)
93 + node = ScalarNode(tag, event.value,
94 + event.start_mark, event.end_mark, style=event.style)
95 + if anchor is not None:
96 + self.anchors[anchor] = node
97 + return node
98 +
99 + def compose_sequence_node(self, anchor):
100 + start_event = self.get_event()
101 + tag = start_event.tag
102 + if tag is None or tag == '!':
103 + tag = self.resolve(SequenceNode, None, start_event.implicit)
104 + node = SequenceNode(tag, [],
105 + start_event.start_mark, None,
106 + flow_style=start_event.flow_style)
107 + if anchor is not None:
108 + self.anchors[anchor] = node
109 + index = 0
110 + while not self.check_event(SequenceEndEvent):
111 + node.value.append(self.compose_node(node, index))
112 + index += 1
113 + end_event = self.get_event()
114 + node.end_mark = end_event.end_mark
115 + return node
116 +
117 + def compose_mapping_node(self, anchor):
118 + start_event = self.get_event()
119 + tag = start_event.tag
120 + if tag is None or tag == '!':
121 + tag = self.resolve(MappingNode, None, start_event.implicit)
122 + node = MappingNode(tag, [],
123 + start_event.start_mark, None,
124 + flow_style=start_event.flow_style)
125 + if anchor is not None:
126 + self.anchors[anchor] = node
127 + while not self.check_event(MappingEndEvent):
128 + #key_event = self.peek_event()
129 + item_key = self.compose_node(node, None)
130 + #if item_key in node.value:
131 + # raise ComposerError("while composing a mapping", start_event.start_mark,
132 + # "found duplicate key", key_event.start_mark)
133 + item_value = self.compose_node(node, item_key)
134 + #node.value[item_key] = item_value
135 + node.value.append((item_key, item_value))
136 + end_event = self.get_event()
137 + node.end_mark = end_event.end_mark
138 + return node
139 +
python.d/python_modules/pyyaml3/constructor.py new
+686
@@ -0,0 +1,686 @@
1 +
2 +__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
3 + 'ConstructorError']
4 +
5 +from .error import *
6 +from .nodes import *
7 +
8 +import collections, datetime, base64, binascii, re, sys, types
9 +
10 +class ConstructorError(MarkedYAMLError):
11 + pass
12 +
13 +class BaseConstructor:
14 +
15 + yaml_constructors = {}
16 + yaml_multi_constructors = {}
17 +
18 + def __init__(self):
19 + self.constructed_objects = {}
20 + self.recursive_objects = {}
21 + self.state_generators = []
22 + self.deep_construct = False
23 +
24 + def check_data(self):
25 + # If there are more documents available?
26 + return self.check_node()
27 +
28 + def get_data(self):
29 + # Construct and return the next document.
30 + if self.check_node():
31 + return self.construct_document(self.get_node())
32 +
33 + def get_single_data(self):
34 + # Ensure that the stream contains a single document and construct it.
35 + node = self.get_single_node()
36 + if node is not None:
37 + return self.construct_document(node)
38 + return None
39 +
40 + def construct_document(self, node):
41 + data = self.construct_object(node)
42 + while self.state_generators:
43 + state_generators = self.state_generators
44 + self.state_generators = []
45 + for generator in state_generators:
46 + for dummy in generator:
47 + pass
48 + self.constructed_objects = {}
49 + self.recursive_objects = {}
50 + self.deep_construct = False
51 + return data
52 +
53 + def construct_object(self, node, deep=False):
54 + if node in self.constructed_objects:
55 + return self.constructed_objects[node]
56 + if deep:
57 + old_deep = self.deep_construct
58 + self.deep_construct = True
59 + if node in self.recursive_objects:
60 + raise ConstructorError(None, None,
61 + "found unconstructable recursive node", node.start_mark)
62 + self.recursive_objects[node] = None
63 + constructor = None
64 + tag_suffix = None
65 + if node.tag in self.yaml_constructors:
66 + constructor = self.yaml_constructors[node.tag]
67 + else:
68 + for tag_prefix in self.yaml_multi_constructors:
69 + if node.tag.startswith(tag_prefix):
70 + tag_suffix = node.tag[len(tag_prefix):]
71 + constructor = self.yaml_multi_constructors[tag_prefix]
72 + break
73 + else:
74 + if None in self.yaml_multi_constructors:
75 + tag_suffix = node.tag
76 + constructor = self.yaml_multi_constructors[None]
77 + elif None in self.yaml_constructors:
78 + constructor = self.yaml_constructors[None]
79 + elif isinstance(node, ScalarNode):
80 + constructor = self.__class__.construct_scalar
81 + elif isinstance(node, SequenceNode):
82 + constructor = self.__class__.construct_sequence
83 + elif isinstance(node, MappingNode):
84 + constructor = self.__class__.construct_mapping
85 + if tag_suffix is None:
86 + data = constructor(self, node)
87 + else:
88 + data = constructor(self, tag_suffix, node)
89 + if isinstance(data, types.GeneratorType):
90 + generator = data
91 + data = next(generator)
92 + if self.deep_construct:
93 + for dummy in generator:
94 + pass
95 + else:
96 + self.state_generators.append(generator)
97 + self.constructed_objects[node] = data
98 + del self.recursive_objects[node]
99 + if deep:
100 + self.deep_construct = old_deep
101 + return data
102 +
103 + def construct_scalar(self, node):
104 + if not isinstance(node, ScalarNode):
105 + raise ConstructorError(None, None,
106 + "expected a scalar node, but found %s" % node.id,
107 + node.start_mark)
108 + return node.value
109 +
110 + def construct_sequence(self, node, deep=False):
111 + if not isinstance(node, SequenceNode):
112 + raise ConstructorError(None, None,
113 + "expected a sequence node, but found %s" % node.id,
114 + node.start_mark)
115 + return [self.construct_object(child, deep=deep)
116 + for child in node.value]
117 +
118 + def construct_mapping(self, node, deep=False):
119 + if not isinstance(node, MappingNode):
120 + raise ConstructorError(None, None,
121 + "expected a mapping node, but found %s" % node.id,
122 + node.start_mark)
123 + mapping = {}
124 + for key_node, value_node in node.value:
125 + key = self.construct_object(key_node, deep=deep)
126 + if not isinstance(key, collections.Hashable):
127 + raise ConstructorError("while constructing a mapping", node.start_mark,
128 + "found unhashable key", key_node.start_mark)
129 + value = self.construct_object(value_node, deep=deep)
130 + mapping[key] = value
131 + return mapping
132 +
133 + def construct_pairs(self, node, deep=False):
134 + if not isinstance(node, MappingNode):
135 + raise ConstructorError(None, None,
136 + "expected a mapping node, but found %s" % node.id,
137 + node.start_mark)
138 + pairs = []
139 + for key_node, value_node in node.value:
140 + key = self.construct_object(key_node, deep=deep)
141 + value = self.construct_object(value_node, deep=deep)
142 + pairs.append((key, value))
143 + return pairs
144 +
145 + @classmethod
146 + def add_constructor(cls, tag, constructor):
147 + if not 'yaml_constructors' in cls.__dict__:
148 + cls.yaml_constructors = cls.yaml_constructors.copy()
149 + cls.yaml_constructors[tag] = constructor
150 +
151 + @classmethod
152 + def add_multi_constructor(cls, tag_prefix, multi_constructor):
153 + if not 'yaml_multi_constructors' in cls.__dict__:
154 + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
155 + cls.yaml_multi_constructors[tag_prefix] = multi_constructor
156 +
157 +class SafeConstructor(BaseConstructor):
158 +
159 + def construct_scalar(self, node):
160 + if isinstance(node, MappingNode):
161 + for key_node, value_node in node.value:
162 + if key_node.tag == 'tag:yaml.org,2002:value':
163 + return self.construct_scalar(value_node)
164 + return super().construct_scalar(node)
165 +
166 + def flatten_mapping(self, node):
167 + merge = []
168 + index = 0
169 + while index < len(node.value):
170 + key_node, value_node = node.value[index]
171 + if key_node.tag == 'tag:yaml.org,2002:merge':
172 + del node.value[index]
173 + if isinstance(value_node, MappingNode):
174 + self.flatten_mapping(value_node)
175 + merge.extend(value_node.value)
176 + elif isinstance(value_node, SequenceNode):
177 + submerge = []
178 + for subnode in value_node.value:
179 + if not isinstance(subnode, MappingNode):
180 + raise ConstructorError("while constructing a mapping",
181 + node.start_mark,
182 + "expected a mapping for merging, but found %s"
183 + % subnode.id, subnode.start_mark)
184 + self.flatten_mapping(subnode)
185 + submerge.append(subnode.value)
186 + submerge.reverse()
187 + for value in submerge:
188 + merge.extend(value)
189 + else:
190 + raise ConstructorError("while constructing a mapping", node.start_mark,
191 + "expected a mapping or list of mappings for merging, but found %s"
192 + % value_node.id, value_node.start_mark)
193 + elif key_node.tag == 'tag:yaml.org,2002:value':
194 + key_node.tag = 'tag:yaml.org,2002:str'
195 + index += 1
196 + else:
197 + index += 1
198 + if merge:
199 + node.value = merge + node.value
200 +
201 + def construct_mapping(self, node, deep=False):
202 + if isinstance(node, MappingNode):
203 + self.flatten_mapping(node)
204 + return super().construct_mapping(node, deep=deep)
205 +
206 + def construct_yaml_null(self, node):
207 + self.construct_scalar(node)
208 + return None
209 +
210 + bool_values = {
211 + 'yes': True,
212 + 'no': False,
213 + 'true': True,
214 + 'false': False,
215 + 'on': True,
216 + 'off': False,
217 + }
218 +
219 + def construct_yaml_bool(self, node):
220 + value = self.construct_scalar(node)
221 + return self.bool_values[value.lower()]
222 +
223 + def construct_yaml_int(self, node):
224 + value = self.construct_scalar(node)
225 + value = value.replace('_', '')
226 + sign = +1
227 + if value[0] == '-':
228 + sign = -1
229 + if value[0] in '+-':
230 + value = value[1:]
231 + if value == '0':
232 + return 0
233 + elif value.startswith('0b'):
234 + return sign*int(value[2:], 2)
235 + elif value.startswith('0x'):
236 + return sign*int(value[2:], 16)
237 + elif value[0] == '0':
238 + return sign*int(value, 8)
239 + elif ':' in value:
240 + digits = [int(part) for part in value.split(':')]
241 + digits.reverse()
242 + base = 1
243 + value = 0
244 + for digit in digits:
245 + value += digit*base
246 + base *= 60
247 + return sign*value
248 + else:
249 + return sign*int(value)
250 +
251 + inf_value = 1e300
252 + while inf_value != inf_value*inf_value:
253 + inf_value *= inf_value
254 + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
255 +
256 + def construct_yaml_float(self, node):
257 + value = self.construct_scalar(node)
258 + value = value.replace('_', '').lower()
259 + sign = +1
260 + if value[0] == '-':
261 + sign = -1
262 + if value[0] in '+-':
263 + value = value[1:]
264 + if value == '.inf':
265 + return sign*self.inf_value
266 + elif value == '.nan':
267 + return self.nan_value
268 + elif ':' in value:
269 + digits = [float(part) for part in value.split(':')]
270 + digits.reverse()
271 + base = 1
272 + value = 0.0
273 + for digit in digits:
274 + value += digit*base
275 + base *= 60
276 + return sign*value
277 + else:
278 + return sign*float(value)
279 +
280 + def construct_yaml_binary(self, node):
281 + try:
282 + value = self.construct_scalar(node).encode('ascii')
283 + except UnicodeEncodeError as exc:
284 + raise ConstructorError(None, None,
285 + "failed to convert base64 data into ascii: %s" % exc,
286 + node.start_mark)
287 + try:
288 + if hasattr(base64, 'decodebytes'):
289 + return base64.decodebytes(value)
290 + else:
291 + return base64.decodestring(value)
292 + except binascii.Error as exc:
293 + raise ConstructorError(None, None,
294 + "failed to decode base64 data: %s" % exc, node.start_mark)
295 +
296 + timestamp_regexp = re.compile(
297 + r'''^(?P<year>[0-9][0-9][0-9][0-9])
298 + -(?P<month>[0-9][0-9]?)
299 + -(?P<day>[0-9][0-9]?)
300 + (?:(?:[Tt]|[ \t]+)
301 + (?P<hour>[0-9][0-9]?)
302 + :(?P<minute>[0-9][0-9])
303 + :(?P<second>[0-9][0-9])
304 + (?:\.(?P<fraction>[0-9]*))?
305 + (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
306 + (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
307 +
308 + def construct_yaml_timestamp(self, node):
309 + value = self.construct_scalar(node)
310 + match = self.timestamp_regexp.match(node.value)
311 + values = match.groupdict()
312 + year = int(values['year'])
313 + month = int(values['month'])
314 + day = int(values['day'])
315 + if not values['hour']:
316 + return datetime.date(year, month, day)
317 + hour = int(values['hour'])
318 + minute = int(values['minute'])
319 + second = int(values['second'])
320 + fraction = 0
321 + if values['fraction']:
322 + fraction = values['fraction'][:6]
323 + while len(fraction) < 6:
324 + fraction += '0'
325 + fraction = int(fraction)
326 + delta = None
327 + if values['tz_sign']:
328 + tz_hour = int(values['tz_hour'])
329 + tz_minute = int(values['tz_minute'] or 0)
330 + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
331 + if values['tz_sign'] == '-':
332 + delta = -delta
333 + data = datetime.datetime(year, month, day, hour, minute, second, fraction)
334 + if delta:
335 + data -= delta
336 + return data
337 +
338 + def construct_yaml_omap(self, node):
339 + # Note: we do not check for duplicate keys, because it's too
340 + # CPU-expensive.
341 + omap = []
342 + yield omap
343 + if not isinstance(node, SequenceNode):
344 + raise ConstructorError("while constructing an ordered map", node.start_mark,
345 + "expected a sequence, but found %s" % node.id, node.start_mark)
346 + for subnode in node.value:
347 + if not isinstance(subnode, MappingNode):
348 + raise ConstructorError("while constructing an ordered map", node.start_mark,
349 + "expected a mapping of length 1, but found %s" % subnode.id,
350 + subnode.start_mark)
351 + if len(subnode.value) != 1:
352 + raise ConstructorError("while constructing an ordered map", node.start_mark,
353 + "expected a single mapping item, but found %d items" % len(subnode.value),
354 + subnode.start_mark)
355 + key_node, value_node = subnode.value[0]
356 + key = self.construct_object(key_node)
357 + value = self.construct_object(value_node)
358 + omap.append((key, value))
359 +
360 + def construct_yaml_pairs(self, node):
361 + # Note: the same code as `construct_yaml_omap`.
362 + pairs = []
363 + yield pairs
364 + if not isinstance(node, SequenceNode):
365 + raise ConstructorError("while constructing pairs", node.start_mark,
366 + "expected a sequence, but found %s" % node.id, node.start_mark)
367 + for subnode in node.value:
368 + if not isinstance(subnode, MappingNode):
369 + raise ConstructorError("while constructing pairs", node.start_mark,
370 + "expected a mapping of length 1, but found %s" % subnode.id,
371 + subnode.start_mark)
372 + if len(subnode.value) != 1:
373 + raise ConstructorError("while constructing pairs", node.start_mark,
374 + "expected a single mapping item, but found %d items" % len(subnode.value),
375 + subnode.start_mark)
376 + key_node, value_node = subnode.value[0]
377 + key = self.construct_object(key_node)
378 + value = self.construct_object(value_node)
379 + pairs.append((key, value))
380 +
381 + def construct_yaml_set(self, node):
382 + data = set()
383 + yield data
384 + value = self.construct_mapping(node)
385 + data.update(value)
386 +
387 + def construct_yaml_str(self, node):
388 + return self.construct_scalar(node)
389 +
390 + def construct_yaml_seq(self, node):
391 + data = []
392 + yield data
393 + data.extend(self.construct_sequence(node))
394 +
395 + def construct_yaml_map(self, node):
396 + data = {}
397 + yield data
398 + value = self.construct_mapping(node)
399 + data.update(value)
400 +
401 + def construct_yaml_object(self, node, cls):
402 + data = cls.__new__(cls)
403 + yield data
404 + if hasattr(data, '__setstate__'):
405 + state = self.construct_mapping(node, deep=True)
406 + data.__setstate__(state)
407 + else:
408 + state = self.construct_mapping(node)
409 + data.__dict__.update(state)
410 +
411 + def construct_undefined(self, node):
412 + raise ConstructorError(None, None,
413 + "could not determine a constructor for the tag %r" % node.tag,
414 + node.start_mark)
415 +
416 +SafeConstructor.add_constructor(
417 + 'tag:yaml.org,2002:null',
418 + SafeConstructor.construct_yaml_null)
419 +
420 +SafeConstructor.add_constructor(
421 + 'tag:yaml.org,2002:bool',
422 + SafeConstructor.construct_yaml_bool)
423 +
424 +SafeConstructor.add_constructor(
425 + 'tag:yaml.org,2002:int',
426 + SafeConstructor.construct_yaml_int)
427 +
428 +SafeConstructor.add_constructor(
429 + 'tag:yaml.org,2002:float',
430 + SafeConstructor.construct_yaml_float)
431 +
432 +SafeConstructor.add_constructor(
433 + 'tag:yaml.org,2002:binary',
434 + SafeConstructor.construct_yaml_binary)
435 +
436 +SafeConstructor.add_constructor(
437 + 'tag:yaml.org,2002:timestamp',
438 + SafeConstructor.construct_yaml_timestamp)
439 +
440 +SafeConstructor.add_constructor(
441 + 'tag:yaml.org,2002:omap',
442 + SafeConstructor.construct_yaml_omap)
443 +
444 +SafeConstructor.add_constructor(
445 + 'tag:yaml.org,2002:pairs',
446 + SafeConstructor.construct_yaml_pairs)
447 +
448 +SafeConstructor.add_constructor(
449 + 'tag:yaml.org,2002:set',
450 + SafeConstructor.construct_yaml_set)
451 +
452 +SafeConstructor.add_constructor(
453 + 'tag:yaml.org,2002:str',
454 + SafeConstructor.construct_yaml_str)
455 +
456 +SafeConstructor.add_constructor(
457 + 'tag:yaml.org,2002:seq',
458 + SafeConstructor.construct_yaml_seq)
459 +
460 +SafeConstructor.add_constructor(
461 + 'tag:yaml.org,2002:map',
462 + SafeConstructor.construct_yaml_map)
463 +
464 +SafeConstructor.add_constructor(None,
465 + SafeConstructor.construct_undefined)
466 +
467 +class Constructor(SafeConstructor):
468 +
469 + def construct_python_str(self, node):
470 + return self.construct_scalar(node)
471 +
472 + def construct_python_unicode(self, node):
473 + return self.construct_scalar(node)
474 +
475 + def construct_python_bytes(self, node):
476 + try:
477 + value = self.construct_scalar(node).encode('ascii')
478 + except UnicodeEncodeError as exc:
479 + raise ConstructorError(None, None,
480 + "failed to convert base64 data into ascii: %s" % exc,
481 + node.start_mark)
482 + try:
483 + if hasattr(base64, 'decodebytes'):
484 + return base64.decodebytes(value)
485 + else:
486 + return base64.decodestring(value)
487 + except binascii.Error as exc:
488 + raise ConstructorError(None, None,
489 + "failed to decode base64 data: %s" % exc, node.start_mark)
490 +
491 + def construct_python_long(self, node):
492 + return self.construct_yaml_int(node)
493 +
494 + def construct_python_complex(self, node):
495 + return complex(self.construct_scalar(node))
496 +
497 + def construct_python_tuple(self, node):
498 + return tuple(self.construct_sequence(node))
499 +
500 + def find_python_module(self, name, mark):
501 + if not name:
502 + raise ConstructorError("while constructing a Python module", mark,
503 + "expected non-empty name appended to the tag", mark)
504 + try:
505 + __import__(name)
506 + except ImportError as exc:
507 + raise ConstructorError("while constructing a Python module", mark,
508 + "cannot find module %r (%s)" % (name, exc), mark)
509 + return sys.modules[name]
510 +
511 + def find_python_name(self, name, mark):
512 + if not name:
513 + raise ConstructorError("while constructing a Python object", mark,
514 + "expected non-empty name appended to the tag", mark)
515 + if '.' in name:
516 + module_name, object_name = name.rsplit('.', 1)
517 + else:
518 + module_name = 'builtins'
519 + object_name = name
520 + try:
521 + __import__(module_name)
522 + except ImportError as exc:
523 + raise ConstructorError("while constructing a Python object", mark,
524 + "cannot find module %r (%s)" % (module_name, exc), mark)
525 + module = sys.modules[module_name]
526 + if not hasattr(module, object_name):
527 + raise ConstructorError("while constructing a Python object", mark,
528 + "cannot find %r in the module %r"
529 + % (object_name, module.__name__), mark)
530 + return getattr(module, object_name)
531 +
532 + def construct_python_name(self, suffix, node):
533 + value = self.construct_scalar(node)
534 + if value:
535 + raise ConstructorError("while constructing a Python name", node.start_mark,
536 + "expected the empty value, but found %r" % value, node.start_mark)
537 + return self.find_python_name(suffix, node.start_mark)
538 +
539 + def construct_python_module(self, suffix, node):
540 + value = self.construct_scalar(node)
541 + if value:
542 + raise ConstructorError("while constructing a Python module", node.start_mark,
543 + "expected the empty value, but found %r" % value, node.start_mark)
544 + return self.find_python_module(suffix, node.start_mark)
545 +
546 + def make_python_instance(self, suffix, node,
547 + args=None, kwds=None, newobj=False):
548 + if not args:
549 + args = []
550 + if not kwds:
551 + kwds = {}
552 + cls = self.find_python_name(suffix, node.start_mark)
553 + if newobj and isinstance(cls, type):
554 + return cls.__new__(cls, *args, **kwds)
555 + else:
556 + return cls(*args, **kwds)
557 +
558 + def set_python_instance_state(self, instance, state):
559 + if hasattr(instance, '__setstate__'):
560 + instance.__setstate__(state)
561 + else:
562 + slotstate = {}
563 + if isinstance(state, tuple) and len(state) == 2:
564 + state, slotstate = state
565 + if hasattr(instance, '__dict__'):
566 + instance.__dict__.update(state)
567 + elif state:
568 + slotstate.update(state)
569 + for key, value in slotstate.items():
570 + setattr(object, key, value)
571 +
572 + def construct_python_object(self, suffix, node):
573 + # Format:
574 + # !!python/object:module.name { ... state ... }
575 + instance = self.make_python_instance(suffix, node, newobj=True)
576 + yield instance
577 + deep = hasattr(instance, '__setstate__')
578 + state = self.construct_mapping(node, deep=deep)
579 + self.set_python_instance_state(instance, state)
580 +
581 + def construct_python_object_apply(self, suffix, node, newobj=False):
582 + # Format:
583 + # !!python/object/apply # (or !!python/object/new)
584 + # args: [ ... arguments ... ]
585 + # kwds: { ... keywords ... }
586 + # state: ... state ...
587 + # listitems: [ ... listitems ... ]
588 + # dictitems: { ... dictitems ... }
589 + # or short format:
590 + # !!python/object/apply [ ... arguments ... ]
591 + # The difference between !!python/object/apply and !!python/object/new
592 + # is how an object is created, check make_python_instance for details.
593 + if isinstance(node, SequenceNode):
594 + args = self.construct_sequence(node, deep=True)
595 + kwds = {}
596 + state = {}
597 + listitems = []
598 + dictitems = {}
599 + else:
600 + value = self.construct_mapping(node, deep=True)
601 + args = value.get('args', [])
602 + kwds = value.get('kwds', {})
603 + state = value.get('state', {})
604 + listitems = value.get('listitems', [])
605 + dictitems = value.get('dictitems', {})
606 + instance = self.make_python_instance(suffix, node, args, kwds, newobj)
607 + if state:
608 + self.set_python_instance_state(instance, state)
609 + if listitems:
610 + instance.extend(listitems)
611 + if dictitems:
612 + for key in dictitems:
613 + instance[key] = dictitems[key]
614 + return instance
615 +
616 + def construct_python_object_new(self, suffix, node):
617 + return self.construct_python_object_apply(suffix, node, newobj=True)
618 +
619 +Constructor.add_constructor(
620 + 'tag:yaml.org,2002:python/none',
621 + Constructor.construct_yaml_null)
622 +
623 +Constructor.add_constructor(
624 + 'tag:yaml.org,2002:python/bool',
625 + Constructor.construct_yaml_bool)
626 +
627 +Constructor.add_constructor(
628 + 'tag:yaml.org,2002:python/str',
629 + Constructor.construct_python_str)
630 +
631 +Constructor.add_constructor(
632 + 'tag:yaml.org,2002:python/unicode',
633 + Constructor.construct_python_unicode)
634 +
635 +Constructor.add_constructor(
636 + 'tag:yaml.org,2002:python/bytes',
637 + Constructor.construct_python_bytes)
638 +
639 +Constructor.add_constructor(
640 + 'tag:yaml.org,2002:python/int',
641 + Constructor.construct_yaml_int)
642 +
643 +Constructor.add_constructor(
644 + 'tag:yaml.org,2002:python/long',
645 + Constructor.construct_python_long)
646 +
647 +Constructor.add_constructor(
648 + 'tag:yaml.org,2002:python/float',
649 + Constructor.construct_yaml_float)
650 +
651 +Constructor.add_constructor(
652 + 'tag:yaml.org,2002:python/complex',
653 + Constructor.construct_python_complex)
654 +
655 +Constructor.add_constructor(
656 + 'tag:yaml.org,2002:python/list',
657 + Constructor.construct_yaml_seq)
658 +
659 +Constructor.add_constructor(
660 + 'tag:yaml.org,2002:python/tuple',
661 + Constructor.construct_python_tuple)
662 +
663 +Constructor.add_constructor(
664 + 'tag:yaml.org,2002:python/dict',
665 + Constructor.construct_yaml_map)
666 +
667 +Constructor.add_multi_constructor(
668 + 'tag:yaml.org,2002:python/name:',
669 + Constructor.construct_python_name)
670 +
671 +Constructor.add_multi_constructor(
672 + 'tag:yaml.org,2002:python/module:',
673 + Constructor.construct_python_module)
674 +
675 +Constructor.add_multi_constructor(
676 + 'tag:yaml.org,2002:python/object:',
677 + Constructor.construct_python_object)
678 +
679 +Constructor.add_multi_constructor(
680 + 'tag:yaml.org,2002:python/object/apply:',
681 + Constructor.construct_python_object_apply)
682 +
683 +Constructor.add_multi_constructor(
684 + 'tag:yaml.org,2002:python/object/new:',
685 + Constructor.construct_python_object_new)
686 +
python.d/python_modules/pyyaml3/cyaml.py new
+85
@@ -0,0 +1,85 @@
1 +
2 +__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
3 + 'CBaseDumper', 'CSafeDumper', 'CDumper']
4 +
5 +from _yaml import CParser, CEmitter
6 +
7 +from .constructor import *
8 +
9 +from .serializer import *
10 +from .representer import *
11 +
12 +from .resolver import *
13 +
14 +class CBaseLoader(CParser, BaseConstructor, BaseResolver):
15 +
16 + def __init__(self, stream):
17 + CParser.__init__(self, stream)
18 + BaseConstructor.__init__(self)
19 + BaseResolver.__init__(self)
20 +
21 +class CSafeLoader(CParser, SafeConstructor, Resolver):
22 +
23 + def __init__(self, stream):
24 + CParser.__init__(self, stream)
25 + SafeConstructor.__init__(self)
26 + Resolver.__init__(self)
27 +
28 +class CLoader(CParser, Constructor, Resolver):
29 +
30 + def __init__(self, stream):
31 + CParser.__init__(self, stream)
32 + Constructor.__init__(self)
33 + Resolver.__init__(self)
34 +
35 +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
36 +
37 + def __init__(self, stream,
38 + default_style=None, default_flow_style=None,
39 + canonical=None, indent=None, width=None,
40 + allow_unicode=None, line_break=None,
41 + encoding=None, explicit_start=None, explicit_end=None,
42 + version=None, tags=None):
43 + CEmitter.__init__(self, stream, canonical=canonical,
44 + indent=indent, width=width, encoding=encoding,
45 + allow_unicode=allow_unicode, line_break=line_break,
46 + explicit_start=explicit_start, explicit_end=explicit_end,
47 + version=version, tags=tags)
48 + Representer.__init__(self, default_style=default_style,
49 + default_flow_style=default_flow_style)
50 + Resolver.__init__(self)
51 +
52 +class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
53 +
54 + def __init__(self, stream,
55 + default_style=None, default_flow_style=None,
56 + canonical=None, indent=None, width=None,
57 + allow_unicode=None, line_break=None,
58 + encoding=None, explicit_start=None, explicit_end=None,
59 + version=None, tags=None):
60 + CEmitter.__init__(self, stream, canonical=canonical,
61 + indent=indent, width=width, encoding=encoding,
62 + allow_unicode=allow_unicode, line_break=line_break,
63 + explicit_start=explicit_start, explicit_end=explicit_end,
64 + version=version, tags=tags)
65 + SafeRepresenter.__init__(self, default_style=default_style,
66 + default_flow_style=default_flow_style)
67 + Resolver.__init__(self)
68 +
69 +class CDumper(CEmitter, Serializer, Representer, Resolver):
70 +
71 + def __init__(self, stream,
72 + default_style=None, default_flow_style=None,
73 + canonical=None, indent=None, width=None,
74 + allow_unicode=None, line_break=None,
75 + encoding=None, explicit_start=None, explicit_end=None,
76 + version=None, tags=None):
77 + CEmitter.__init__(self, stream, canonical=canonical,
78 + indent=indent, width=width, encoding=encoding,
79 + allow_unicode=allow_unicode, line_break=line_break,
80 + explicit_start=explicit_start, explicit_end=explicit_end,
81 + version=version, tags=tags)
82 + Representer.__init__(self, default_style=default_style,
83 + default_flow_style=default_flow_style)
84 + Resolver.__init__(self)
85 +
python.d/python_modules/pyyaml3/dumper.py new
+62
@@ -0,0 +1,62 @@
1 +
2 +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper']
3 +
4 +from .emitter import *
5 +from .serializer import *
6 +from .representer import *
7 +from .resolver import *
8 +
9 +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
10 +
11 + def __init__(self, stream,
12 + default_style=None, default_flow_style=None,
13 + canonical=None, indent=None, width=None,
14 + allow_unicode=None, line_break=None,
15 + encoding=None, explicit_start=None, explicit_end=None,
16 + version=None, tags=None):
17 + Emitter.__init__(self, stream, canonical=canonical,
18 + indent=indent, width=width,
19 + allow_unicode=allow_unicode, line_break=line_break)
20 + Serializer.__init__(self, encoding=encoding,
21 + explicit_start=explicit_start, explicit_end=explicit_end,
22 + version=version, tags=tags)
23 + Representer.__init__(self, default_style=default_style,
24 + default_flow_style=default_flow_style)
25 + Resolver.__init__(self)
26 +
27 +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
28 +
29 + def __init__(self, stream,
30 + default_style=None, default_flow_style=None,
31 + canonical=None, indent=None, width=None,
32 + allow_unicode=None, line_break=None,
33 + encoding=None, explicit_start=None, explicit_end=None,
34 + version=None, tags=None):
35 + Emitter.__init__(self, stream, canonical=canonical,
36 + indent=indent, width=width,
37 + allow_unicode=allow_unicode, line_break=line_break)
38 + Serializer.__init__(self, encoding=encoding,
39 + explicit_start=explicit_start, explicit_end=explicit_end,
40 + version=version, tags=tags)
41 + SafeRepresenter.__init__(self, default_style=default_style,
42 + default_flow_style=default_flow_style)
43 + Resolver.__init__(self)
44 +
45 +class Dumper(Emitter, Serializer, Representer, Resolver):
46 +
47 + def __init__(self, stream,
48 + default_style=None, default_flow_style=None,
49 + canonical=None, indent=None, width=None,
50 + allow_unicode=None, line_break=None,
51 + encoding=None, explicit_start=None, explicit_end=None,
52 + version=None, tags=None):
53 + Emitter.__init__(self, stream, canonical=canonical,
54 + indent=indent, width=width,
55 + allow_unicode=allow_unicode, line_break=line_break)
56 + Serializer.__init__(self, encoding=encoding,
57 + explicit_start=explicit_start, explicit_end=explicit_end,
58 + version=version, tags=tags)
59 + Representer.__init__(self, default_style=default_style,
60 + default_flow_style=default_flow_style)
61 + Resolver.__init__(self)
62 +
python.d/python_modules/pyyaml3/emitter.py new
+1137
@@ -0,0 +1,1137 @@
1 +
2 +# Emitter expects events obeying the following grammar:
3 +# stream ::= STREAM-START document* STREAM-END
4 +# document ::= DOCUMENT-START node DOCUMENT-END
5 +# node ::= SCALAR | sequence | mapping
6 +# sequence ::= SEQUENCE-START node* SEQUENCE-END
7 +# mapping ::= MAPPING-START (node node)* MAPPING-END
8 +
9 +__all__ = ['Emitter', 'EmitterError']
10 +
11 +from .error import YAMLError
12 +from .events import *
13 +
14 +class EmitterError(YAMLError):
15 + pass
16 +
17 +class ScalarAnalysis:
18 + def __init__(self, scalar, empty, multiline,
19 + allow_flow_plain, allow_block_plain,
20 + allow_single_quoted, allow_double_quoted,
21 + allow_block):
22 + self.scalar = scalar
23 + self.empty = empty
24 + self.multiline = multiline
25 + self.allow_flow_plain = allow_flow_plain
26 + self.allow_block_plain = allow_block_plain
27 + self.allow_single_quoted = allow_single_quoted
28 + self.allow_double_quoted = allow_double_quoted
29 + self.allow_block = allow_block
30 +
31 +class Emitter:
32 +
33 + DEFAULT_TAG_PREFIXES = {
34 + '!' : '!',
35 + 'tag:yaml.org,2002:' : '!!',
36 + }
37 +
38 + def __init__(self, stream, canonical=None, indent=None, width=None,
39 + allow_unicode=None, line_break=None):
40 +
41 + # The stream should have the methods `write` and possibly `flush`.
42 + self.stream = stream
43 +
44 + # Encoding can be overriden by STREAM-START.
45 + self.encoding = None
46 +
47 + # Emitter is a state machine with a stack of states to handle nested
48 + # structures.
49 + self.states = []
50 + self.state = self.expect_stream_start
51 +
52 + # Current event and the event queue.
53 + self.events = []
54 + self.event = None
55 +
56 + # The current indentation level and the stack of previous indents.
57 + self.indents = []
58 + self.indent = None
59 +
60 + # Flow level.
61 + self.flow_level = 0
62 +
63 + # Contexts.
64 + self.root_context = False
65 + self.sequence_context = False
66 + self.mapping_context = False
67 + self.simple_key_context = False
68 +
69 + # Characteristics of the last emitted character:
70 + # - current position.
71 + # - is it a whitespace?
72 + # - is it an indention character
73 + # (indentation space, '-', '?', or ':')?
74 + self.line = 0
75 + self.column = 0
76 + self.whitespace = True
77 + self.indention = True
78 +
79 + # Whether the document requires an explicit document indicator
80 + self.open_ended = False
81 +
82 + # Formatting details.
83 + self.canonical = canonical
84 + self.allow_unicode = allow_unicode
85 + self.best_indent = 2
86 + if indent and 1 < indent < 10:
87 + self.best_indent = indent
88 + self.best_width = 80
89 + if width and width > self.best_indent*2:
90 + self.best_width = width
91 + self.best_line_break = '\n'
92 + if line_break in ['\r', '\n', '\r\n']:
93 + self.best_line_break = line_break
94 +
95 + # Tag prefixes.
96 + self.tag_prefixes = None
97 +
98 + # Prepared anchor and tag.
99 + self.prepared_anchor = None
100 + self.prepared_tag = None
101 +
102 + # Scalar analysis and style.
103 + self.analysis = None
104 + self.style = None
105 +
106 + def dispose(self):
107 + # Reset the state attributes (to clear self-references)
108 + self.states = []
109 + self.state = None
110 +
111 + def emit(self, event):
112 + self.events.append(event)
113 + while not self.need_more_events():
114 + self.event = self.events.pop(0)
115 + self.state()
116 + self.event = None
117 +
118 + # In some cases, we wait for a few next events before emitting.
119 +
120 + def need_more_events(self):
121 + if not self.events:
122 + return True
123 + event = self.events[0]
124 + if isinstance(event, DocumentStartEvent):
125 + return self.need_events(1)
126 + elif isinstance(event, SequenceStartEvent):
127 + return self.need_events(2)
128 + elif isinstance(event, MappingStartEvent):
129 + return self.need_events(3)
130 + else:
131 + return False
132 +
133 + def need_events(self, count):
134 + level = 0
135 + for event in self.events[1:]:
136 + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)):
137 + level += 1
138 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)):
139 + level -= 1
140 + elif isinstance(event, StreamEndEvent):
141 + level = -1
142 + if level < 0:
143 + return False
144 + return (len(self.events) < count+1)
145 +
146 + def increase_indent(self, flow=False, indentless=False):
147 + self.indents.append(self.indent)
148 + if self.indent is None:
149 + if flow:
150 + self.indent = self.best_indent
151 + else:
152 + self.indent = 0
153 + elif not indentless:
154 + self.indent += self.best_indent
155 +
156 + # States.
157 +
158 + # Stream handlers.
159 +
160 + def expect_stream_start(self):
161 + if isinstance(self.event, StreamStartEvent):
162 + if self.event.encoding and not hasattr(self.stream, 'encoding'):
163 + self.encoding = self.event.encoding
164 + self.write_stream_start()
165 + self.state = self.expect_first_document_start
166 + else:
167 + raise EmitterError("expected StreamStartEvent, but got %s"
168 + % self.event)
169 +
170 + def expect_nothing(self):
171 + raise EmitterError("expected nothing, but got %s" % self.event)
172 +
173 + # Document handlers.
174 +
175 + def expect_first_document_start(self):
176 + return self.expect_document_start(first=True)
177 +
178 + def expect_document_start(self, first=False):
179 + if isinstance(self.event, DocumentStartEvent):
180 + if (self.event.version or self.event.tags) and self.open_ended:
181 + self.write_indicator('...', True)
182 + self.write_indent()
183 + if self.event.version:
184 + version_text = self.prepare_version(self.event.version)
185 + self.write_version_directive(version_text)
186 + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
187 + if self.event.tags:
188 + handles = sorted(self.event.tags.keys())
189 + for handle in handles:
190 + prefix = self.event.tags[handle]
191 + self.tag_prefixes[prefix] = handle
192 + handle_text = self.prepare_tag_handle(handle)
193 + prefix_text = self.prepare_tag_prefix(prefix)
194 + self.write_tag_directive(handle_text, prefix_text)
195 + implicit = (first and not self.event.explicit and not self.canonical
196 + and not self.event.version and not self.event.tags
197 + and not self.check_empty_document())
198 + if not implicit:
199 + self.write_indent()
200 + self.write_indicator('---', True)
201 + if self.canonical:
202 + self.write_indent()
203 + self.state = self.expect_document_root
204 + elif isinstance(self.event, StreamEndEvent):
205 + if self.open_ended:
206 + self.write_indicator('...', True)
207 + self.write_indent()
208 + self.write_stream_end()
209 + self.state = self.expect_nothing
210 + else:
211 + raise EmitterError("expected DocumentStartEvent, but got %s"
212 + % self.event)
213 +
214 + def expect_document_end(self):
215 + if isinstance(self.event, DocumentEndEvent):
216 + self.write_indent()
217 + if self.event.explicit:
218 + self.write_indicator('...', True)
219 + self.write_indent()
220 + self.flush_stream()
221 + self.state = self.expect_document_start
222 + else:
223 + raise EmitterError("expected DocumentEndEvent, but got %s"
224 + % self.event)
225 +
226 + def expect_document_root(self):
227 + self.states.append(self.expect_document_end)
228 + self.expect_node(root=True)
229 +
230 + # Node handlers.
231 +
232 + def expect_node(self, root=False, sequence=False, mapping=False,
233 + simple_key=False):
234 + self.root_context = root
235 + self.sequence_context = sequence
236 + self.mapping_context = mapping
237 + self.simple_key_context = simple_key
238 + if isinstance(self.event, AliasEvent):
239 + self.expect_alias()
240 + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)):
241 + self.process_anchor('&')
242 + self.process_tag()
243 + if isinstance(self.event, ScalarEvent):
244 + self.expect_scalar()
245 + elif isinstance(self.event, SequenceStartEvent):
246 + if self.flow_level or self.canonical or self.event.flow_style \
247 + or self.check_empty_sequence():
248 + self.expect_flow_sequence()
249 + else:
250 + self.expect_block_sequence()
251 + elif isinstance(self.event, MappingStartEvent):
252 + if self.flow_level or self.canonical or self.event.flow_style \
253 + or self.check_empty_mapping():
254 + self.expect_flow_mapping()
255 + else:
256 + self.expect_block_mapping()
257 + else:
258 + raise EmitterError("expected NodeEvent, but got %s" % self.event)
259 +
260 + def expect_alias(self):
261 + if self.event.anchor is None:
262 + raise EmitterError("anchor is not specified for alias")
263 + self.process_anchor('*')
264 + self.state = self.states.pop()
265 +
266 + def expect_scalar(self):
267 + self.increase_indent(flow=True)
268 + self.process_scalar()
269 + self.indent = self.indents.pop()
270 + self.state = self.states.pop()
271 +
272 + # Flow sequence handlers.
273 +
274 + def expect_flow_sequence(self):
275 + self.write_indicator('[', True, whitespace=True)
276 + self.flow_level += 1
277 + self.increase_indent(flow=True)
278 + self.state = self.expect_first_flow_sequence_item
279 +
280 + def expect_first_flow_sequence_item(self):
281 + if isinstance(self.event, SequenceEndEvent):
282 + self.indent = self.indents.pop()
283 + self.flow_level -= 1
284 + self.write_indicator(']', False)
285 + self.state = self.states.pop()
286 + else:
287 + if self.canonical or self.column > self.best_width:
288 + self.write_indent()
289 + self.states.append(self.expect_flow_sequence_item)
290 + self.expect_node(sequence=True)
291 +
292 + def expect_flow_sequence_item(self):
293 + if isinstance(self.event, SequenceEndEvent):
294 + self.indent = self.indents.pop()
295 + self.flow_level -= 1
296 + if self.canonical:
297 + self.write_indicator(',', False)
298 + self.write_indent()
299 + self.write_indicator(']', False)
300 + self.state = self.states.pop()
301 + else:
302 + self.write_indicator(',', False)
303 + if self.canonical or self.column > self.best_width:
304 + self.write_indent()
305 + self.states.append(self.expect_flow_sequence_item)
306 + self.expect_node(sequence=True)
307 +
308 + # Flow mapping handlers.
309 +
310 + def expect_flow_mapping(self):
311 + self.write_indicator('{', True, whitespace=True)
312 + self.flow_level += 1
313 + self.increase_indent(flow=True)
314 + self.state = self.expect_first_flow_mapping_key
315 +
316 + def expect_first_flow_mapping_key(self):
317 + if isinstance(self.event, MappingEndEvent):
318 + self.indent = self.indents.pop()
319 + self.flow_level -= 1
320 + self.write_indicator('}', False)
321 + self.state = self.states.pop()
322 + else:
323 + if self.canonical or self.column > self.best_width:
324 + self.write_indent()
325 + if not self.canonical and self.check_simple_key():
326 + self.states.append(self.expect_flow_mapping_simple_value)
327 + self.expect_node(mapping=True, simple_key=True)
328 + else:
329 + self.write_indicator('?', True)
330 + self.states.append(self.expect_flow_mapping_value)
331 + self.expect_node(mapping=True)
332 +
333 + def expect_flow_mapping_key(self):
334 + if isinstance(self.event, MappingEndEvent):
335 + self.indent = self.indents.pop()
336 + self.flow_level -= 1
337 + if self.canonical:
338 + self.write_indicator(',', False)
339 + self.write_indent()
340 + self.write_indicator('}', False)
341 + self.state = self.states.pop()
342 + else:
343 + self.write_indicator(',', False)
344 + if self.canonical or self.column > self.best_width:
345 + self.write_indent()
346 + if not self.canonical and self.check_simple_key():
347 + self.states.append(self.expect_flow_mapping_simple_value)
348 + self.expect_node(mapping=True, simple_key=True)
349 + else:
350 + self.write_indicator('?', True)
351 + self.states.append(self.expect_flow_mapping_value)
352 + self.expect_node(mapping=True)
353 +
354 + def expect_flow_mapping_simple_value(self):
355 + self.write_indicator(':', False)
356 + self.states.append(self.expect_flow_mapping_key)
357 + self.expect_node(mapping=True)
358 +
359 + def expect_flow_mapping_value(self):
360 + if self.canonical or self.column > self.best_width:
361 + self.write_indent()
362 + self.write_indicator(':', True)
363 + self.states.append(self.expect_flow_mapping_key)
364 + self.expect_node(mapping=True)
365 +
366 + # Block sequence handlers.
367 +
368 + def expect_block_sequence(self):
369 + indentless = (self.mapping_context and not self.indention)
370 + self.increase_indent(flow=False, indentless=indentless)
371 + self.state = self.expect_first_block_sequence_item
372 +
373 + def expect_first_block_sequence_item(self):
374 + return self.expect_block_sequence_item(first=True)
375 +
376 + def expect_block_sequence_item(self, first=False):
377 + if not first and isinstance(self.event, SequenceEndEvent):
378 + self.indent = self.indents.pop()
379 + self.state = self.states.pop()
380 + else:
381 + self.write_indent()
382 + self.write_indicator('-', True, indention=True)
383 + self.states.append(self.expect_block_sequence_item)
384 + self.expect_node(sequence=True)
385 +
386 + # Block mapping handlers.
387 +
388 + def expect_block_mapping(self):
389 + self.increase_indent(flow=False)
390 + self.state = self.expect_first_block_mapping_key
391 +
392 + def expect_first_block_mapping_key(self):
393 + return self.expect_block_mapping_key(first=True)
394 +
395 + def expect_block_mapping_key(self, first=False):
396 + if not first and isinstance(self.event, MappingEndEvent):
397 + self.indent = self.indents.pop()
398 + self.state = self.states.pop()
399 + else:
400 + self.write_indent()
401 + if self.check_simple_key():
402 + self.states.append(self.expect_block_mapping_simple_value)
403 + self.expect_node(mapping=True, simple_key=True)
404 + else:
405 + self.write_indicator('?', True, indention=True)
406 + self.states.append(self.expect_block_mapping_value)
407 + self.expect_node(mapping=True)
408 +
409 + def expect_block_mapping_simple_value(self):
410 + self.write_indicator(':', False)
411 + self.states.append(self.expect_block_mapping_key)
412 + self.expect_node(mapping=True)
413 +
414 + def expect_block_mapping_value(self):
415 + self.write_indent()
416 + self.write_indicator(':', True, indention=True)
417 + self.states.append(self.expect_block_mapping_key)
418 + self.expect_node(mapping=True)
419 +
420 + # Checkers.
421 +
422 + def check_empty_sequence(self):
423 + return (isinstance(self.event, SequenceStartEvent) and self.events
424 + and isinstance(self.events[0], SequenceEndEvent))
425 +
426 + def check_empty_mapping(self):
427 + return (isinstance(self.event, MappingStartEvent) and self.events
428 + and isinstance(self.events[0], MappingEndEvent))
429 +
430 + def check_empty_document(self):
431 + if not isinstance(self.event, DocumentStartEvent) or not self.events:
432 + return False
433 + event = self.events[0]
434 + return (isinstance(event, ScalarEvent) and event.anchor is None
435 + and event.tag is None and event.implicit and event.value == '')
436 +
437 + def check_simple_key(self):
438 + length = 0
439 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
440 + if self.prepared_anchor is None:
441 + self.prepared_anchor = self.prepare_anchor(self.event.anchor)
442 + length += len(self.prepared_anchor)
443 + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \
444 + and self.event.tag is not None:
445 + if self.prepared_tag is None:
446 + self.prepared_tag = self.prepare_tag(self.event.tag)
447 + length += len(self.prepared_tag)
448 + if isinstance(self.event, ScalarEvent):
449 + if self.analysis is None:
450 + self.analysis = self.analyze_scalar(self.event.value)
451 + length += len(self.analysis.scalar)
452 + return (length < 128 and (isinstance(self.event, AliasEvent)
453 + or (isinstance(self.event, ScalarEvent)
454 + and not self.analysis.empty and not self.analysis.multiline)
455 + or self.check_empty_sequence() or self.check_empty_mapping()))
456 +
457 + # Anchor, Tag, and Scalar processors.
458 +
459 + def process_anchor(self, indicator):
460 + if self.event.anchor is None:
461 + self.prepared_anchor = None
462 + return
463 + if self.prepared_anchor is None:
464 + self.prepared_anchor = self.prepare_anchor(self.event.anchor)
465 + if self.prepared_anchor:
466 + self.write_indicator(indicator+self.prepared_anchor, True)
467 + self.prepared_anchor = None
468 +
469 + def process_tag(self):
470 + tag = self.event.tag
471 + if isinstance(self.event, ScalarEvent):
472 + if self.style is None:
473 + self.style = self.choose_scalar_style()
474 + if ((not self.canonical or tag is None) and
475 + ((self.style == '' and self.event.implicit[0])
476 + or (self.style != '' and self.event.implicit[1]))):
477 + self.prepared_tag = None
478 + return
479 + if self.event.implicit[0] and tag is None:
480 + tag = '!'
481 + self.prepared_tag = None
482 + else:
483 + if (not self.canonical or tag is None) and self.event.implicit:
484 + self.prepared_tag = None
485 + return
486 + if tag is None:
487 + raise EmitterError("tag is not specified")
488 + if self.prepared_tag is None:
489 + self.prepared_tag = self.prepare_tag(tag)
490 + if self.prepared_tag:
491 + self.write_indicator(self.prepared_tag, True)
492 + self.prepared_tag = None
493 +
494 + def choose_scalar_style(self):
495 + if self.analysis is None:
496 + self.analysis = self.analyze_scalar(self.event.value)
497 + if self.event.style == '"' or self.canonical:
498 + return '"'
499 + if not self.event.style and self.event.implicit[0]:
500 + if (not (self.simple_key_context and
501 + (self.analysis.empty or self.analysis.multiline))
502 + and (self.flow_level and self.analysis.allow_flow_plain
503 + or (not self.flow_level and self.analysis.allow_block_plain))):
504 + return ''
505 + if self.event.style and self.event.style in '|>':
506 + if (not self.flow_level and not self.simple_key_context
507 + and self.analysis.allow_block):
508 + return self.event.style
509 + if not self.event.style or self.event.style == '\'':
510 + if (self.analysis.allow_single_quoted and
511 + not (self.simple_key_context and self.analysis.multiline)):
512 + return '\''
513 + return '"'
514 +
515 + def process_scalar(self):
516 + if self.analysis is None:
517 + self.analysis = self.analyze_scalar(self.event.value)
518 + if self.style is None:
519 + self.style = self.choose_scalar_style()
520 + split = (not self.simple_key_context)
521 + #if self.analysis.multiline and split \
522 + # and (not self.style or self.style in '\'\"'):
523 + # self.write_indent()
524 + if self.style == '"':
525 + self.write_double_quoted(self.analysis.scalar, split)
526 + elif self.style == '\'':
527 + self.write_single_quoted(self.analysis.scalar, split)
528 + elif self.style == '>':
529 + self.write_folded(self.analysis.scalar)
530 + elif self.style == '|':
531 + self.write_literal(self.analysis.scalar)
532 + else:
533 + self.write_plain(self.analysis.scalar, split)
534 + self.analysis = None
535 + self.style = None
536 +
537 + # Analyzers.
538 +
539 + def prepare_version(self, version):
540 + major, minor = version
541 + if major != 1:
542 + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor))
543 + return '%d.%d' % (major, minor)
544 +
545 + def prepare_tag_handle(self, handle):
546 + if not handle:
547 + raise EmitterError("tag handle must not be empty")
548 + if handle[0] != '!' or handle[-1] != '!':
549 + raise EmitterError("tag handle must start and end with '!': %r" % handle)
550 + for ch in handle[1:-1]:
551 + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
552 + or ch in '-_'):
553 + raise EmitterError("invalid character %r in the tag handle: %r"
554 + % (ch, handle))
555 + return handle
556 +
557 + def prepare_tag_prefix(self, prefix):
558 + if not prefix:
559 + raise EmitterError("tag prefix must not be empty")
560 + chunks = []
561 + start = end = 0
562 + if prefix[0] == '!':
563 + end = 1
564 + while end < len(prefix):
565 + ch = prefix[end]
566 + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
567 + or ch in '-;/?!:@&=+$,_.~*\'()[]':
568 + end += 1
569 + else:
570 + if start < end:
571 + chunks.append(prefix[start:end])
572 + start = end = end+1
573 + data = ch.encode('utf-8')
574 + for ch in data:
575 + chunks.append('%%%02X' % ord(ch))
576 + if start < end:
577 + chunks.append(prefix[start:end])
578 + return ''.join(chunks)
579 +
580 + def prepare_tag(self, tag):
581 + if not tag:
582 + raise EmitterError("tag must not be empty")
583 + if tag == '!':
584 + return tag
585 + handle = None
586 + suffix = tag
587 + prefixes = sorted(self.tag_prefixes.keys())
588 + for prefix in prefixes:
589 + if tag.startswith(prefix) \
590 + and (prefix == '!' or len(prefix) < len(tag)):
591 + handle = self.tag_prefixes[prefix]
592 + suffix = tag[len(prefix):]
593 + chunks = []
594 + start = end = 0
595 + while end < len(suffix):
596 + ch = suffix[end]
597 + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
598 + or ch in '-;/?:@&=+$,_.~*\'()[]' \
599 + or (ch == '!' and handle != '!'):
600 + end += 1
601 + else:
602 + if start < end:
603 + chunks.append(suffix[start:end])
604 + start = end = end+1
605 + data = ch.encode('utf-8')
606 + for ch in data:
607 + chunks.append('%%%02X' % ord(ch))
608 + if start < end:
609 + chunks.append(suffix[start:end])
610 + suffix_text = ''.join(chunks)
611 + if handle:
612 + return '%s%s' % (handle, suffix_text)
613 + else:
614 + return '!<%s>' % suffix_text
615 +
616 + def prepare_anchor(self, anchor):
617 + if not anchor:
618 + raise EmitterError("anchor must not be empty")
619 + for ch in anchor:
620 + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
621 + or ch in '-_'):
622 + raise EmitterError("invalid character %r in the anchor: %r"
623 + % (ch, anchor))
624 + return anchor
625 +
626 + def analyze_scalar(self, scalar):
627 +
628 + # Empty scalar is a special case.
629 + if not scalar:
630 + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
631 + allow_flow_plain=False, allow_block_plain=True,
632 + allow_single_quoted=True, allow_double_quoted=True,
633 + allow_block=False)
634 +
635 + # Indicators and special characters.
636 + block_indicators = False
637 + flow_indicators = False
638 + line_breaks = False
639 + special_characters = False
640 +
641 + # Important whitespace combinations.
642 + leading_space = False
643 + leading_break = False
644 + trailing_space = False
645 + trailing_break = False
646 + break_space = False
647 + space_break = False
648 +
649 + # Check document indicators.
650 + if scalar.startswith('---') or scalar.startswith('...'):
651 + block_indicators = True
652 + flow_indicators = True
653 +
654 + # First character or preceded by a whitespace.
655 + preceeded_by_whitespace = True
656 +
657 + # Last character or followed by a whitespace.
658 + followed_by_whitespace = (len(scalar) == 1 or
659 + scalar[1] in '\0 \t\r\n\x85\u2028\u2029')
660 +
661 + # The previous character is a space.
662 + previous_space = False
663 +
664 + # The previous character is a break.
665 + previous_break = False
666 +
667 + index = 0
668 + while index < len(scalar):
669 + ch = scalar[index]
670 +
671 + # Check for indicators.
672 + if index == 0:
673 + # Leading indicators are special characters.
674 + if ch in '#,[]{}&*!|>\'\"%@`':
675 + flow_indicators = True
676 + block_indicators = True
677 + if ch in '?:':
678 + flow_indicators = True
679 + if followed_by_whitespace:
680 + block_indicators = True
681 + if ch == '-' and followed_by_whitespace:
682 + flow_indicators = True
683 + block_indicators = True
684 + else:
685 + # Some indicators cannot appear within a scalar as well.
686 + if ch in ',?[]{}':
687 + flow_indicators = True
688 + if ch == ':':
689 + flow_indicators = True
690 + if followed_by_whitespace:
691 + block_indicators = True
692 + if ch == '#' and preceeded_by_whitespace:
693 + flow_indicators = True
694 + block_indicators = True
695 +
696 + # Check for line breaks, special, and unicode characters.
697 + if ch in '\n\x85\u2028\u2029':
698 + line_breaks = True
699 + if not (ch == '\n' or '\x20' <= ch <= '\x7E'):
700 + if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF'
701 + or '\uE000' <= ch <= '\uFFFD') and ch != '\uFEFF':
702 + unicode_characters = True
703 + if not self.allow_unicode:
704 + special_characters = True
705 + else:
706 + special_characters = True
707 +
708 + # Detect important whitespace combinations.
709 + if ch == ' ':
710 + if index == 0:
711 + leading_space = True
712 + if index == len(scalar)-1:
713 + trailing_space = True
714 + if previous_break:
715 + break_space = True
716 + previous_space = True
717 + previous_break = False
718 + elif ch in '\n\x85\u2028\u2029':
719 + if index == 0:
720 + leading_break = True
721 + if index == len(scalar)-1:
722 + trailing_break = True
723 + if previous_space:
724 + space_break = True
725 + previous_space = False
726 + previous_break = True
727 + else:
728 + previous_space = False
729 + previous_break = False
730 +
731 + # Prepare for the next character.
732 + index += 1
733 + preceeded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029')
734 + followed_by_whitespace = (index+1 >= len(scalar) or
735 + scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029')
736 +
737 + # Let's decide what styles are allowed.
738 + allow_flow_plain = True
739 + allow_block_plain = True
740 + allow_single_quoted = True
741 + allow_double_quoted = True
742 + allow_block = True
743 +
744 + # Leading and trailing whitespaces are bad for plain scalars.
745 + if (leading_space or leading_break
746 + or trailing_space or trailing_break):
747 + allow_flow_plain = allow_block_plain = False
748 +
749 + # We do not permit trailing spaces for block scalars.
750 + if trailing_space:
751 + allow_block = False
752 +
753 + # Spaces at the beginning of a new line are only acceptable for block
754 + # scalars.
755 + if break_space:
756 + allow_flow_plain = allow_block_plain = allow_single_quoted = False
757 +
758 + # Spaces followed by breaks, as well as special character are only
759 + # allowed for double quoted scalars.
760 + if space_break or special_characters:
761 + allow_flow_plain = allow_block_plain = \
762 + allow_single_quoted = allow_block = False
763 +
764 + # Although the plain scalar writer supports breaks, we never emit
765 + # multiline plain scalars.
766 + if line_breaks:
767 + allow_flow_plain = allow_block_plain = False
768 +
769 + # Flow indicators are forbidden for flow plain scalars.
770 + if flow_indicators:
771 + allow_flow_plain = False
772 +
773 + # Block indicators are forbidden for block plain scalars.
774 + if block_indicators:
775 + allow_block_plain = False
776 +
777 + return ScalarAnalysis(scalar=scalar,
778 + empty=False, multiline=line_breaks,
779 + allow_flow_plain=allow_flow_plain,
780 + allow_block_plain=allow_block_plain,
781 + allow_single_quoted=allow_single_quoted,
782 + allow_double_quoted=allow_double_quoted,
783 + allow_block=allow_block)
784 +
785 + # Writers.
786 +
787 + def flush_stream(self):
788 + if hasattr(self.stream, 'flush'):
789 + self.stream.flush()
790 +
791 + def write_stream_start(self):
792 + # Write BOM if needed.
793 + if self.encoding and self.encoding.startswith('utf-16'):
794 + self.stream.write('\uFEFF'.encode(self.encoding))
795 +
796 + def write_stream_end(self):
797 + self.flush_stream()
798 +
799 + def write_indicator(self, indicator, need_whitespace,
800 + whitespace=False, indention=False):
801 + if self.whitespace or not need_whitespace:
802 + data = indicator
803 + else:
804 + data = ' '+indicator
805 + self.whitespace = whitespace
806 + self.indention = self.indention and indention
807 + self.column += len(data)
808 + self.open_ended = False
809 + if self.encoding:
810 + data = data.encode(self.encoding)
811 + self.stream.write(data)
812 +
813 + def write_indent(self):
814 + indent = self.indent or 0
815 + if not self.indention or self.column > indent \
816 + or (self.column == indent and not self.whitespace):
817 + self.write_line_break()
818 + if self.column < indent:
819 + self.whitespace = True
820 + data = ' '*(indent-self.column)
821 + self.column = indent
822 + if self.encoding:
823 + data = data.encode(self.encoding)
824 + self.stream.write(data)
825 +
826 + def write_line_break(self, data=None):
827 + if data is None:
828 + data = self.best_line_break
829 + self.whitespace = True
830 + self.indention = True
831 + self.line += 1
832 + self.column = 0
833 + if self.encoding:
834 + data = data.encode(self.encoding)
835 + self.stream.write(data)
836 +
837 + def write_version_directive(self, version_text):
838 + data = '%%YAML %s' % version_text
839 + if self.encoding:
840 + data = data.encode(self.encoding)
841 + self.stream.write(data)
842 + self.write_line_break()
843 +
844 + def write_tag_directive(self, handle_text, prefix_text):
845 + data = '%%TAG %s %s' % (handle_text, prefix_text)
846 + if self.encoding:
847 + data = data.encode(self.encoding)
848 + self.stream.write(data)
849 + self.write_line_break()
850 +
851 + # Scalar streams.
852 +
853 + def write_single_quoted(self, text, split=True):
854 + self.write_indicator('\'', True)
855 + spaces = False
856 + breaks = False
857 + start = end = 0
858 + while end <= len(text):
859 + ch = None
860 + if end < len(text):
861 + ch = text[end]
862 + if spaces:
863 + if ch is None or ch != ' ':
864 + if start+1 == end and self.column > self.best_width and split \
865 + and start != 0 and end != len(text):
866 + self.write_indent()
867 + else:
868 + data = text[start:end]
869 + self.column += len(data)
870 + if self.encoding:
871 + data = data.encode(self.encoding)
872 + self.stream.write(data)
873 + start = end
874 + elif breaks:
875 + if ch is None or ch not in '\n\x85\u2028\u2029':
876 + if text[start] == '\n':
877 + self.write_line_break()
878 + for br in text[start:end]:
879 + if br == '\n':
880 + self.write_line_break()
881 + else:
882 + self.write_line_break(br)
883 + self.write_indent()
884 + start = end
885 + else:
886 + if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'':
887 + if start < end:
888 + data = text[start:end]
889 + self.column += len(data)
890 + if self.encoding:
891 + data = data.encode(self.encoding)
892 + self.stream.write(data)
893 + start = end
894 + if ch == '\'':
895 + data = '\'\''
896 + self.column += 2
897 + if self.encoding:
898 + data = data.encode(self.encoding)
899 + self.stream.write(data)
900 + start = end + 1
901 + if ch is not None:
902 + spaces = (ch == ' ')
903 + breaks = (ch in '\n\x85\u2028\u2029')
904 + end += 1
905 + self.write_indicator('\'', False)
906 +
907 + ESCAPE_REPLACEMENTS = {
908 + '\0': '0',
909 + '\x07': 'a',
910 + '\x08': 'b',
911 + '\x09': 't',
912 + '\x0A': 'n',
913 + '\x0B': 'v',
914 + '\x0C': 'f',
915 + '\x0D': 'r',
916 + '\x1B': 'e',
917 + '\"': '\"',
918 + '\\': '\\',
919 + '\x85': 'N',
920 + '\xA0': '_',
921 + '\u2028': 'L',
922 + '\u2029': 'P',
923 + }
924 +
925 + def write_double_quoted(self, text, split=True):
926 + self.write_indicator('"', True)
927 + start = end = 0
928 + while end <= len(text):
929 + ch = None
930 + if end < len(text):
931 + ch = text[end]
932 + if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \
933 + or not ('\x20' <= ch <= '\x7E'
934 + or (self.allow_unicode
935 + and ('\xA0' <= ch <= '\uD7FF'
936 + or '\uE000' <= ch <= '\uFFFD'))):
937 + if start < end:
938 + data = text[start:end]
939 + self.column += len(data)
940 + if self.encoding:
941 + data = data.encode(self.encoding)
942 + self.stream.write(data)
943 + start = end
944 + if ch is not None:
945 + if ch in self.ESCAPE_REPLACEMENTS:
946 + data = '\\'+self.ESCAPE_REPLACEMENTS[ch]
947 + elif ch <= '\xFF':
948 + data = '\\x%02X' % ord(ch)
949 + elif ch <= '\uFFFF':
950 + data = '\\u%04X' % ord(ch)
951 + else:
952 + data = '\\U%08X' % ord(ch)
953 + self.column += len(data)
954 + if self.encoding:
955 + data = data.encode(self.encoding)
956 + self.stream.write(data)
957 + start = end+1
958 + if 0 < end < len(text)-1 and (ch == ' ' or start >= end) \
959 + and self.column+(end-start) > self.best_width and split:
960 + data = text[start:end]+'\\'
961 + if start < end:
962 + start = end
963 + self.column += len(data)
964 + if self.encoding:
965 + data = data.encode(self.encoding)
966 + self.stream.write(data)
967 + self.write_indent()
968 + self.whitespace = False
969 + self.indention = False
970 + if text[start] == ' ':
971 + data = '\\'
972 + self.column += len(data)
973 + if self.encoding:
974 + data = data.encode(self.encoding)
975 + self.stream.write(data)
976 + end += 1
977 + self.write_indicator('"', False)
978 +
979 + def determine_block_hints(self, text):
980 + hints = ''
981 + if text:
982 + if text[0] in ' \n\x85\u2028\u2029':
983 + hints += str(self.best_indent)
984 + if text[-1] not in '\n\x85\u2028\u2029':
985 + hints += '-'
986 + elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029':
987 + hints += '+'
988 + return hints
989 +
990 + def write_folded(self, text):
991 + hints = self.determine_block_hints(text)
992 + self.write_indicator('>'+hints, True)
993 + if hints[-1:] == '+':
994 + self.open_ended = True
995 + self.write_line_break()
996 + leading_space = True
997 + spaces = False
998 + breaks = True
999 + start = end = 0
1000 + while end <= len(text):
1001 + ch = None
1002 + if end < len(text):
1003 + ch = text[end]
1004 + if breaks:
1005 + if ch is None or ch not in '\n\x85\u2028\u2029':
1006 + if not leading_space and ch is not None and ch != ' ' \
1007 + and text[start] == '\n':
1008 + self.write_line_break()
1009 + leading_space = (ch == ' ')
1010 + for br in text[start:end]:
1011 + if br == '\n':
1012 + self.write_line_break()
1013 + else:
1014 + self.write_line_break(br)
1015 + if ch is not None:
1016 + self.write_indent()
1017 + start = end
1018 + elif spaces:
1019 + if ch != ' ':
1020 + if start+1 == end and self.column > self.best_width:
1021 + self.write_indent()
1022 + else:
1023 + data = text[start:end]
1024 + self.column += len(data)
1025 + if self.encoding:
1026 + data = data.encode(self.encoding)
1027 + self.stream.write(data)
1028 + start = end
1029 + else:
1030 + if ch is None or ch in ' \n\x85\u2028\u2029':
1031 + data = text[start:end]
1032 + self.column += len(data)
1033 + if self.encoding:
1034 + data = data.encode(self.encoding)
1035 + self.stream.write(data)
1036 + if ch is None:
1037 + self.write_line_break()
1038 + start = end
1039 + if ch is not None:
1040 + breaks = (ch in '\n\x85\u2028\u2029')
1041 + spaces = (ch == ' ')
1042 + end += 1
1043 +
1044 + def write_literal(self, text):
1045 + hints = self.determine_block_hints(text)
1046 + self.write_indicator('|'+hints, True)
1047 + if hints[-1:] == '+':
1048 + self.open_ended = True
1049 + self.write_line_break()
1050 + breaks = True
1051 + start = end = 0
1052 + while end <= len(text):
1053 + ch = None
1054 + if end < len(text):
1055 + ch = text[end]
1056 + if breaks:
1057 + if ch is None or ch not in '\n\x85\u2028\u2029':
1058 + for br in text[start:end]:
1059 + if br == '\n':
1060 + self.write_line_break()
1061 + else:
1062 + self.write_line_break(br)
1063 + if ch is not None:
1064 + self.write_indent()
1065 + start = end
1066 + else:
1067 + if ch is None or ch in '\n\x85\u2028\u2029':
1068 + data = text[start:end]
1069 + if self.encoding:
1070 + data = data.encode(self.encoding)
1071 + self.stream.write(data)
1072 + if ch is None:
1073 + self.write_line_break()
1074 + start = end
1075 + if ch is not None:
1076 + breaks = (ch in '\n\x85\u2028\u2029')
1077 + end += 1
1078 +
1079 + def write_plain(self, text, split=True):
1080 + if self.root_context:
1081 + self.open_ended = True
1082 + if not text:
1083 + return
1084 + if not self.whitespace:
1085 + data = ' '
1086 + self.column += len(data)
1087 + if self.encoding:
1088 + data = data.encode(self.encoding)
1089 + self.stream.write(data)
1090 + self.whitespace = False
1091 + self.indention = False
1092 + spaces = False
1093 + breaks = False
1094 + start = end = 0
1095 + while end <= len(text):
1096 + ch = None
1097 + if end < len(text):
1098 + ch = text[end]
1099 + if spaces:
1100 + if ch != ' ':
1101 + if start+1 == end and self.column > self.best_width and split:
1102 + self.write_indent()
1103 + self.whitespace = False
1104 + self.indention = False
1105 + else:
1106 + data = text[start:end]
1107 + self.column += len(data)
1108 + if self.encoding:
1109 + data = data.encode(self.encoding)
1110 + self.stream.write(data)
1111 + start = end
1112 + elif breaks:
1113 + if ch not in '\n\x85\u2028\u2029':
1114 + if text[start] == '\n':
1115 + self.write_line_break()
1116 + for br in text[start:end]:
1117 + if br == '\n':
1118 + self.write_line_break()
1119 + else:
1120 + self.write_line_break(br)
1121 + self.write_indent()
1122 + self.whitespace = False
1123 + self.indention = False
1124 + start = end
1125 + else:
1126 + if ch is None or ch in ' \n\x85\u2028\u2029':
1127 + data = text[start:end]
1128 + self.column += len(data)
1129 + if self.encoding:
1130 + data = data.encode(self.encoding)
1131 + self.stream.write(data)
1132 + start = end
1133 + if ch is not None:
1134 + spaces = (ch == ' ')
1135 + breaks = (ch in '\n\x85\u2028\u2029')
1136 + end += 1
1137 +
python.d/python_modules/pyyaml3/error.py new
+75
@@ -0,0 +1,75 @@
1 +
2 +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError']
3 +
4 +class Mark:
5 +
6 + def __init__(self, name, index, line, column, buffer, pointer):
7 + self.name = name
8 + self.index = index
9 + self.line = line
10 + self.column = column
11 + self.buffer = buffer
12 + self.pointer = pointer
13 +
14 + def get_snippet(self, indent=4, max_length=75):
15 + if self.buffer is None:
16 + return None
17 + head = ''
18 + start = self.pointer
19 + while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029':
20 + start -= 1
21 + if self.pointer-start > max_length/2-1:
22 + head = ' ... '
23 + start += 5
24 + break
25 + tail = ''
26 + end = self.pointer
27 + while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029':
28 + end += 1
29 + if end-self.pointer > max_length/2-1:
30 + tail = ' ... '
31 + end -= 5
32 + break
33 + snippet = self.buffer[start:end]
34 + return ' '*indent + head + snippet + tail + '\n' \
35 + + ' '*(indent+self.pointer-start+len(head)) + '^'
36 +
37 + def __str__(self):
38 + snippet = self.get_snippet()
39 + where = " in \"%s\", line %d, column %d" \
40 + % (self.name, self.line+1, self.column+1)
41 + if snippet is not None:
42 + where += ":\n"+snippet
43 + return where
44 +
45 +class YAMLError(Exception):
46 + pass
47 +
48 +class MarkedYAMLError(YAMLError):
49 +
50 + def __init__(self, context=None, context_mark=None,
51 + problem=None, problem_mark=None, note=None):
52 + self.context = context
53 + self.context_mark = context_mark
54 + self.problem = problem
55 + self.problem_mark = problem_mark
56 + self.note = note
57 +
58 + def __str__(self):
59 + lines = []
60 + if self.context is not None:
61 + lines.append(self.context)
62 + if self.context_mark is not None \
63 + and (self.problem is None or self.problem_mark is None
64 + or self.context_mark.name != self.problem_mark.name
65 + or self.context_mark.line != self.problem_mark.line
66 + or self.context_mark.column != self.problem_mark.column):
67 + lines.append(str(self.context_mark))
68 + if self.problem is not None:
69 + lines.append(self.problem)
70 + if self.problem_mark is not None:
71 + lines.append(str(self.problem_mark))
72 + if self.note is not None:
73 + lines.append(self.note)
74 + return '\n'.join(lines)
75 +
python.d/python_modules/pyyaml3/events.py new
+86
@@ -0,0 +1,86 @@
1 +
2 +# Abstract classes.
3 +
4 +class Event(object):
5 + def __init__(self, start_mark=None, end_mark=None):
6 + self.start_mark = start_mark
7 + self.end_mark = end_mark
8 + def __repr__(self):
9 + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value']
10 + if hasattr(self, key)]
11 + arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
12 + for key in attributes])
13 + return '%s(%s)' % (self.__class__.__name__, arguments)
14 +
15 +class NodeEvent(Event):
16 + def __init__(self, anchor, start_mark=None, end_mark=None):
17 + self.anchor = anchor
18 + self.start_mark = start_mark
19 + self.end_mark = end_mark
20 +
21 +class CollectionStartEvent(NodeEvent):
22 + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None,
23 + flow_style=None):
24 + self.anchor = anchor
25 + self.tag = tag
26 + self.implicit = implicit
27 + self.start_mark = start_mark
28 + self.end_mark = end_mark
29 + self.flow_style = flow_style
30 +
31 +class CollectionEndEvent(Event):
32 + pass
33 +
34 +# Implementations.
35 +
36 +class StreamStartEvent(Event):
37 + def __init__(self, start_mark=None, end_mark=None, encoding=None):
38 + self.start_mark = start_mark
39 + self.end_mark = end_mark
40 + self.encoding = encoding
41 +
42 +class StreamEndEvent(Event):
43 + pass
44 +
45 +class DocumentStartEvent(Event):
46 + def __init__(self, start_mark=None, end_mark=None,
47 + explicit=None, version=None, tags=None):
48 + self.start_mark = start_mark
49 + self.end_mark = end_mark
50 + self.explicit = explicit
51 + self.version = version
52 + self.tags = tags
53 +
54 +class DocumentEndEvent(Event):
55 + def __init__(self, start_mark=None, end_mark=None,
56 + explicit=None):
57 + self.start_mark = start_mark
58 + self.end_mark = end_mark
59 + self.explicit = explicit
60 +
61 +class AliasEvent(NodeEvent):
62 + pass
63 +
64 +class ScalarEvent(NodeEvent):
65 + def __init__(self, anchor, tag, implicit, value,
66 + start_mark=None, end_mark=None, style=None):
67 + self.anchor = anchor
68 + self.tag = tag
69 + self.implicit = implicit
70 + self.value = value
71 + self.start_mark = start_mark
72 + self.end_mark = end_mark
73 + self.style = style
74 +
75 +class SequenceStartEvent(CollectionStartEvent):
76 + pass
77 +
78 +class SequenceEndEvent(CollectionEndEvent):
79 + pass
80 +
81 +class MappingStartEvent(CollectionStartEvent):
82 + pass
83 +
84 +class MappingEndEvent(CollectionEndEvent):
85 + pass
86 +
python.d/python_modules/pyyaml3/loader.py new
+40
@@ -0,0 +1,40 @@
1 +
2 +__all__ = ['BaseLoader', 'SafeLoader', 'Loader']
3 +
4 +from .reader import *
5 +from .scanner import *
6 +from .parser import *
7 +from .composer import *
8 +from .constructor import *
9 +from .resolver import *
10 +
11 +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):
12 +
13 + def __init__(self, stream):
14 + Reader.__init__(self, stream)
15 + Scanner.__init__(self)
16 + Parser.__init__(self)
17 + Composer.__init__(self)
18 + BaseConstructor.__init__(self)
19 + BaseResolver.__init__(self)
20 +
21 +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver):
22 +
23 + def __init__(self, stream):
24 + Reader.__init__(self, stream)
25 + Scanner.__init__(self)
26 + Parser.__init__(self)
27 + Composer.__init__(self)
28 + SafeConstructor.__init__(self)
29 + Resolver.__init__(self)
30 +
31 +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
32 +
33 + def __init__(self, stream):
34 + Reader.__init__(self, stream)
35 + Scanner.__init__(self)
36 + Parser.__init__(self)
37 + Composer.__init__(self)
38 + Constructor.__init__(self)
39 + Resolver.__init__(self)
40 +
python.d/python_modules/pyyaml3/nodes.py new
+49
@@ -0,0 +1,49 @@
1 +
2 +class Node(object):
3 + def __init__(self, tag, value, start_mark, end_mark):
4 + self.tag = tag
5 + self.value = value
6 + self.start_mark = start_mark
7 + self.end_mark = end_mark
8 + def __repr__(self):
9 + value = self.value
10 + #if isinstance(value, list):
11 + # if len(value) == 0:
12 + # value = '<empty>'
13 + # elif len(value) == 1:
14 + # value = '<1 item>'
15 + # else:
16 + # value = '<%d items>' % len(value)
17 + #else:
18 + # if len(value) > 75:
19 + # value = repr(value[:70]+u' ... ')
20 + # else:
21 + # value = repr(value)
22 + value = repr(value)
23 + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
24 +
25 +class ScalarNode(Node):
26 + id = 'scalar'
27 + def __init__(self, tag, value,
28 + start_mark=None, end_mark=None, style=None):
29 + self.tag = tag
30 + self.value = value
31 + self.start_mark = start_mark
32 + self.end_mark = end_mark
33 + self.style = style
34 +
35 +class CollectionNode(Node):
36 + def __init__(self, tag, value,
37 + start_mark=None, end_mark=None, flow_style=None):
38 + self.tag = tag
39 + self.value = value
40 + self.start_mark = start_mark
41 + self.end_mark = end_mark
42 + self.flow_style = flow_style
43 +
44 +class SequenceNode(CollectionNode):
45 + id = 'sequence'
46 +
47 +class MappingNode(CollectionNode):
48 + id = 'mapping'
49 +
python.d/python_modules/pyyaml3/parser.py new
+589
@@ -0,0 +1,589 @@
1 +
2 +# The following YAML grammar is LL(1) and is parsed by a recursive descent
3 +# parser.
4 +#
5 +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
6 +# implicit_document ::= block_node DOCUMENT-END*
7 +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
8 +# block_node_or_indentless_sequence ::=
9 +# ALIAS
10 +# | properties (block_content | indentless_block_sequence)?
11 +# | block_content
12 +# | indentless_block_sequence
13 +# block_node ::= ALIAS
14 +# | properties block_content?
15 +# | block_content
16 +# flow_node ::= ALIAS
17 +# | properties flow_content?
18 +# | flow_content
19 +# properties ::= TAG ANCHOR? | ANCHOR TAG?
20 +# block_content ::= block_collection | flow_collection | SCALAR
21 +# flow_content ::= flow_collection | SCALAR
22 +# block_collection ::= block_sequence | block_mapping
23 +# flow_collection ::= flow_sequence | flow_mapping
24 +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
25 +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
26 +# block_mapping ::= BLOCK-MAPPING_START
27 +# ((KEY block_node_or_indentless_sequence?)?
28 +# (VALUE block_node_or_indentless_sequence?)?)*
29 +# BLOCK-END
30 +# flow_sequence ::= FLOW-SEQUENCE-START
31 +# (flow_sequence_entry FLOW-ENTRY)*
32 +# flow_sequence_entry?
33 +# FLOW-SEQUENCE-END
34 +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
35 +# flow_mapping ::= FLOW-MAPPING-START
36 +# (flow_mapping_entry FLOW-ENTRY)*
37 +# flow_mapping_entry?
38 +# FLOW-MAPPING-END
39 +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
40 +#
41 +# FIRST sets:
42 +#
43 +# stream: { STREAM-START }
44 +# explicit_document: { DIRECTIVE DOCUMENT-START }
45 +# implicit_document: FIRST(block_node)
46 +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
47 +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
48 +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
49 +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
50 +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
51 +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
52 +# block_sequence: { BLOCK-SEQUENCE-START }
53 +# block_mapping: { BLOCK-MAPPING-START }
54 +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
55 +# indentless_sequence: { ENTRY }
56 +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
57 +# flow_sequence: { FLOW-SEQUENCE-START }
58 +# flow_mapping: { FLOW-MAPPING-START }
59 +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
60 +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
61 +
62 +__all__ = ['Parser', 'ParserError']
63 +
64 +from .error import MarkedYAMLError
65 +from .tokens import *
66 +from .events import *
67 +from .scanner import *
68 +
69 +class ParserError(MarkedYAMLError):
70 + pass
71 +
72 +class Parser:
73 + # Since writing a recursive-descendant parser is a straightforward task, we
74 + # do not give many comments here.
75 +
76 + DEFAULT_TAGS = {
77 + '!': '!',
78 + '!!': 'tag:yaml.org,2002:',
79 + }
80 +
81 + def __init__(self):
82 + self.current_event = None
83 + self.yaml_version = None
84 + self.tag_handles = {}
85 + self.states = []
86 + self.marks = []
87 + self.state = self.parse_stream_start
88 +
89 + def dispose(self):
90 + # Reset the state attributes (to clear self-references)
91 + self.states = []
92 + self.state = None
93 +
94 + def check_event(self, *choices):
95 + # Check the type of the next event.
96 + if self.current_event is None:
97 + if self.state:
98 + self.current_event = self.state()
99 + if self.current_event is not None:
100 + if not choices:
101 + return True
102 + for choice in choices:
103 + if isinstance(self.current_event, choice):
104 + return True
105 + return False
106 +
107 + def peek_event(self):
108 + # Get the next event.
109 + if self.current_event is None:
110 + if self.state:
111 + self.current_event = self.state()
112 + return self.current_event
113 +
114 + def get_event(self):
115 + # Get the next event and proceed further.
116 + if self.current_event is None:
117 + if self.state:
118 + self.current_event = self.state()
119 + value = self.current_event
120 + self.current_event = None
121 + return value
122 +
123 + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
124 + # implicit_document ::= block_node DOCUMENT-END*
125 + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
126 +
127 + def parse_stream_start(self):
128 +
129 + # Parse the stream start.
130 + token = self.get_token()
131 + event = StreamStartEvent(token.start_mark, token.end_mark,
132 + encoding=token.encoding)
133 +
134 + # Prepare the next state.
135 + self.state = self.parse_implicit_document_start
136 +
137 + return event
138 +
139 + def parse_implicit_document_start(self):
140 +
141 + # Parse an implicit document.
142 + if not self.check_token(DirectiveToken, DocumentStartToken,
143 + StreamEndToken):
144 + self.tag_handles = self.DEFAULT_TAGS
145 + token = self.peek_token()
146 + start_mark = end_mark = token.start_mark
147 + event = DocumentStartEvent(start_mark, end_mark,
148 + explicit=False)
149 +
150 + # Prepare the next state.
151 + self.states.append(self.parse_document_end)
152 + self.state = self.parse_block_node
153 +
154 + return event
155 +
156 + else:
157 + return self.parse_document_start()
158 +
159 + def parse_document_start(self):
160 +
161 + # Parse any extra document end indicators.
162 + while self.check_token(DocumentEndToken):
163 + self.get_token()
164 +
165 + # Parse an explicit document.
166 + if not self.check_token(StreamEndToken):
167 + token = self.peek_token()
168 + start_mark = token.start_mark
169 + version, tags = self.process_directives()
170 + if not self.check_token(DocumentStartToken):
171 + raise ParserError(None, None,
172 + "expected '<document start>', but found %r"
173 + % self.peek_token().id,
174 + self.peek_token().start_mark)
175 + token = self.get_token()
176 + end_mark = token.end_mark
177 + event = DocumentStartEvent(start_mark, end_mark,
178 + explicit=True, version=version, tags=tags)
179 + self.states.append(self.parse_document_end)
180 + self.state = self.parse_document_content
181 + else:
182 + # Parse the end of the stream.
183 + token = self.get_token()
184 + event = StreamEndEvent(token.start_mark, token.end_mark)
185 + assert not self.states
186 + assert not self.marks
187 + self.state = None
188 + return event
189 +
190 + def parse_document_end(self):
191 +
192 + # Parse the document end.
193 + token = self.peek_token()
194 + start_mark = end_mark = token.start_mark
195 + explicit = False
196 + if self.check_token(DocumentEndToken):
197 + token = self.get_token()
198 + end_mark = token.end_mark
199 + explicit = True
200 + event = DocumentEndEvent(start_mark, end_mark,
201 + explicit=explicit)
202 +
203 + # Prepare the next state.
204 + self.state = self.parse_document_start
205 +
206 + return event
207 +
208 + def parse_document_content(self):
209 + if self.check_token(DirectiveToken,
210 + DocumentStartToken, DocumentEndToken, StreamEndToken):
211 + event = self.process_empty_scalar(self.peek_token().start_mark)
212 + self.state = self.states.pop()
213 + return event
214 + else:
215 + return self.parse_block_node()
216 +
217 + def process_directives(self):
218 + self.yaml_version = None
219 + self.tag_handles = {}
220 + while self.check_token(DirectiveToken):
221 + token = self.get_token()
222 + if token.name == 'YAML':
223 + if self.yaml_version is not None:
224 + raise ParserError(None, None,
225 + "found duplicate YAML directive", token.start_mark)
226 + major, minor = token.value
227 + if major != 1:
228 + raise ParserError(None, None,
229 + "found incompatible YAML document (version 1.* is required)",
230 + token.start_mark)
231 + self.yaml_version = token.value
232 + elif token.name == 'TAG':
233 + handle, prefix = token.value
234 + if handle in self.tag_handles:
235 + raise ParserError(None, None,
236 + "duplicate tag handle %r" % handle,
237 + token.start_mark)
238 + self.tag_handles[handle] = prefix
239 + if self.tag_handles:
240 + value = self.yaml_version, self.tag_handles.copy()
241 + else:
242 + value = self.yaml_version, None
243 + for key in self.DEFAULT_TAGS:
244 + if key not in self.tag_handles:
245 + self.tag_handles[key] = self.DEFAULT_TAGS[key]
246 + return value
247 +
248 + # block_node_or_indentless_sequence ::= ALIAS
249 + # | properties (block_content | indentless_block_sequence)?
250 + # | block_content
251 + # | indentless_block_sequence
252 + # block_node ::= ALIAS
253 + # | properties block_content?
254 + # | block_content
255 + # flow_node ::= ALIAS
256 + # | properties flow_content?
257 + # | flow_content
258 + # properties ::= TAG ANCHOR? | ANCHOR TAG?
259 + # block_content ::= block_collection | flow_collection | SCALAR
260 + # flow_content ::= flow_collection | SCALAR
261 + # block_collection ::= block_sequence | block_mapping
262 + # flow_collection ::= flow_sequence | flow_mapping
263 +
264 + def parse_block_node(self):
265 + return self.parse_node(block=True)
266 +
267 + def parse_flow_node(self):
268 + return self.parse_node()
269 +
270 + def parse_block_node_or_indentless_sequence(self):
271 + return self.parse_node(block=True, indentless_sequence=True)
272 +
273 + def parse_node(self, block=False, indentless_sequence=False):
274 + if self.check_token(AliasToken):
275 + token = self.get_token()
276 + event = AliasEvent(token.value, token.start_mark, token.end_mark)
277 + self.state = self.states.pop()
278 + else:
279 + anchor = None
280 + tag = None
281 + start_mark = end_mark = tag_mark = None
282 + if self.check_token(AnchorToken):
283 + token = self.get_token()
284 + start_mark = token.start_mark
285 + end_mark = token.end_mark
286 + anchor = token.value
287 + if self.check_token(TagToken):
288 + token = self.get_token()
289 + tag_mark = token.start_mark
290 + end_mark = token.end_mark
291 + tag = token.value
292 + elif self.check_token(TagToken):
293 + token = self.get_token()
294 + start_mark = tag_mark = token.start_mark
295 + end_mark = token.end_mark
296 + tag = token.value
297 + if self.check_token(AnchorToken):
298 + token = self.get_token()
299 + end_mark = token.end_mark
300 + anchor = token.value
301 + if tag is not None:
302 + handle, suffix = tag
303 + if handle is not None:
304 + if handle not in self.tag_handles:
305 + raise ParserError("while parsing a node", start_mark,
306 + "found undefined tag handle %r" % handle,
307 + tag_mark)
308 + tag = self.tag_handles[handle]+suffix
309 + else:
310 + tag = suffix
311 + #if tag == '!':
312 + # raise ParserError("while parsing a node", start_mark,
313 + # "found non-specific tag '!'", tag_mark,
314 + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
315 + if start_mark is None:
316 + start_mark = end_mark = self.peek_token().start_mark
317 + event = None
318 + implicit = (tag is None or tag == '!')
319 + if indentless_sequence and self.check_token(BlockEntryToken):
320 + end_mark = self.peek_token().end_mark
321 + event = SequenceStartEvent(anchor, tag, implicit,
322 + start_mark, end_mark)
323 + self.state = self.parse_indentless_sequence_entry
324 + else:
325 + if self.check_token(ScalarToken):
326 + token = self.get_token()
327 + end_mark = token.end_mark
328 + if (token.plain and tag is None) or tag == '!':
329 + implicit = (True, False)
330 + elif tag is None:
331 + implicit = (False, True)
332 + else:
333 + implicit = (False, False)
334 + event = ScalarEvent(anchor, tag, implicit, token.value,
335 + start_mark, end_mark, style=token.style)
336 + self.state = self.states.pop()
337 + elif self.check_token(FlowSequenceStartToken):
338 + end_mark = self.peek_token().end_mark
339 + event = SequenceStartEvent(anchor, tag, implicit,
340 + start_mark, end_mark, flow_style=True)
341 + self.state = self.parse_flow_sequence_first_entry
342 + elif self.check_token(FlowMappingStartToken):
343 + end_mark = self.peek_token().end_mark
344 + event = MappingStartEvent(anchor, tag, implicit,
345 + start_mark, end_mark, flow_style=True)
346 + self.state = self.parse_flow_mapping_first_key
347 + elif block and self.check_token(BlockSequenceStartToken):
348 + end_mark = self.peek_token().start_mark
349 + event = SequenceStartEvent(anchor, tag, implicit,
350 + start_mark, end_mark, flow_style=False)
351 + self.state = self.parse_block_sequence_first_entry
352 + elif block and self.check_token(BlockMappingStartToken):
353 + end_mark = self.peek_token().start_mark
354 + event = MappingStartEvent(anchor, tag, implicit,
355 + start_mark, end_mark, flow_style=False)
356 + self.state = self.parse_block_mapping_first_key
357 + elif anchor is not None or tag is not None:
358 + # Empty scalars are allowed even if a tag or an anchor is
359 + # specified.
360 + event = ScalarEvent(anchor, tag, (implicit, False), '',
361 + start_mark, end_mark)
362 + self.state = self.states.pop()
363 + else:
364 + if block:
365 + node = 'block'
366 + else:
367 + node = 'flow'
368 + token = self.peek_token()
369 + raise ParserError("while parsing a %s node" % node, start_mark,
370 + "expected the node content, but found %r" % token.id,
371 + token.start_mark)
372 + return event
373 +
374 + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
375 +
376 + def parse_block_sequence_first_entry(self):
377 + token = self.get_token()
378 + self.marks.append(token.start_mark)
379 + return self.parse_block_sequence_entry()
380 +
381 + def parse_block_sequence_entry(self):
382 + if self.check_token(BlockEntryToken):
383 + token = self.get_token()
384 + if not self.check_token(BlockEntryToken, BlockEndToken):
385 + self.states.append(self.parse_block_sequence_entry)
386 + return self.parse_block_node()
387 + else:
388 + self.state = self.parse_block_sequence_entry
389 + return self.process_empty_scalar(token.end_mark)
390 + if not self.check_token(BlockEndToken):
391 + token = self.peek_token()
392 + raise ParserError("while parsing a block collection", self.marks[-1],
393 + "expected <block end>, but found %r" % token.id, token.start_mark)
394 + token = self.get_token()
395 + event = SequenceEndEvent(token.start_mark, token.end_mark)
396 + self.state = self.states.pop()
397 + self.marks.pop()
398 + return event
399 +
400 + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
401 +
402 + def parse_indentless_sequence_entry(self):
403 + if self.check_token(BlockEntryToken):
404 + token = self.get_token()
405 + if not self.check_token(BlockEntryToken,
406 + KeyToken, ValueToken, BlockEndToken):
407 + self.states.append(self.parse_indentless_sequence_entry)
408 + return self.parse_block_node()
409 + else:
410 + self.state = self.parse_indentless_sequence_entry
411 + return self.process_empty_scalar(token.end_mark)
412 + token = self.peek_token()
413 + event = SequenceEndEvent(token.start_mark, token.start_mark)
414 + self.state = self.states.pop()
415 + return event
416 +
417 + # block_mapping ::= BLOCK-MAPPING_START
418 + # ((KEY block_node_or_indentless_sequence?)?
419 + # (VALUE block_node_or_indentless_sequence?)?)*
420 + # BLOCK-END
421 +
422 + def parse_block_mapping_first_key(self):
423 + token = self.get_token()
424 + self.marks.append(token.start_mark)
425 + return self.parse_block_mapping_key()
426 +
427 + def parse_block_mapping_key(self):
428 + if self.check_token(KeyToken):
429 + token = self.get_token()
430 + if not self.check_token(KeyToken, ValueToken, BlockEndToken):
431 + self.states.append(self.parse_block_mapping_value)
432 + return self.parse_block_node_or_indentless_sequence()
433 + else:
434 + self.state = self.parse_block_mapping_value
435 + return self.process_empty_scalar(token.end_mark)
436 + if not self.check_token(BlockEndToken):
437 + token = self.peek_token()
438 + raise ParserError("while parsing a block mapping", self.marks[-1],
439 + "expected <block end>, but found %r" % token.id, token.start_mark)
440 + token = self.get_token()
441 + event = MappingEndEvent(token.start_mark, token.end_mark)
442 + self.state = self.states.pop()
443 + self.marks.pop()
444 + return event
445 +
446 + def parse_block_mapping_value(self):
447 + if self.check_token(ValueToken):
448 + token = self.get_token()
449 + if not self.check_token(KeyToken, ValueToken, BlockEndToken):
450 + self.states.append(self.parse_block_mapping_key)
451 + return self.parse_block_node_or_indentless_sequence()
452 + else:
453 + self.state = self.parse_block_mapping_key
454 + return self.process_empty_scalar(token.end_mark)
455 + else:
456 + self.state = self.parse_block_mapping_key
457 + token = self.peek_token()
458 + return self.process_empty_scalar(token.start_mark)
459 +
460 + # flow_sequence ::= FLOW-SEQUENCE-START
461 + # (flow_sequence_entry FLOW-ENTRY)*
462 + # flow_sequence_entry?
463 + # FLOW-SEQUENCE-END
464 + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
465 + #
466 + # Note that while production rules for both flow_sequence_entry and
467 + # flow_mapping_entry are equal, their interpretations are different.
468 + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
469 + # generate an inline mapping (set syntax).
470 +
471 + def parse_flow_sequence_first_entry(self):
472 + token = self.get_token()
473 + self.marks.append(token.start_mark)
474 + return self.parse_flow_sequence_entry(first=True)
475 +
476 + def parse_flow_sequence_entry(self, first=False):
477 + if not self.check_token(FlowSequenceEndToken):
478 + if not first:
479 + if self.check_token(FlowEntryToken):
480 + self.get_token()
481 + else:
482 + token = self.peek_token()
483 + raise ParserError("while parsing a flow sequence", self.marks[-1],
484 + "expected ',' or ']', but got %r" % token.id, token.start_mark)
485 +
486 + if self.check_token(KeyToken):
487 + token = self.peek_token()
488 + event = MappingStartEvent(None, None, True,
489 + token.start_mark, token.end_mark,
490 + flow_style=True)
491 + self.state = self.parse_flow_sequence_entry_mapping_key
492 + return event
493 + elif not self.check_token(FlowSequenceEndToken):
494 + self.states.append(self.parse_flow_sequence_entry)
495 + return self.parse_flow_node()
496 + token = self.get_token()
497 + event = SequenceEndEvent(token.start_mark, token.end_mark)
498 + self.state = self.states.pop()
499 + self.marks.pop()
500 + return event
501 +
502 + def parse_flow_sequence_entry_mapping_key(self):
503 + token = self.get_token()
504 + if not self.check_token(ValueToken,
505 + FlowEntryToken, FlowSequenceEndToken):
506 + self.states.append(self.parse_flow_sequence_entry_mapping_value)
507 + return self.parse_flow_node()
508 + else:
509 + self.state = self.parse_flow_sequence_entry_mapping_value
510 + return self.process_empty_scalar(token.end_mark)
511 +
512 + def parse_flow_sequence_entry_mapping_value(self):
513 + if self.check_token(ValueToken):
514 + token = self.get_token()
515 + if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
516 + self.states.append(self.parse_flow_sequence_entry_mapping_end)
517 + return self.parse_flow_node()
518 + else:
519 + self.state = self.parse_flow_sequence_entry_mapping_end
520 + return self.process_empty_scalar(token.end_mark)
521 + else:
522 + self.state = self.parse_flow_sequence_entry_mapping_end
523 + token = self.peek_token()
524 + return self.process_empty_scalar(token.start_mark)
525 +
526 + def parse_flow_sequence_entry_mapping_end(self):
527 + self.state = self.parse_flow_sequence_entry
528 + token = self.peek_token()
529 + return MappingEndEvent(token.start_mark, token.start_mark)
530 +
531 + # flow_mapping ::= FLOW-MAPPING-START
532 + # (flow_mapping_entry FLOW-ENTRY)*
533 + # flow_mapping_entry?
534 + # FLOW-MAPPING-END
535 + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
536 +
537 + def parse_flow_mapping_first_key(self):
538 + token = self.get_token()
539 + self.marks.append(token.start_mark)
540 + return self.parse_flow_mapping_key(first=True)
541 +
542 + def parse_flow_mapping_key(self, first=False):
543 + if not self.check_token(FlowMappingEndToken):
544 + if not first:
545 + if self.check_token(FlowEntryToken):
546 + self.get_token()
547 + else:
548 + token = self.peek_token()
549 + raise ParserError("while parsing a flow mapping", self.marks[-1],
550 + "expected ',' or '}', but got %r" % token.id, token.start_mark)
551 + if self.check_token(KeyToken):
552 + token = self.get_token()
553 + if not self.check_token(ValueToken,
554 + FlowEntryToken, FlowMappingEndToken):
555 + self.states.append(self.parse_flow_mapping_value)
556 + return self.parse_flow_node()
557 + else:
558 + self.state = self.parse_flow_mapping_value
559 + return self.process_empty_scalar(token.end_mark)
560 + elif not self.check_token(FlowMappingEndToken):
561 + self.states.append(self.parse_flow_mapping_empty_value)
562 + return self.parse_flow_node()
563 + token = self.get_token()
564 + event = MappingEndEvent(token.start_mark, token.end_mark)
565 + self.state = self.states.pop()
566 + self.marks.pop()
567 + return event
568 +
569 + def parse_flow_mapping_value(self):
570 + if self.check_token(ValueToken):
571 + token = self.get_token()
572 + if not self.check_token(FlowEntryToken, FlowMappingEndToken):
573 + self.states.append(self.parse_flow_mapping_key)
574 + return self.parse_flow_node()
575 + else:
576 + self.state = self.parse_flow_mapping_key
577 + return self.process_empty_scalar(token.end_mark)
578 + else:
579 + self.state = self.parse_flow_mapping_key
580 + token = self.peek_token()
581 + return self.process_empty_scalar(token.start_mark)
582 +
583 + def parse_flow_mapping_empty_value(self):
584 + self.state = self.parse_flow_mapping_key
585 + return self.process_empty_scalar(self.peek_token().start_mark)
586 +
587 + def process_empty_scalar(self, mark):
588 + return ScalarEvent(None, None, (True, False), '', mark, mark)
589 +
python.d/python_modules/pyyaml3/reader.py new
+192
@@ -0,0 +1,192 @@
1 +# This module contains abstractions for the input stream. You don't have to
2 +# looks further, there are no pretty code.
3 +#
4 +# We define two classes here.
5 +#
6 +# Mark(source, line, column)
7 +# It's just a record and its only use is producing nice error messages.
8 +# Parser does not use it for any other purposes.
9 +#
10 +# Reader(source, data)
11 +# Reader determines the encoding of `data` and converts it to unicode.
12 +# Reader provides the following methods and attributes:
13 +# reader.peek(length=1) - return the next `length` characters
14 +# reader.forward(length=1) - move the current position to `length` characters.
15 +# reader.index - the number of the current character.
16 +# reader.line, stream.column - the line and the column of the current character.
17 +
18 +__all__ = ['Reader', 'ReaderError']
19 +
20 +from .error import YAMLError, Mark
21 +
22 +import codecs, re
23 +
24 +class ReaderError(YAMLError):
25 +
26 + def __init__(self, name, position, character, encoding, reason):
27 + self.name = name
28 + self.character = character
29 + self.position = position
30 + self.encoding = encoding
31 + self.reason = reason
32 +
33 + def __str__(self):
34 + if isinstance(self.character, bytes):
35 + return "'%s' codec can't decode byte #x%02x: %s\n" \
36 + " in \"%s\", position %d" \
37 + % (self.encoding, ord(self.character), self.reason,
38 + self.name, self.position)
39 + else:
40 + return "unacceptable character #x%04x: %s\n" \
41 + " in \"%s\", position %d" \
42 + % (self.character, self.reason,
43 + self.name, self.position)
44 +
45 +class Reader(object):
46 + # Reader:
47 + # - determines the data encoding and converts it to a unicode string,
48 + # - checks if characters are in allowed range,
49 + # - adds '\0' to the end.
50 +
51 + # Reader accepts
52 + # - a `bytes` object,
53 + # - a `str` object,
54 + # - a file-like object with its `read` method returning `str`,
55 + # - a file-like object with its `read` method returning `unicode`.
56 +
57 + # Yeah, it's ugly and slow.
58 +
59 + def __init__(self, stream):
60 + self.name = None
61 + self.stream = None
62 + self.stream_pointer = 0
63 + self.eof = True
64 + self.buffer = ''
65 + self.pointer = 0
66 + self.raw_buffer = None
67 + self.raw_decode = None
68 + self.encoding = None
69 + self.index = 0
70 + self.line = 0
71 + self.column = 0
72 + if isinstance(stream, str):
73 + self.name = "<unicode string>"
74 + self.check_printable(stream)
75 + self.buffer = stream+'\0'
76 + elif isinstance(stream, bytes):
77 + self.name = "<byte string>"
78 + self.raw_buffer = stream
79 + self.determine_encoding()
80 + else:
81 + self.stream = stream
82 + self.name = getattr(stream, 'name', "<file>")
83 + self.eof = False
84 + self.raw_buffer = None
85 + self.determine_encoding()
86 +
87 + def peek(self, index=0):
88 + try:
89 + return self.buffer[self.pointer+index]
90 + except IndexError:
91 + self.update(index+1)
92 + return self.buffer[self.pointer+index]
93 +
94 + def prefix(self, length=1):
95 + if self.pointer+length >= len(self.buffer):
96 + self.update(length)
97 + return self.buffer[self.pointer:self.pointer+length]
98 +
99 + def forward(self, length=1):
100 + if self.pointer+length+1 >= len(self.buffer):
101 + self.update(length+1)
102 + while length:
103 + ch = self.buffer[self.pointer]
104 + self.pointer += 1
105 + self.index += 1
106 + if ch in '\n\x85\u2028\u2029' \
107 + or (ch == '\r' and self.buffer[self.pointer] != '\n'):
108 + self.line += 1
109 + self.column = 0
110 + elif ch != '\uFEFF':
111 + self.column += 1
112 + length -= 1
113 +
114 + def get_mark(self):
115 + if self.stream is None:
116 + return Mark(self.name, self.index, self.line, self.column,
117 + self.buffer, self.pointer)
118 + else:
119 + return Mark(self.name, self.index, self.line, self.column,
120 + None, None)
121 +
122 + def determine_encoding(self):
123 + while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
124 + self.update_raw()
125 + if isinstance(self.raw_buffer, bytes):
126 + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
127 + self.raw_decode = codecs.utf_16_le_decode
128 + self.encoding = 'utf-16-le'
129 + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
130 + self.raw_decode = codecs.utf_16_be_decode
131 + self.encoding = 'utf-16-be'
132 + else:
133 + self.raw_decode = codecs.utf_8_decode
134 + self.encoding = 'utf-8'
135 + self.update(1)
136 +
137 + NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')
138 + def check_printable(self, data):
139 + match = self.NON_PRINTABLE.search(data)
140 + if match:
141 + character = match.group()
142 + position = self.index+(len(self.buffer)-self.pointer)+match.start()
143 + raise ReaderError(self.name, position, ord(character),
144 + 'unicode', "special characters are not allowed")
145 +
146 + def update(self, length):
147 + if self.raw_buffer is None:
148 + return
149 + self.buffer = self.buffer[self.pointer:]
150 + self.pointer = 0
151 + while len(self.buffer) < length:
152 + if not self.eof:
153 + self.update_raw()
154 + if self.raw_decode is not None:
155 + try:
156 + data, converted = self.raw_decode(self.raw_buffer,
157 + 'strict', self.eof)
158 + except UnicodeDecodeError as exc:
159 + character = self.raw_buffer[exc.start]
160 + if self.stream is not None:
161 + position = self.stream_pointer-len(self.raw_buffer)+exc.start
162 + else:
163 + position = exc.start
164 + raise ReaderError(self.name, position, character,
165 + exc.encoding, exc.reason)
166 + else:
167 + data = self.raw_buffer
168 + converted = len(data)
169 + self.check_printable(data)
170 + self.buffer += data
171 + self.raw_buffer = self.raw_buffer[converted:]
172 + if self.eof:
173 + self.buffer += '\0'
174 + self.raw_buffer = None
175 + break
176 +
177 + def update_raw(self, size=4096):
178 + data = self.stream.read(size)
179 + if self.raw_buffer is None:
180 + self.raw_buffer = data
181 + else:
182 + self.raw_buffer += data
183 + self.stream_pointer += len(data)
184 + if not data:
185 + self.eof = True
186 +
187 +#try:
188 +# import psyco
189 +# psyco.bind(Reader)
190 +#except ImportError:
191 +# pass
192 +
python.d/python_modules/pyyaml3/representer.py new
+374
@@ -0,0 +1,374 @@
1 +
2 +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer',
3 + 'RepresenterError']
4 +
5 +from .error import *
6 +from .nodes import *
7 +
8 +import datetime, sys, copyreg, types, base64
9 +
10 +class RepresenterError(YAMLError):
11 + pass
12 +
13 +class BaseRepresenter:
14 +
15 + yaml_representers = {}
16 + yaml_multi_representers = {}
17 +
18 + def __init__(self, default_style=None, default_flow_style=None):
19 + self.default_style = default_style
20 + self.default_flow_style = default_flow_style
21 + self.represented_objects = {}
22 + self.object_keeper = []
23 + self.alias_key = None
24 +
25 + def represent(self, data):
26 + node = self.represent_data(data)
27 + self.serialize(node)
28 + self.represented_objects = {}
29 + self.object_keeper = []
30 + self.alias_key = None
31 +
32 + def represent_data(self, data):
33 + if self.ignore_aliases(data):
34 + self.alias_key = None
35 + else:
36 + self.alias_key = id(data)
37 + if self.alias_key is not None:
38 + if self.alias_key in self.represented_objects:
39 + node = self.represented_objects[self.alias_key]
40 + #if node is None:
41 + # raise RepresenterError("recursive objects are not allowed: %r" % data)
42 + return node
43 + #self.represented_objects[alias_key] = None
44 + self.object_keeper.append(data)
45 + data_types = type(data).__mro__
46 + if data_types[0] in self.yaml_representers:
47 + node = self.yaml_representers[data_types[0]](self, data)
48 + else:
49 + for data_type in data_types:
50 + if data_type in self.yaml_multi_representers:
51 + node = self.yaml_multi_representers[data_type](self, data)
52 + break
53 + else:
54 + if None in self.yaml_multi_representers:
55 + node = self.yaml_multi_representers[None](self, data)
56 + elif None in self.yaml_representers:
57 + node = self.yaml_representers[None](self, data)
58 + else:
59 + node = ScalarNode(None, str(data))
60 + #if alias_key is not None:
61 + # self.represented_objects[alias_key] = node
62 + return node
63 +
64 + @classmethod
65 + def add_representer(cls, data_type, representer):
66 + if not 'yaml_representers' in cls.__dict__:
67 + cls.yaml_representers = cls.yaml_representers.copy()
68 + cls.yaml_representers[data_type] = representer
69 +
70 + @classmethod
71 + def add_multi_representer(cls, data_type, representer):
72 + if not 'yaml_multi_representers' in cls.__dict__:
73 + cls.yaml_multi_representers = cls.yaml_multi_representers.copy()
74 + cls.yaml_multi_representers[data_type] = representer
75 +
76 + def represent_scalar(self, tag, value, style=None):
77 + if style is None:
78 + style = self.default_style
79 + node = ScalarNode(tag, value, style=style)
80 + if self.alias_key is not None:
81 + self.represented_objects[self.alias_key] = node
82 + return node
83 +
84 + def represent_sequence(self, tag, sequence, flow_style=None):
85 + value = []
86 + node = SequenceNode(tag, value, flow_style=flow_style)
87 + if self.alias_key is not None:
88 + self.represented_objects[self.alias_key] = node
89 + best_style = True
90 + for item in sequence:
91 + node_item = self.represent_data(item)
92 + if not (isinstance(node_item, ScalarNode) and not node_item.style):
93 + best_style = False
94 + value.append(node_item)
95 + if flow_style is None:
96 + if self.default_flow_style is not None:
97 + node.flow_style = self.default_flow_style
98 + else:
99 + node.flow_style = best_style
100 + return node
101 +
102 + def represent_mapping(self, tag, mapping, flow_style=None):
103 + value = []
104 + node = MappingNode(tag, value, flow_style=flow_style)
105 + if self.alias_key is not None:
106 + self.represented_objects[self.alias_key] = node
107 + best_style = True
108 + if hasattr(mapping, 'items'):
109 + mapping = list(mapping.items())
110 + try:
111 + mapping = sorted(mapping)
112 + except TypeError:
113 + pass
114 + for item_key, item_value in mapping:
115 + node_key = self.represent_data(item_key)
116 + node_value = self.represent_data(item_value)
117 + if not (isinstance(node_key, ScalarNode) and not node_key.style):
118 + best_style = False
119 + if not (isinstance(node_value, ScalarNode) and not node_value.style):
120 + best_style = False
121 + value.append((node_key, node_value))
122 + if flow_style is None:
123 + if self.default_flow_style is not None:
124 + node.flow_style = self.default_flow_style
125 + else:
126 + node.flow_style = best_style
127 + return node
128 +
129 + def ignore_aliases(self, data):
130 + return False
131 +
132 +class SafeRepresenter(BaseRepresenter):
133 +
134 + def ignore_aliases(self, data):
135 + if data in [None, ()]:
136 + return True
137 + if isinstance(data, (str, bytes, bool, int, float)):
138 + return True
139 +
140 + def represent_none(self, data):
141 + return self.represent_scalar('tag:yaml.org,2002:null', 'null')
142 +
143 + def represent_str(self, data):
144 + return self.represent_scalar('tag:yaml.org,2002:str', data)
145 +
146 + def represent_binary(self, data):
147 + if hasattr(base64, 'encodebytes'):
148 + data = base64.encodebytes(data).decode('ascii')
149 + else:
150 + data = base64.encodestring(data).decode('ascii')
151 + return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|')
152 +
153 + def represent_bool(self, data):
154 + if data:
155 + value = 'true'
156 + else:
157 + value = 'false'
158 + return self.represent_scalar('tag:yaml.org,2002:bool', value)
159 +
160 + def represent_int(self, data):
161 + return self.represent_scalar('tag:yaml.org,2002:int', str(data))
162 +
163 + inf_value = 1e300
164 + while repr(inf_value) != repr(inf_value*inf_value):
165 + inf_value *= inf_value
166 +
167 + def represent_float(self, data):
168 + if data != data or (data == 0.0 and data == 1.0):
169 + value = '.nan'
170 + elif data == self.inf_value:
171 + value = '.inf'
172 + elif data == -self.inf_value:
173 + value = '-.inf'
174 + else:
175 + value = repr(data).lower()
176 + # Note that in some cases `repr(data)` represents a float number
177 + # without the decimal parts. For instance:
178 + # >>> repr(1e17)
179 + # '1e17'
180 + # Unfortunately, this is not a valid float representation according
181 + # to the definition of the `!!float` tag. We fix this by adding
182 + # '.0' before the 'e' symbol.
183 + if '.' not in value and 'e' in value:
184 + value = value.replace('e', '.0e', 1)
185 + return self.represent_scalar('tag:yaml.org,2002:float', value)
186 +
187 + def represent_list(self, data):
188 + #pairs = (len(data) > 0 and isinstance(data, list))
189 + #if pairs:
190 + # for item in data:
191 + # if not isinstance(item, tuple) or len(item) != 2:
192 + # pairs = False
193 + # break
194 + #if not pairs:
195 + return self.represent_sequence('tag:yaml.org,2002:seq', data)
196 + #value = []
197 + #for item_key, item_value in data:
198 + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
199 + # [(item_key, item_value)]))
200 + #return SequenceNode(u'tag:yaml.org,2002:pairs', value)
201 +
202 + def represent_dict(self, data):
203 + return self.represent_mapping('tag:yaml.org,2002:map', data)
204 +
205 + def represent_set(self, data):
206 + value = {}
207 + for key in data:
208 + value[key] = None
209 + return self.represent_mapping('tag:yaml.org,2002:set', value)
210 +
211 + def represent_date(self, data):
212 + value = data.isoformat()
213 + return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
214 +
215 + def represent_datetime(self, data):
216 + value = data.isoformat(' ')
217 + return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
218 +
219 + def represent_yaml_object(self, tag, data, cls, flow_style=None):
220 + if hasattr(data, '__getstate__'):
221 + state = data.__getstate__()
222 + else:
223 + state = data.__dict__.copy()
224 + return self.represent_mapping(tag, state, flow_style=flow_style)
225 +
226 + def represent_undefined(self, data):
227 + raise RepresenterError("cannot represent an object: %s" % data)
228 +
229 +SafeRepresenter.add_representer(type(None),
230 + SafeRepresenter.represent_none)
231 +
232 +SafeRepresenter.add_representer(str,
233 + SafeRepresenter.represent_str)
234 +
235 +SafeRepresenter.add_representer(bytes,
236 + SafeRepresenter.represent_binary)
237 +
238 +SafeRepresenter.add_representer(bool,
239 + SafeRepresenter.represent_bool)
240 +
241 +SafeRepresenter.add_representer(int,
242 + SafeRepresenter.represent_int)
243 +
244 +SafeRepresenter.add_representer(float,
245 + SafeRepresenter.represent_float)
246 +
247 +SafeRepresenter.add_representer(list,
248 + SafeRepresenter.represent_list)
249 +
250 +SafeRepresenter.add_representer(tuple,
251 + SafeRepresenter.represent_list)
252 +
253 +SafeRepresenter.add_representer(dict,
254 + SafeRepresenter.represent_dict)
255 +
256 +SafeRepresenter.add_representer(set,
257 + SafeRepresenter.represent_set)
258 +
259 +SafeRepresenter.add_representer(datetime.date,
260 + SafeRepresenter.represent_date)
261 +
262 +SafeRepresenter.add_representer(datetime.datetime,
263 + SafeRepresenter.represent_datetime)
264 +
265 +SafeRepresenter.add_representer(None,
266 + SafeRepresenter.represent_undefined)
267 +
268 +class Representer(SafeRepresenter):
269 +
270 + def represent_complex(self, data):
271 + if data.imag == 0.0:
272 + data = '%r' % data.real
273 + elif data.real == 0.0:
274 + data = '%rj' % data.imag
275 + elif data.imag > 0:
276 + data = '%r+%rj' % (data.real, data.imag)
277 + else:
278 + data = '%r%rj' % (data.real, data.imag)
279 + return self.represent_scalar('tag:yaml.org,2002:python/complex', data)
280 +
281 + def represent_tuple(self, data):
282 + return self.represent_sequence('tag:yaml.org,2002:python/tuple', data)
283 +
284 + def represent_name(self, data):
285 + name = '%s.%s' % (data.__module__, data.__name__)
286 + return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '')
287 +
288 + def represent_module(self, data):
289 + return self.represent_scalar(
290 + 'tag:yaml.org,2002:python/module:'+data.__name__, '')
291 +
292 + def represent_object(self, data):
293 + # We use __reduce__ API to save the data. data.__reduce__ returns
294 + # a tuple of length 2-5:
295 + # (function, args, state, listitems, dictitems)
296 +
297 + # For reconstructing, we calls function(*args), then set its state,
298 + # listitems, and dictitems if they are not None.
299 +
300 + # A special case is when function.__name__ == '__newobj__'. In this
301 + # case we create the object with args[0].__new__(*args).
302 +
303 + # Another special case is when __reduce__ returns a string - we don't
304 + # support it.
305 +
306 + # We produce a !!python/object, !!python/object/new or
307 + # !!python/object/apply node.
308 +
309 + cls = type(data)
310 + if cls in copyreg.dispatch_table:
311 + reduce = copyreg.dispatch_table[cls](data)
312 + elif hasattr(data, '__reduce_ex__'):
313 + reduce = data.__reduce_ex__(2)
314 + elif hasattr(data, '__reduce__'):
315 + reduce = data.__reduce__()
316 + else:
317 + raise RepresenterError("cannot represent object: %r" % data)
318 + reduce = (list(reduce)+[None]*5)[:5]
319 + function, args, state, listitems, dictitems = reduce
320 + args = list(args)
321 + if state is None:
322 + state = {}
323 + if listitems is not None:
324 + listitems = list(listitems)
325 + if dictitems is not None:
326 + dictitems = dict(dictitems)
327 + if function.__name__ == '__newobj__':
328 + function = args[0]
329 + args = args[1:]
330 + tag = 'tag:yaml.org,2002:python/object/new:'
331 + newobj = True
332 + else:
333 + tag = 'tag:yaml.org,2002:python/object/apply:'
334 + newobj = False
335 + function_name = '%s.%s' % (function.__module__, function.__name__)
336 + if not args and not listitems and not dictitems \
337 + and isinstance(state, dict) and newobj:
338 + return self.represent_mapping(
339 + 'tag:yaml.org,2002:python/object:'+function_name, state)
340 + if not listitems and not dictitems \
341 + and isinstance(state, dict) and not state:
342 + return self.represent_sequence(tag+function_name, args)
343 + value = {}
344 + if args:
345 + value['args'] = args
346 + if state or not isinstance(state, dict):
347 + value['state'] = state
348 + if listitems:
349 + value['listitems'] = listitems
350 + if dictitems:
351 + value['dictitems'] = dictitems
352 + return self.represent_mapping(tag+function_name, value)
353 +
354 +Representer.add_representer(complex,
355 + Representer.represent_complex)
356 +
357 +Representer.add_representer(tuple,
358 + Representer.represent_tuple)
359 +
360 +Representer.add_representer(type,
361 + Representer.represent_name)
362 +
363 +Representer.add_representer(types.FunctionType,
364 + Representer.represent_name)
365 +
366 +Representer.add_representer(types.BuiltinFunctionType,
367 + Representer.represent_name)
368 +
369 +Representer.add_representer(types.ModuleType,
370 + Representer.represent_module)
371 +
372 +Representer.add_multi_representer(object,
373 + Representer.represent_object)
374 +
python.d/python_modules/pyyaml3/resolver.py new
+224
@@ -0,0 +1,224 @@
1 +
2 +__all__ = ['BaseResolver', 'Resolver']
3 +
4 +from .error import *
5 +from .nodes import *
6 +
7 +import re
8 +
9 +class ResolverError(YAMLError):
10 + pass
11 +
12 +class BaseResolver:
13 +
14 + DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str'
15 + DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq'
16 + DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map'
17 +
18 + yaml_implicit_resolvers = {}
19 + yaml_path_resolvers = {}
20 +
21 + def __init__(self):
22 + self.resolver_exact_paths = []
23 + self.resolver_prefix_paths = []
24 +
25 + @classmethod
26 + def add_implicit_resolver(cls, tag, regexp, first):
27 + if not 'yaml_implicit_resolvers' in cls.__dict__:
28 + cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
29 + if first is None:
30 + first = [None]
31 + for ch in first:
32 + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
33 +
34 + @classmethod
35 + def add_path_resolver(cls, tag, path, kind=None):
36 + # Note: `add_path_resolver` is experimental. The API could be changed.
37 + # `new_path` is a pattern that is matched against the path from the
38 + # root to the node that is being considered. `node_path` elements are
39 + # tuples `(node_check, index_check)`. `node_check` is a node class:
40 + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None`
41 + # matches any kind of a node. `index_check` could be `None`, a boolean
42 + # value, a string value, or a number. `None` and `False` match against
43 + # any _value_ of sequence and mapping nodes. `True` matches against
44 + # any _key_ of a mapping node. A string `index_check` matches against
45 + # a mapping value that corresponds to a scalar key which content is
46 + # equal to the `index_check` value. An integer `index_check` matches
47 + # against a sequence value with the index equal to `index_check`.
48 + if not 'yaml_path_resolvers' in cls.__dict__:
49 + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
50 + new_path = []
51 + for element in path:
52 + if isinstance(element, (list, tuple)):
53 + if len(element) == 2:
54 + node_check, index_check = element
55 + elif len(element) == 1:
56 + node_check = element[0]
57 + index_check = True
58 + else:
59 + raise ResolverError("Invalid path element: %s" % element)
60 + else:
61 + node_check = None
62 + index_check = element
63 + if node_check is str:
64 + node_check = ScalarNode
65 + elif node_check is list:
66 + node_check = SequenceNode
67 + elif node_check is dict:
68 + node_check = MappingNode
69 + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \
70 + and not isinstance(node_check, str) \
71 + and node_check is not None:
72 + raise ResolverError("Invalid node checker: %s" % node_check)
73 + if not isinstance(index_check, (str, int)) \
74 + and index_check is not None:
75 + raise ResolverError("Invalid index checker: %s" % index_check)
76 + new_path.append((node_check, index_check))
77 + if kind is str:
78 + kind = ScalarNode
79 + elif kind is list:
80 + kind = SequenceNode
81 + elif kind is dict:
82 + kind = MappingNode
83 + elif kind not in [ScalarNode, SequenceNode, MappingNode] \
84 + and kind is not None:
85 + raise ResolverError("Invalid node kind: %s" % kind)
86 + cls.yaml_path_resolvers[tuple(new_path), kind] = tag
87 +
88 + def descend_resolver(self, current_node, current_index):
89 + if not self.yaml_path_resolvers:
90 + return
91 + exact_paths = {}
92 + prefix_paths = []
93 + if current_node:
94 + depth = len(self.resolver_prefix_paths)
95 + for path, kind in self.resolver_prefix_paths[-1]:
96 + if self.check_resolver_prefix(depth, path, kind,
97 + current_node, current_index):
98 + if len(path) > depth:
99 + prefix_paths.append((path, kind))
100 + else:
101 + exact_paths[kind] = self.yaml_path_resolvers[path, kind]
102 + else:
103 + for path, kind in self.yaml_path_resolvers:
104 + if not path:
105 + exact_paths[kind] = self.yaml_path_resolvers[path, kind]
106 + else:
107 + prefix_paths.append((path, kind))
108 + self.resolver_exact_paths.append(exact_paths)
109 + self.resolver_prefix_paths.append(prefix_paths)
110 +
111 + def ascend_resolver(self):
112 + if not self.yaml_path_resolvers:
113 + return
114 + self.resolver_exact_paths.pop()
115 + self.resolver_prefix_paths.pop()
116 +
117 + def check_resolver_prefix(self, depth, path, kind,
118 + current_node, current_index):
119 + node_check, index_check = path[depth-1]
120 + if isinstance(node_check, str):
121 + if current_node.tag != node_check:
122 + return
123 + elif node_check is not None:
124 + if not isinstance(current_node, node_check):
125 + return
126 + if index_check is True and current_index is not None:
127 + return
128 + if (index_check is False or index_check is None) \
129 + and current_index is None:
130 + return
131 + if isinstance(index_check, str):
132 + if not (isinstance(current_index, ScalarNode)
133 + and index_check == current_index.value):
134 + return
135 + elif isinstance(index_check, int) and not isinstance(index_check, bool):
136 + if index_check != current_index:
137 + return
138 + return True
139 +
140 + def resolve(self, kind, value, implicit):
141 + if kind is ScalarNode and implicit[0]:
142 + if value == '':
143 + resolvers = self.yaml_implicit_resolvers.get('', [])
144 + else:
145 + resolvers = self.yaml_implicit_resolvers.get(value[0], [])
146 + resolvers += self.yaml_implicit_resolvers.get(None, [])
147 + for tag, regexp in resolvers:
148 + if regexp.match(value):
149 + return tag
150 + implicit = implicit[1]
151 + if self.yaml_path_resolvers:
152 + exact_paths = self.resolver_exact_paths[-1]
153 + if kind in exact_paths:
154 + return exact_paths[kind]
155 + if None in exact_paths:
156 + return exact_paths[None]
157 + if kind is ScalarNode:
158 + return self.DEFAULT_SCALAR_TAG
159 + elif kind is SequenceNode:
160 + return self.DEFAULT_SEQUENCE_TAG
161 + elif kind is MappingNode:
162 + return self.DEFAULT_MAPPING_TAG
163 +
164 +class Resolver(BaseResolver):
165 + pass
166 +
167 +Resolver.add_implicit_resolver(
168 + 'tag:yaml.org,2002:bool',
169 + re.compile(r'''^(?:yes|Yes|YES|no|No|NO
170 + |true|True|TRUE|false|False|FALSE
171 + |on|On|ON|off|Off|OFF)$''', re.X),
172 + list('yYnNtTfFoO'))
173 +
174 +Resolver.add_implicit_resolver(
175 + 'tag:yaml.org,2002:float',
176 + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?
177 + |\.[0-9_]+(?:[eE][-+][0-9]+)?
178 + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
179 + |[-+]?\.(?:inf|Inf|INF)
180 + |\.(?:nan|NaN|NAN))$''', re.X),
181 + list('-+0123456789.'))
182 +
183 +Resolver.add_implicit_resolver(
184 + 'tag:yaml.org,2002:int',
185 + re.compile(r'''^(?:[-+]?0b[0-1_]+
186 + |[-+]?0[0-7_]+
187 + |[-+]?(?:0|[1-9][0-9_]*)
188 + |[-+]?0x[0-9a-fA-F_]+
189 + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
190 + list('-+0123456789'))
191 +
192 +Resolver.add_implicit_resolver(
193 + 'tag:yaml.org,2002:merge',
194 + re.compile(r'^(?:<<)$'),
195 + ['<'])
196 +
197 +Resolver.add_implicit_resolver(
198 + 'tag:yaml.org,2002:null',
199 + re.compile(r'''^(?: ~
200 + |null|Null|NULL
201 + | )$''', re.X),
202 + ['~', 'n', 'N', ''])
203 +
204 +Resolver.add_implicit_resolver(
205 + 'tag:yaml.org,2002:timestamp',
206 + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
207 + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
208 + (?:[Tt]|[ \t]+)[0-9][0-9]?
209 + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)?
210 + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
211 + list('0123456789'))
212 +
213 +Resolver.add_implicit_resolver(
214 + 'tag:yaml.org,2002:value',
215 + re.compile(r'^(?:=)$'),
216 + ['='])
217 +
218 +# The following resolver is only for documentation purposes. It cannot work
219 +# because plain scalars cannot start with '!', '&', or '*'.
220 +Resolver.add_implicit_resolver(
221 + 'tag:yaml.org,2002:yaml',
222 + re.compile(r'^(?:!|&|\*)$'),
223 + list('!&*'))
224 +
python.d/python_modules/pyyaml3/scanner.py new
+1448
@@ -0,0 +1,1448 @@
1 +
2 +# Scanner produces tokens of the following types:
3 +# STREAM-START
4 +# STREAM-END
5 +# DIRECTIVE(name, value)
6 +# DOCUMENT-START
7 +# DOCUMENT-END
8 +# BLOCK-SEQUENCE-START
9 +# BLOCK-MAPPING-START
10 +# BLOCK-END
11 +# FLOW-SEQUENCE-START
12 +# FLOW-MAPPING-START
13 +# FLOW-SEQUENCE-END
14 +# FLOW-MAPPING-END
15 +# BLOCK-ENTRY
16 +# FLOW-ENTRY
17 +# KEY
18 +# VALUE
19 +# ALIAS(value)
20 +# ANCHOR(value)
21 +# TAG(value)
22 +# SCALAR(value, plain, style)
23 +#
24 +# Read comments in the Scanner code for more details.
25 +#
26 +
27 +__all__ = ['Scanner', 'ScannerError']
28 +
29 +from .error import MarkedYAMLError
30 +from .tokens import *
31 +
32 +class ScannerError(MarkedYAMLError):
33 + pass
34 +
35 +class SimpleKey:
36 + # See below simple keys treatment.
37 +
38 + def __init__(self, token_number, required, index, line, column, mark):
39 + self.token_number = token_number
40 + self.required = required
41 + self.index = index
42 + self.line = line
43 + self.column = column
44 + self.mark = mark
45 +
46 +class Scanner:
47 +
48 + def __init__(self):
49 + """Initialize the scanner."""
50 + # It is assumed that Scanner and Reader will have a common descendant.
51 + # Reader do the dirty work of checking for BOM and converting the
52 + # input data to Unicode. It also adds NUL to the end.
53 + #
54 + # Reader supports the following methods
55 + # self.peek(i=0) # peek the next i-th character
56 + # self.prefix(l=1) # peek the next l characters
57 + # self.forward(l=1) # read the next l characters and move the pointer.
58 +
59 + # Had we reached the end of the stream?
60 + self.done = False
61 +
62 + # The number of unclosed '{' and '['. `flow_level == 0` means block
63 + # context.
64 + self.flow_level = 0
65 +
66 + # List of processed tokens that are not yet emitted.
67 + self.tokens = []
68 +
69 + # Add the STREAM-START token.
70 + self.fetch_stream_start()
71 +
72 + # Number of tokens that were emitted through the `get_token` method.
73 + self.tokens_taken = 0
74 +
75 + # The current indentation level.
76 + self.indent = -1
77 +
78 + # Past indentation levels.
79 + self.indents = []
80 +
81 + # Variables related to simple keys treatment.
82 +
83 + # A simple key is a key that is not denoted by the '?' indicator.
84 + # Example of simple keys:
85 + # ---
86 + # block simple key: value
87 + # ? not a simple key:
88 + # : { flow simple key: value }
89 + # We emit the KEY token before all keys, so when we find a potential
90 + # simple key, we try to locate the corresponding ':' indicator.
91 + # Simple keys should be limited to a single line and 1024 characters.
92 +
93 + # Can a simple key start at the current position? A simple key may
94 + # start:
95 + # - at the beginning of the line, not counting indentation spaces
96 + # (in block context),
97 + # - after '{', '[', ',' (in the flow context),
98 + # - after '?', ':', '-' (in the block context).
99 + # In the block context, this flag also signifies if a block collection
100 + # may start at the current position.
101 + self.allow_simple_key = True
102 +
103 + # Keep track of possible simple keys. This is a dictionary. The key
104 + # is `flow_level`; there can be no more that one possible simple key
105 + # for each level. The value is a SimpleKey record:
106 + # (token_number, required, index, line, column, mark)
107 + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
108 + # '[', or '{' tokens.
109 + self.possible_simple_keys = {}
110 +
111 + # Public methods.
112 +
113 + def check_token(self, *choices):
114 + # Check if the next token is one of the given types.
115 + while self.need_more_tokens():
116 + self.fetch_more_tokens()
117 + if self.tokens:
118 + if not choices:
119 + return True
120 + for choice in choices:
121 + if isinstance(self.tokens[0], choice):
122 + return True
123 + return False
124 +
125 + def peek_token(self):
126 + # Return the next token, but do not delete if from the queue.
127 + while self.need_more_tokens():
128 + self.fetch_more_tokens()
129 + if self.tokens:
130 + return self.tokens[0]
131 +
132 + def get_token(self):
133 + # Return the next token.
134 + while self.need_more_tokens():
135 + self.fetch_more_tokens()
136 + if self.tokens:
137 + self.tokens_taken += 1
138 + return self.tokens.pop(0)
139 +
140 + # Private methods.
141 +
142 + def need_more_tokens(self):
143 + if self.done:
144 + return False
145 + if not self.tokens:
146 + return True
147 + # The current token may be a potential simple key, so we
148 + # need to look further.
149 + self.stale_possible_simple_keys()
150 + if self.next_possible_simple_key() == self.tokens_taken:
151 + return True
152 +
153 + def fetch_more_tokens(self):
154 +
155 + # Eat whitespaces and comments until we reach the next token.
156 + self.scan_to_next_token()
157 +
158 + # Remove obsolete possible simple keys.
159 + self.stale_possible_simple_keys()
160 +
161 + # Compare the current indentation and column. It may add some tokens
162 + # and decrease the current indentation level.
163 + self.unwind_indent(self.column)
164 +
165 + # Peek the next character.
166 + ch = self.peek()
167 +
168 + # Is it the end of stream?
169 + if ch == '\0':
170 + return self.fetch_stream_end()
171 +
172 + # Is it a directive?
173 + if ch == '%' and self.check_directive():
174 + return self.fetch_directive()
175 +
176 + # Is it the document start?
177 + if ch == '-' and self.check_document_start():
178 + return self.fetch_document_start()
179 +
180 + # Is it the document end?
181 + if ch == '.' and self.check_document_end():
182 + return self.fetch_document_end()
183 +
184 + # TODO: support for BOM within a stream.
185 + #if ch == '\uFEFF':
186 + # return self.fetch_bom() <-- issue BOMToken
187 +
188 + # Note: the order of the following checks is NOT significant.
189 +
190 + # Is it the flow sequence start indicator?
191 + if ch == '[':
192 + return self.fetch_flow_sequence_start()
193 +
194 + # Is it the flow mapping start indicator?
195 + if ch == '{':
196 + return self.fetch_flow_mapping_start()
197 +
198 + # Is it the flow sequence end indicator?
199 + if ch == ']':
200 + return self.fetch_flow_sequence_end()
201 +
202 + # Is it the flow mapping end indicator?
203 + if ch == '}':
204 + return self.fetch_flow_mapping_end()
205 +
206 + # Is it the flow entry indicator?
207 + if ch == ',':
208 + return self.fetch_flow_entry()
209 +
210 + # Is it the block entry indicator?
211 + if ch == '-' and self.check_block_entry():
212 + return self.fetch_block_entry()
213 +
214 + # Is it the key indicator?
215 + if ch == '?' and self.check_key():
216 + return self.fetch_key()
217 +
218 + # Is it the value indicator?
219 + if ch == ':' and self.check_value():
220 + return self.fetch_value()
221 +
222 + # Is it an alias?
223 + if ch == '*':
224 + return self.fetch_alias()
225 +
226 + # Is it an anchor?
227 + if ch == '&':
228 + return self.fetch_anchor()
229 +
230 + # Is it a tag?
231 + if ch == '!':
232 + return self.fetch_tag()
233 +
234 + # Is it a literal scalar?
235 + if ch == '|' and not self.flow_level:
236 + return self.fetch_literal()
237 +
238 + # Is it a folded scalar?
239 + if ch == '>' and not self.flow_level:
240 + return self.fetch_folded()
241 +
242 + # Is it a single quoted scalar?
243 + if ch == '\'':
244 + return self.fetch_single()
245 +
246 + # Is it a double quoted scalar?
247 + if ch == '\"':
248 + return self.fetch_double()
249 +
250 + # It must be a plain scalar then.
251 + if self.check_plain():
252 + return self.fetch_plain()
253 +
254 + # No? It's an error. Let's produce a nice error message.
255 + raise ScannerError("while scanning for the next token", None,
256 + "found character %r that cannot start any token" % ch,
257 + self.get_mark())
258 +
259 + # Simple keys treatment.
260 +
261 + def next_possible_simple_key(self):
262 + # Return the number of the nearest possible simple key. Actually we
263 + # don't need to loop through the whole dictionary. We may replace it
264 + # with the following code:
265 + # if not self.possible_simple_keys:
266 + # return None
267 + # return self.possible_simple_keys[
268 + # min(self.possible_simple_keys.keys())].token_number
269 + min_token_number = None
270 + for level in self.possible_simple_keys:
271 + key = self.possible_simple_keys[level]
272 + if min_token_number is None or key.token_number < min_token_number:
273 + min_token_number = key.token_number
274 + return min_token_number
275 +
276 + def stale_possible_simple_keys(self):
277 + # Remove entries that are no longer possible simple keys. According to
278 + # the YAML specification, simple keys
279 + # - should be limited to a single line,
280 + # - should be no longer than 1024 characters.
281 + # Disabling this procedure will allow simple keys of any length and
282 + # height (may cause problems if indentation is broken though).
283 + for level in list(self.possible_simple_keys):
284 + key = self.possible_simple_keys[level]
285 + if key.line != self.line \
286 + or self.index-key.index > 1024:
287 + if key.required:
288 + raise ScannerError("while scanning a simple key", key.mark,
289 + "could not found expected ':'", self.get_mark())
290 + del self.possible_simple_keys[level]
291 +
292 + def save_possible_simple_key(self):
293 + # The next token may start a simple key. We check if it's possible
294 + # and save its position. This function is called for
295 + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
296 +
297 + # Check if a simple key is required at the current position.
298 + required = not self.flow_level and self.indent == self.column
299 +
300 + # A simple key is required only if it is the first token in the current
301 + # line. Therefore it is always allowed.
302 + assert self.allow_simple_key or not required
303 +
304 + # The next token might be a simple key. Let's save it's number and
305 + # position.
306 + if self.allow_simple_key:
307 + self.remove_possible_simple_key()
308 + token_number = self.tokens_taken+len(self.tokens)
309 + key = SimpleKey(token_number, required,
310 + self.index, self.line, self.column, self.get_mark())
311 + self.possible_simple_keys[self.flow_level] = key
312 +
313 + def remove_possible_simple_key(self):
314 + # Remove the saved possible key position at the current flow level.
315 + if self.flow_level in self.possible_simple_keys:
316 + key = self.possible_simple_keys[self.flow_level]
317 +
318 + if key.required:
319 + raise ScannerError("while scanning a simple key", key.mark,
320 + "could not found expected ':'", self.get_mark())
321 +
322 + del self.possible_simple_keys[self.flow_level]
323 +
324 + # Indentation functions.
325 +
326 + def unwind_indent(self, column):
327 +
328 + ## In flow context, tokens should respect indentation.
329 + ## Actually the condition should be `self.indent >= column` according to
330 + ## the spec. But this condition will prohibit intuitively correct
331 + ## constructions such as
332 + ## key : {
333 + ## }
334 + #if self.flow_level and self.indent > column:
335 + # raise ScannerError(None, None,
336 + # "invalid intendation or unclosed '[' or '{'",
337 + # self.get_mark())
338 +
339 + # In the flow context, indentation is ignored. We make the scanner less
340 + # restrictive then specification requires.
341 + if self.flow_level:
342 + return
343 +
344 + # In block context, we may need to issue the BLOCK-END tokens.
345 + while self.indent > column:
346 + mark = self.get_mark()
347 + self.indent = self.indents.pop()
348 + self.tokens.append(BlockEndToken(mark, mark))
349 +
350 + def add_indent(self, column):
351 + # Check if we need to increase indentation.
352 + if self.indent < column:
353 + self.indents.append(self.indent)
354 + self.indent = column
355 + return True
356 + return False
357 +
358 + # Fetchers.
359 +
360 + def fetch_stream_start(self):
361 + # We always add STREAM-START as the first token and STREAM-END as the
362 + # last token.
363 +
364 + # Read the token.
365 + mark = self.get_mark()
366 +
367 + # Add STREAM-START.
368 + self.tokens.append(StreamStartToken(mark, mark,
369 + encoding=self.encoding))
370 +
371 +
372 + def fetch_stream_end(self):
373 +
374 + # Set the current intendation to -1.
375 + self.unwind_indent(-1)
376 +
377 + # Reset simple keys.
378 + self.remove_possible_simple_key()
379 + self.allow_simple_key = False
380 + self.possible_simple_keys = {}
381 +
382 + # Read the token.
383 + mark = self.get_mark()
384 +
385 + # Add STREAM-END.
386 + self.tokens.append(StreamEndToken(mark, mark))
387 +
388 + # The steam is finished.
389 + self.done = True
390 +
391 + def fetch_directive(self):
392 +
393 + # Set the current intendation to -1.
394 + self.unwind_indent(-1)
395 +
396 + # Reset simple keys.
397 + self.remove_possible_simple_key()
398 + self.allow_simple_key = False
399 +
400 + # Scan and add DIRECTIVE.
401 + self.tokens.append(self.scan_directive())
402 +
403 + def fetch_document_start(self):
404 + self.fetch_document_indicator(DocumentStartToken)
405 +
406 + def fetch_document_end(self):
407 + self.fetch_document_indicator(DocumentEndToken)
408 +
409 + def fetch_document_indicator(self, TokenClass):
410 +
411 + # Set the current intendation to -1.
412 + self.unwind_indent(-1)
413 +
414 + # Reset simple keys. Note that there could not be a block collection
415 + # after '---'.
416 + self.remove_possible_simple_key()
417 + self.allow_simple_key = False
418 +
419 + # Add DOCUMENT-START or DOCUMENT-END.
420 + start_mark = self.get_mark()
421 + self.forward(3)
422 + end_mark = self.get_mark()
423 + self.tokens.append(TokenClass(start_mark, end_mark))
424 +
425 + def fetch_flow_sequence_start(self):
426 + self.fetch_flow_collection_start(FlowSequenceStartToken)
427 +
428 + def fetch_flow_mapping_start(self):
429 + self.fetch_flow_collection_start(FlowMappingStartToken)
430 +
431 + def fetch_flow_collection_start(self, TokenClass):
432 +
433 + # '[' and '{' may start a simple key.
434 + self.save_possible_simple_key()
435 +
436 + # Increase the flow level.
437 + self.flow_level += 1
438 +
439 + # Simple keys are allowed after '[' and '{'.
440 + self.allow_simple_key = True
441 +
442 + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
443 + start_mark = self.get_mark()
444 + self.forward()
445 + end_mark = self.get_mark()
446 + self.tokens.append(TokenClass(start_mark, end_mark))
447 +
448 + def fetch_flow_sequence_end(self):
449 + self.fetch_flow_collection_end(FlowSequenceEndToken)
450 +
451 + def fetch_flow_mapping_end(self):
452 + self.fetch_flow_collection_end(FlowMappingEndToken)
453 +
454 + def fetch_flow_collection_end(self, TokenClass):
455 +
456 + # Reset possible simple key on the current level.
457 + self.remove_possible_simple_key()
458 +
459 + # Decrease the flow level.
460 + self.flow_level -= 1
461 +
462 + # No simple keys after ']' or '}'.
463 + self.allow_simple_key = False
464 +
465 + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
466 + start_mark = self.get_mark()
467 + self.forward()
468 + end_mark = self.get_mark()
469 + self.tokens.append(TokenClass(start_mark, end_mark))
470 +
471 + def fetch_flow_entry(self):
472 +
473 + # Simple keys are allowed after ','.
474 + self.allow_simple_key = True
475 +
476 + # Reset possible simple key on the current level.
477 + self.remove_possible_simple_key()
478 +
479 + # Add FLOW-ENTRY.
480 + start_mark = self.get_mark()
481 + self.forward()
482 + end_mark = self.get_mark()
483 + self.tokens.append(FlowEntryToken(start_mark, end_mark))
484 +
485 + def fetch_block_entry(self):
486 +
487 + # Block context needs additional checks.
488 + if not self.flow_level:
489 +
490 + # Are we allowed to start a new entry?
491 + if not self.allow_simple_key:
492 + raise ScannerError(None, None,
493 + "sequence entries are not allowed here",
494 + self.get_mark())
495 +
496 + # We may need to add BLOCK-SEQUENCE-START.
497 + if self.add_indent(self.column):
498 + mark = self.get_mark()
499 + self.tokens.append(BlockSequenceStartToken(mark, mark))
500 +
501 + # It's an error for the block entry to occur in the flow context,
502 + # but we let the parser detect this.
503 + else:
504 + pass
505 +
506 + # Simple keys are allowed after '-'.
507 + self.allow_simple_key = True
508 +
509 + # Reset possible simple key on the current level.
510 + self.remove_possible_simple_key()
511 +
512 + # Add BLOCK-ENTRY.
513 + start_mark = self.get_mark()
514 + self.forward()
515 + end_mark = self.get_mark()
516 + self.tokens.append(BlockEntryToken(start_mark, end_mark))
517 +
518 + def fetch_key(self):
519 +
520 + # Block context needs additional checks.
521 + if not self.flow_level:
522 +
523 + # Are we allowed to start a key (not nessesary a simple)?
524 + if not self.allow_simple_key:
525 + raise ScannerError(None, None,
526 + "mapping keys are not allowed here",
527 + self.get_mark())
528 +
529 + # We may need to add BLOCK-MAPPING-START.
530 + if self.add_indent(self.column):
531 + mark = self.get_mark()
532 + self.tokens.append(BlockMappingStartToken(mark, mark))
533 +
534 + # Simple keys are allowed after '?' in the block context.
535 + self.allow_simple_key = not self.flow_level
536 +
537 + # Reset possible simple key on the current level.
538 + self.remove_possible_simple_key()
539 +
540 + # Add KEY.
541 + start_mark = self.get_mark()
542 + self.forward()
543 + end_mark = self.get_mark()
544 + self.tokens.append(KeyToken(start_mark, end_mark))
545 +
546 + def fetch_value(self):
547 +
548 + # Do we determine a simple key?
549 + if self.flow_level in self.possible_simple_keys:
550 +
551 + # Add KEY.
552 + key = self.possible_simple_keys[self.flow_level]
553 + del self.possible_simple_keys[self.flow_level]
554 + self.tokens.insert(key.token_number-self.tokens_taken,
555 + KeyToken(key.mark, key.mark))
556 +
557 + # If this key starts a new block mapping, we need to add
558 + # BLOCK-MAPPING-START.
559 + if not self.flow_level:
560 + if self.add_indent(key.column):
561 + self.tokens.insert(key.token_number-self.tokens_taken,
562 + BlockMappingStartToken(key.mark, key.mark))
563 +
564 + # There cannot be two simple keys one after another.
565 + self.allow_simple_key = False
566 +
567 + # It must be a part of a complex key.
568 + else:
569 +
570 + # Block context needs additional checks.
571 + # (Do we really need them? They will be catched by the parser
572 + # anyway.)
573 + if not self.flow_level:
574 +
575 + # We are allowed to start a complex value if and only if
576 + # we can start a simple key.
577 + if not self.allow_simple_key:
578 + raise ScannerError(None, None,
579 + "mapping values are not allowed here",
580 + self.get_mark())
581 +
582 + # If this value starts a new block mapping, we need to add
583 + # BLOCK-MAPPING-START. It will be detected as an error later by
584 + # the parser.
585 + if not self.flow_level:
586 + if self.add_indent(self.column):
587 + mark = self.get_mark()
588 + self.tokens.append(BlockMappingStartToken(mark, mark))
589 +
590 + # Simple keys are allowed after ':' in the block context.
591 + self.allow_simple_key = not self.flow_level
592 +
593 + # Reset possible simple key on the current level.
594 + self.remove_possible_simple_key()
595 +
596 + # Add VALUE.
597 + start_mark = self.get_mark()
598 + self.forward()
599 + end_mark = self.get_mark()
600 + self.tokens.append(ValueToken(start_mark, end_mark))
601 +
602 + def fetch_alias(self):
603 +
604 + # ALIAS could be a simple key.
605 + self.save_possible_simple_key()
606 +
607 + # No simple keys after ALIAS.
608 + self.allow_simple_key = False
609 +
610 + # Scan and add ALIAS.
611 + self.tokens.append(self.scan_anchor(AliasToken))
612 +
613 + def fetch_anchor(self):
614 +
615 + # ANCHOR could start a simple key.
616 + self.save_possible_simple_key()
617 +
618 + # No simple keys after ANCHOR.
619 + self.allow_simple_key = False
620 +
621 + # Scan and add ANCHOR.
622 + self.tokens.append(self.scan_anchor(AnchorToken))
623 +
624 + def fetch_tag(self):
625 +
626 + # TAG could start a simple key.
627 + self.save_possible_simple_key()
628 +
629 + # No simple keys after TAG.
630 + self.allow_simple_key = False
631 +
632 + # Scan and add TAG.
633 + self.tokens.append(self.scan_tag())
634 +
635 + def fetch_literal(self):
636 + self.fetch_block_scalar(style='|')
637 +
638 + def fetch_folded(self):
639 + self.fetch_block_scalar(style='>')
640 +
641 + def fetch_block_scalar(self, style):
642 +
643 + # A simple key may follow a block scalar.
644 + self.allow_simple_key = True
645 +
646 + # Reset possible simple key on the current level.
647 + self.remove_possible_simple_key()
648 +
649 + # Scan and add SCALAR.
650 + self.tokens.append(self.scan_block_scalar(style))
651 +
652 + def fetch_single(self):
653 + self.fetch_flow_scalar(style='\'')
654 +
655 + def fetch_double(self):
656 + self.fetch_flow_scalar(style='"')
657 +
658 + def fetch_flow_scalar(self, style):
659 +
660 + # A flow scalar could be a simple key.
661 + self.save_possible_simple_key()
662 +
663 + # No simple keys after flow scalars.
664 + self.allow_simple_key = False
665 +
666 + # Scan and add SCALAR.
667 + self.tokens.append(self.scan_flow_scalar(style))
668 +
669 + def fetch_plain(self):
670 +
671 + # A plain scalar could be a simple key.
672 + self.save_possible_simple_key()
673 +
674 + # No simple keys after plain scalars. But note that `scan_plain` will
675 + # change this flag if the scan is finished at the beginning of the
676 + # line.
677 + self.allow_simple_key = False
678 +
679 + # Scan and add SCALAR. May change `allow_simple_key`.
680 + self.tokens.append(self.scan_plain())
681 +
682 + # Checkers.
683 +
684 + def check_directive(self):
685 +
686 + # DIRECTIVE: ^ '%' ...
687 + # The '%' indicator is already checked.
688 + if self.column == 0:
689 + return True
690 +
691 + def check_document_start(self):
692 +
693 + # DOCUMENT-START: ^ '---' (' '|'\n')
694 + if self.column == 0:
695 + if self.prefix(3) == '---' \
696 + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
697 + return True
698 +
699 + def check_document_end(self):
700 +
701 + # DOCUMENT-END: ^ '...' (' '|'\n')
702 + if self.column == 0:
703 + if self.prefix(3) == '...' \
704 + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
705 + return True
706 +
707 + def check_block_entry(self):
708 +
709 + # BLOCK-ENTRY: '-' (' '|'\n')
710 + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
711 +
712 + def check_key(self):
713 +
714 + # KEY(flow context): '?'
715 + if self.flow_level:
716 + return True
717 +
718 + # KEY(block context): '?' (' '|'\n')
719 + else:
720 + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
721 +
722 + def check_value(self):
723 +
724 + # VALUE(flow context): ':'
725 + if self.flow_level:
726 + return True
727 +
728 + # VALUE(block context): ':' (' '|'\n')
729 + else:
730 + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
731 +
732 + def check_plain(self):
733 +
734 + # A plain scalar may start with any non-space character except:
735 + # '-', '?', ':', ',', '[', ']', '{', '}',
736 + # '#', '&', '*', '!', '|', '>', '\'', '\"',
737 + # '%', '@', '`'.
738 + #
739 + # It may also start with
740 + # '-', '?', ':'
741 + # if it is followed by a non-space character.
742 + #
743 + # Note that we limit the last rule to the block context (except the
744 + # '-' character) because we want the flow context to be space
745 + # independent.
746 + ch = self.peek()
747 + return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \
748 + or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029'
749 + and (ch == '-' or (not self.flow_level and ch in '?:')))
750 +
751 + # Scanners.
752 +
753 + def scan_to_next_token(self):
754 + # We ignore spaces, line breaks and comments.
755 + # If we find a line break in the block context, we set the flag
756 + # `allow_simple_key` on.
757 + # The byte order mark is stripped if it's the first character in the
758 + # stream. We do not yet support BOM inside the stream as the
759 + # specification requires. Any such mark will be considered as a part
760 + # of the document.
761 + #
762 + # TODO: We need to make tab handling rules more sane. A good rule is
763 + # Tabs cannot precede tokens
764 + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
765 + # KEY(block), VALUE(block), BLOCK-ENTRY
766 + # So the checking code is
767 + # if <TAB>:
768 + # self.allow_simple_keys = False
769 + # We also need to add the check for `allow_simple_keys == True` to
770 + # `unwind_indent` before issuing BLOCK-END.
771 + # Scanners for block, flow, and plain scalars need to be modified.
772 +
773 + if self.index == 0 and self.peek() == '\uFEFF':
774 + self.forward()
775 + found = False
776 + while not found:
777 + while self.peek() == ' ':
778 + self.forward()
779 + if self.peek() == '#':
780 + while self.peek() not in '\0\r\n\x85\u2028\u2029':
781 + self.forward()
782 + if self.scan_line_break():
783 + if not self.flow_level:
784 + self.allow_simple_key = True
785 + else:
786 + found = True
787 +
788 + def scan_directive(self):
789 + # See the specification for details.
790 + start_mark = self.get_mark()
791 + self.forward()
792 + name = self.scan_directive_name(start_mark)
793 + value = None
794 + if name == 'YAML':
795 + value = self.scan_yaml_directive_value(start_mark)
796 + end_mark = self.get_mark()
797 + elif name == 'TAG':
798 + value = self.scan_tag_directive_value(start_mark)
799 + end_mark = self.get_mark()
800 + else:
801 + end_mark = self.get_mark()
802 + while self.peek() not in '\0\r\n\x85\u2028\u2029':
803 + self.forward()
804 + self.scan_directive_ignored_line(start_mark)
805 + return DirectiveToken(name, value, start_mark, end_mark)
806 +
807 + def scan_directive_name(self, start_mark):
808 + # See the specification for details.
809 + length = 0
810 + ch = self.peek(length)
811 + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
812 + or ch in '-_':
813 + length += 1
814 + ch = self.peek(length)
815 + if not length:
816 + raise ScannerError("while scanning a directive", start_mark,
817 + "expected alphabetic or numeric character, but found %r"
818 + % ch, self.get_mark())
819 + value = self.prefix(length)
820 + self.forward(length)
821 + ch = self.peek()
822 + if ch not in '\0 \r\n\x85\u2028\u2029':
823 + raise ScannerError("while scanning a directive", start_mark,
824 + "expected alphabetic or numeric character, but found %r"
825 + % ch, self.get_mark())
826 + return value
827 +
828 + def scan_yaml_directive_value(self, start_mark):
829 + # See the specification for details.
830 + while self.peek() == ' ':
831 + self.forward()
832 + major = self.scan_yaml_directive_number(start_mark)
833 + if self.peek() != '.':
834 + raise ScannerError("while scanning a directive", start_mark,
835 + "expected a digit or '.', but found %r" % self.peek(),
836 + self.get_mark())
837 + self.forward()
838 + minor = self.scan_yaml_directive_number(start_mark)
839 + if self.peek() not in '\0 \r\n\x85\u2028\u2029':
840 + raise ScannerError("while scanning a directive", start_mark,
841 + "expected a digit or ' ', but found %r" % self.peek(),
842 + self.get_mark())
843 + return (major, minor)
844 +
845 + def scan_yaml_directive_number(self, start_mark):
846 + # See the specification for details.
847 + ch = self.peek()
848 + if not ('0' <= ch <= '9'):
849 + raise ScannerError("while scanning a directive", start_mark,
850 + "expected a digit, but found %r" % ch, self.get_mark())
851 + length = 0
852 + while '0' <= self.peek(length) <= '9':
853 + length += 1
854 + value = int(self.prefix(length))
855 + self.forward(length)
856 + return value
857 +
858 + def scan_tag_directive_value(self, start_mark):
859 + # See the specification for details.
860 + while self.peek() == ' ':
861 + self.forward()
862 + handle = self.scan_tag_directive_handle(start_mark)
863 + while self.peek() == ' ':
864 + self.forward()
865 + prefix = self.scan_tag_directive_prefix(start_mark)
866 + return (handle, prefix)
867 +
868 + def scan_tag_directive_handle(self, start_mark):
869 + # See the specification for details.
870 + value = self.scan_tag_handle('directive', start_mark)
871 + ch = self.peek()
872 + if ch != ' ':
873 + raise ScannerError("while scanning a directive", start_mark,
874 + "expected ' ', but found %r" % ch, self.get_mark())
875 + return value
876 +
877 + def scan_tag_directive_prefix(self, start_mark):
878 + # See the specification for details.
879 + value = self.scan_tag_uri('directive', start_mark)
880 + ch = self.peek()
881 + if ch not in '\0 \r\n\x85\u2028\u2029':
882 + raise ScannerError("while scanning a directive", start_mark,
883 + "expected ' ', but found %r" % ch, self.get_mark())
884 + return value
885 +
886 + def scan_directive_ignored_line(self, start_mark):
887 + # See the specification for details.
888 + while self.peek() == ' ':
889 + self.forward()
890 + if self.peek() == '#':
891 + while self.peek() not in '\0\r\n\x85\u2028\u2029':
892 + self.forward()
893 + ch = self.peek()
894 + if ch not in '\0\r\n\x85\u2028\u2029':
895 + raise ScannerError("while scanning a directive", start_mark,
896 + "expected a comment or a line break, but found %r"
897 + % ch, self.get_mark())
898 + self.scan_line_break()
899 +
900 + def scan_anchor(self, TokenClass):
901 + # The specification does not restrict characters for anchors and
902 + # aliases. This may lead to problems, for instance, the document:
903 + # [ *alias, value ]
904 + # can be interpteted in two ways, as
905 + # [ "value" ]
906 + # and
907 + # [ *alias , "value" ]
908 + # Therefore we restrict aliases to numbers and ASCII letters.
909 + start_mark = self.get_mark()
910 + indicator = self.peek()
911 + if indicator == '*':
912 + name = 'alias'
913 + else:
914 + name = 'anchor'
915 + self.forward()
916 + length = 0
917 + ch = self.peek(length)
918 + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
919 + or ch in '-_':
920 + length += 1
921 + ch = self.peek(length)
922 + if not length:
923 + raise ScannerError("while scanning an %s" % name, start_mark,
924 + "expected alphabetic or numeric character, but found %r"
925 + % ch, self.get_mark())
926 + value = self.prefix(length)
927 + self.forward(length)
928 + ch = self.peek()
929 + if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
930 + raise ScannerError("while scanning an %s" % name, start_mark,
931 + "expected alphabetic or numeric character, but found %r"
932 + % ch, self.get_mark())
933 + end_mark = self.get_mark()
934 + return TokenClass(value, start_mark, end_mark)
935 +
936 + def scan_tag(self):
937 + # See the specification for details.
938 + start_mark = self.get_mark()
939 + ch = self.peek(1)
940 + if ch == '<':
941 + handle = None
942 + self.forward(2)
943 + suffix = self.scan_tag_uri('tag', start_mark)
944 + if self.peek() != '>':
945 + raise ScannerError("while parsing a tag", start_mark,
946 + "expected '>', but found %r" % self.peek(),
947 + self.get_mark())
948 + self.forward()
949 + elif ch in '\0 \t\r\n\x85\u2028\u2029':
950 + handle = None
951 + suffix = '!'
952 + self.forward()
953 + else:
954 + length = 1
955 + use_handle = False
956 + while ch not in '\0 \r\n\x85\u2028\u2029':
957 + if ch == '!':
958 + use_handle = True
959 + break
960 + length += 1
961 + ch = self.peek(length)
962 + handle = '!'
963 + if use_handle:
964 + handle = self.scan_tag_handle('tag', start_mark)
965 + else:
966 + handle = '!'
967 + self.forward()
968 + suffix = self.scan_tag_uri('tag', start_mark)
969 + ch = self.peek()
970 + if ch not in '\0 \r\n\x85\u2028\u2029':
971 + raise ScannerError("while scanning a tag", start_mark,
972 + "expected ' ', but found %r" % ch, self.get_mark())
973 + value = (handle, suffix)
974 + end_mark = self.get_mark()
975 + return TagToken(value, start_mark, end_mark)
976 +
977 + def scan_block_scalar(self, style):
978 + # See the specification for details.
979 +
980 + if style == '>':
981 + folded = True
982 + else:
983 + folded = False
984 +
985 + chunks = []
986 + start_mark = self.get_mark()
987 +
988 + # Scan the header.
989 + self.forward()
990 + chomping, increment = self.scan_block_scalar_indicators(start_mark)
991 + self.scan_block_scalar_ignored_line(start_mark)
992 +
993 + # Determine the indentation level and go to the first non-empty line.
994 + min_indent = self.indent+1
995 + if min_indent < 1:
996 + min_indent = 1
997 + if increment is None:
998 + breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
999 + indent = max(min_indent, max_indent)
1000 + else:
1001 + indent = min_indent+increment-1
1002 + breaks, end_mark = self.scan_block_scalar_breaks(indent)
1003 + line_break = ''
1004 +
1005 + # Scan the inner part of the block scalar.
1006 + while self.column == indent and self.peek() != '\0':
1007 + chunks.extend(breaks)
1008 + leading_non_space = self.peek() not in ' \t'
1009 + length = 0
1010 + while self.peek(length) not in '\0\r\n\x85\u2028\u2029':
1011 + length += 1
1012 + chunks.append(self.prefix(length))
1013 + self.forward(length)
1014 + line_break = self.scan_line_break()
1015 + breaks, end_mark = self.scan_block_scalar_breaks(indent)
1016 + if self.column == indent and self.peek() != '\0':
1017 +
1018 + # Unfortunately, folding rules are ambiguous.
1019 + #
1020 + # This is the folding according to the specification:
1021 +
1022 + if folded and line_break == '\n' \
1023 + and leading_non_space and self.peek() not in ' \t':
1024 + if not breaks:
1025 + chunks.append(' ')
1026 + else:
1027 + chunks.append(line_break)
1028 +
1029 + # This is Clark Evans's interpretation (also in the spec
1030 + # examples):
1031 + #
1032 + #if folded and line_break == '\n':
1033 + # if not breaks:
1034 + # if self.peek() not in ' \t':
1035 + # chunks.append(' ')
1036 + # else:
1037 + # chunks.append(line_break)
1038 + #else:
1039 + # chunks.append(line_break)
1040 + else:
1041 + break
1042 +
1043 + # Chomp the tail.
1044 + if chomping is not False:
1045 + chunks.append(line_break)
1046 + if chomping is True:
1047 + chunks.extend(breaks)
1048 +
1049 + # We are done.
1050 + return ScalarToken(''.join(chunks), False, start_mark, end_mark,
1051 + style)
1052 +
1053 + def scan_block_scalar_indicators(self, start_mark):
1054 + # See the specification for details.
1055 + chomping = None
1056 + increment = None
1057 + ch = self.peek()
1058 + if ch in '+-':
1059 + if ch == '+':
1060 + chomping = True
1061 + else:
1062 + chomping = False
1063 + self.forward()
1064 + ch = self.peek()
1065 + if ch in '0123456789':
1066 + increment = int(ch)
1067 + if increment == 0:
1068 + raise ScannerError("while scanning a block scalar", start_mark,
1069 + "expected indentation indicator in the range 1-9, but found 0",
1070 + self.get_mark())
1071 + self.forward()
1072 + elif ch in '0123456789':
1073 + increment = int(ch)
1074 + if increment == 0:
1075 + raise ScannerError("while scanning a block scalar", start_mark,
1076 + "expected indentation indicator in the range 1-9, but found 0",
1077 + self.get_mark())
1078 + self.forward()
1079 + ch = self.peek()
1080 + if ch in '+-':
1081 + if ch == '+':
1082 + chomping = True
1083 + else:
1084 + chomping = False
1085 + self.forward()
1086 + ch = self.peek()
1087 + if ch not in '\0 \r\n\x85\u2028\u2029':
1088 + raise ScannerError("while scanning a block scalar", start_mark,
1089 + "expected chomping or indentation indicators, but found %r"
1090 + % ch, self.get_mark())
1091 + return chomping, increment
1092 +
1093 + def scan_block_scalar_ignored_line(self, start_mark):
1094 + # See the specification for details.
1095 + while self.peek() == ' ':
1096 + self.forward()
1097 + if self.peek() == '#':
1098 + while self.peek() not in '\0\r\n\x85\u2028\u2029':
1099 + self.forward()
1100 + ch = self.peek()
1101 + if ch not in '\0\r\n\x85\u2028\u2029':
1102 + raise ScannerError("while scanning a block scalar", start_mark,
1103 + "expected a comment or a line break, but found %r" % ch,
1104 + self.get_mark())
1105 + self.scan_line_break()
1106 +
1107 + def scan_block_scalar_indentation(self):
1108 + # See the specification for details.
1109 + chunks = []
1110 + max_indent = 0
1111 + end_mark = self.get_mark()
1112 + while self.peek() in ' \r\n\x85\u2028\u2029':
1113 + if self.peek() != ' ':
1114 + chunks.append(self.scan_line_break())
1115 + end_mark = self.get_mark()
1116 + else:
1117 + self.forward()
1118 + if self.column > max_indent:
1119 + max_indent = self.column
1120 + return chunks, max_indent, end_mark
1121 +
1122 + def scan_block_scalar_breaks(self, indent):
1123 + # See the specification for details.
1124 + chunks = []
1125 + end_mark = self.get_mark()
1126 + while self.column < indent and self.peek() == ' ':
1127 + self.forward()
1128 + while self.peek() in '\r\n\x85\u2028\u2029':
1129 + chunks.append(self.scan_line_break())
1130 + end_mark = self.get_mark()
1131 + while self.column < indent and self.peek() == ' ':
1132 + self.forward()
1133 + return chunks, end_mark
1134 +
1135 + def scan_flow_scalar(self, style):
1136 + # See the specification for details.
1137 + # Note that we loose indentation rules for quoted scalars. Quoted
1138 + # scalars don't need to adhere indentation because " and ' clearly
1139 + # mark the beginning and the end of them. Therefore we are less
1140 + # restrictive then the specification requires. We only need to check
1141 + # that document separators are not included in scalars.
1142 + if style == '"':
1143 + double = True
1144 + else:
1145 + double = False
1146 + chunks = []
1147 + start_mark = self.get_mark()
1148 + quote = self.peek()
1149 + self.forward()
1150 + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1151 + while self.peek() != quote:
1152 + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
1153 + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1154 + self.forward()
1155 + end_mark = self.get_mark()
1156 + return ScalarToken(''.join(chunks), False, start_mark, end_mark,
1157 + style)
1158 +
1159 + ESCAPE_REPLACEMENTS = {
1160 + '0': '\0',
1161 + 'a': '\x07',
1162 + 'b': '\x08',
1163 + 't': '\x09',
1164 + '\t': '\x09',
1165 + 'n': '\x0A',
1166 + 'v': '\x0B',
1167 + 'f': '\x0C',
1168 + 'r': '\x0D',
1169 + 'e': '\x1B',
1170 + ' ': '\x20',
1171 + '\"': '\"',
1172 + '\\': '\\',
1173 + 'N': '\x85',
1174 + '_': '\xA0',
1175 + 'L': '\u2028',
1176 + 'P': '\u2029',
1177 + }
1178 +
1179 + ESCAPE_CODES = {
1180 + 'x': 2,
1181 + 'u': 4,
1182 + 'U': 8,
1183 + }
1184 +
1185 + def scan_flow_scalar_non_spaces(self, double, start_mark):
1186 + # See the specification for details.
1187 + chunks = []
1188 + while True:
1189 + length = 0
1190 + while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029':
1191 + length += 1
1192 + if length:
1193 + chunks.append(self.prefix(length))
1194 + self.forward(length)
1195 + ch = self.peek()
1196 + if not double and ch == '\'' and self.peek(1) == '\'':
1197 + chunks.append('\'')
1198 + self.forward(2)
1199 + elif (double and ch == '\'') or (not double and ch in '\"\\'):
1200 + chunks.append(ch)
1201 + self.forward()
1202 + elif double and ch == '\\':
1203 + self.forward()
1204 + ch = self.peek()
1205 + if ch in self.ESCAPE_REPLACEMENTS:
1206 + chunks.append(self.ESCAPE_REPLACEMENTS[ch])
1207 + self.forward()
1208 + elif ch in self.ESCAPE_CODES:
1209 + length = self.ESCAPE_CODES[ch]
1210 + self.forward()
1211 + for k in range(length):
1212 + if self.peek(k) not in '0123456789ABCDEFabcdef':
1213 + raise ScannerError("while scanning a double-quoted scalar", start_mark,
1214 + "expected escape sequence of %d hexdecimal numbers, but found %r" %
1215 + (length, self.peek(k)), self.get_mark())
1216 + code = int(self.prefix(length), 16)
1217 + chunks.append(chr(code))
1218 + self.forward(length)
1219 + elif ch in '\r\n\x85\u2028\u2029':
1220 + self.scan_line_break()
1221 + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
1222 + else:
1223 + raise ScannerError("while scanning a double-quoted scalar", start_mark,
1224 + "found unknown escape character %r" % ch, self.get_mark())
1225 + else:
1226 + return chunks
1227 +
1228 + def scan_flow_scalar_spaces(self, double, start_mark):
1229 + # See the specification for details.
1230 + chunks = []
1231 + length = 0
1232 + while self.peek(length) in ' \t':
1233 + length += 1
1234 + whitespaces = self.prefix(length)
1235 + self.forward(length)
1236 + ch = self.peek()
1237 + if ch == '\0':
1238 + raise ScannerError("while scanning a quoted scalar", start_mark,
1239 + "found unexpected end of stream", self.get_mark())
1240 + elif ch in '\r\n\x85\u2028\u2029':
1241 + line_break = self.scan_line_break()
1242 + breaks = self.scan_flow_scalar_breaks(double, start_mark)
1243 + if line_break != '\n':
1244 + chunks.append(line_break)
1245 + elif not breaks:
1246 + chunks.append(' ')
1247 + chunks.extend(breaks)
1248 + else:
1249 + chunks.append(whitespaces)
1250 + return chunks
1251 +
1252 + def scan_flow_scalar_breaks(self, double, start_mark):
1253 + # See the specification for details.
1254 + chunks = []
1255 + while True:
1256 + # Instead of checking indentation, we check for document
1257 + # separators.
1258 + prefix = self.prefix(3)
1259 + if (prefix == '---' or prefix == '...') \
1260 + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
1261 + raise ScannerError("while scanning a quoted scalar", start_mark,
1262 + "found unexpected document separator", self.get_mark())
1263 + while self.peek() in ' \t':
1264 + self.forward()
1265 + if self.peek() in '\r\n\x85\u2028\u2029':
1266 + chunks.append(self.scan_line_break())
1267 + else:
1268 + return chunks
1269 +
1270 + def scan_plain(self):
1271 + # See the specification for details.
1272 + # We add an additional restriction for the flow context:
1273 + # plain scalars in the flow context cannot contain ',', ':' and '?'.
1274 + # We also keep track of the `allow_simple_key` flag here.
1275 + # Indentation rules are loosed for the flow context.
1276 + chunks = []
1277 + start_mark = self.get_mark()
1278 + end_mark = start_mark
1279 + indent = self.indent+1
1280 + # We allow zero indentation for scalars, but then we need to check for
1281 + # document separators at the beginning of the line.
1282 + #if indent == 0:
1283 + # indent = 1
1284 + spaces = []
1285 + while True:
1286 + length = 0
1287 + if self.peek() == '#':
1288 + break
1289 + while True:
1290 + ch = self.peek(length)
1291 + if ch in '\0 \t\r\n\x85\u2028\u2029' \
1292 + or (not self.flow_level and ch == ':' and
1293 + self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029') \
1294 + or (self.flow_level and ch in ',:?[]{}'):
1295 + break
1296 + length += 1
1297 + # It's not clear what we should do with ':' in the flow context.
1298 + if (self.flow_level and ch == ':'
1299 + and self.peek(length+1) not in '\0 \t\r\n\x85\u2028\u2029,[]{}'):
1300 + self.forward(length)
1301 + raise ScannerError("while scanning a plain scalar", start_mark,
1302 + "found unexpected ':'", self.get_mark(),
1303 + "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
1304 + if length == 0:
1305 + break
1306 + self.allow_simple_key = False
1307 + chunks.extend(spaces)
1308 + chunks.append(self.prefix(length))
1309 + self.forward(length)
1310 + end_mark = self.get_mark()
1311 + spaces = self.scan_plain_spaces(indent, start_mark)
1312 + if not spaces or self.peek() == '#' \
1313 + or (not self.flow_level and self.column < indent):
1314 + break
1315 + return ScalarToken(''.join(chunks), True, start_mark, end_mark)
1316 +
1317 + def scan_plain_spaces(self, indent, start_mark):
1318 + # See the specification for details.
1319 + # The specification is really confusing about tabs in plain scalars.
1320 + # We just forbid them completely. Do not use tabs in YAML!
1321 + chunks = []
1322 + length = 0
1323 + while self.peek(length) in ' ':
1324 + length += 1
1325 + whitespaces = self.prefix(length)
1326 + self.forward(length)
1327 + ch = self.peek()
1328 + if ch in '\r\n\x85\u2028\u2029':
1329 + line_break = self.scan_line_break()
1330 + self.allow_simple_key = True
1331 + prefix = self.prefix(3)
1332 + if (prefix == '---' or prefix == '...') \
1333 + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
1334 + return
1335 + breaks = []
1336 + while self.peek() in ' \r\n\x85\u2028\u2029':
1337 + if self.peek() == ' ':
1338 + self.forward()
1339 + else:
1340 + breaks.append(self.scan_line_break())
1341 + prefix = self.prefix(3)
1342 + if (prefix == '---' or prefix == '...') \
1343 + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
1344 + return
1345 + if line_break != '\n':
1346 + chunks.append(line_break)
1347 + elif not breaks:
1348 + chunks.append(' ')
1349 + chunks.extend(breaks)
1350 + elif whitespaces:
1351 + chunks.append(whitespaces)
1352 + return chunks
1353 +
1354 + def scan_tag_handle(self, name, start_mark):
1355 + # See the specification for details.
1356 + # For some strange reasons, the specification does not allow '_' in
1357 + # tag handles. I have allowed it anyway.
1358 + ch = self.peek()
1359 + if ch != '!':
1360 + raise ScannerError("while scanning a %s" % name, start_mark,
1361 + "expected '!', but found %r" % ch, self.get_mark())
1362 + length = 1
1363 + ch = self.peek(length)
1364 + if ch != ' ':
1365 + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
1366 + or ch in '-_':
1367 + length += 1
1368 + ch = self.peek(length)
1369 + if ch != '!':
1370 + self.forward(length)
1371 + raise ScannerError("while scanning a %s" % name, start_mark,
1372 + "expected '!', but found %r" % ch, self.get_mark())
1373 + length += 1
1374 + value = self.prefix(length)
1375 + self.forward(length)
1376 + return value
1377 +
1378 + def scan_tag_uri(self, name, start_mark):
1379 + # See the specification for details.
1380 + # Note: we do not check if URI is well-formed.
1381 + chunks = []
1382 + length = 0
1383 + ch = self.peek(length)
1384 + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
1385 + or ch in '-;/?:@&=+$,_.!~*\'()[]%':
1386 + if ch == '%':
1387 + chunks.append(self.prefix(length))
1388 + self.forward(length)
1389 + length = 0
1390 + chunks.append(self.scan_uri_escapes(name, start_mark))
1391 + else:
1392 + length += 1
1393 + ch = self.peek(length)
1394 + if length:
1395 + chunks.append(self.prefix(length))
1396 + self.forward(length)
1397 + length = 0
1398 + if not chunks:
1399 + raise ScannerError("while parsing a %s" % name, start_mark,
1400 + "expected URI, but found %r" % ch, self.get_mark())
1401 + return ''.join(chunks)
1402 +
1403 + def scan_uri_escapes(self, name, start_mark):
1404 + # See the specification for details.
1405 + codes = []
1406 + mark = self.get_mark()
1407 + while self.peek() == '%':
1408 + self.forward()
1409 + for k in range(2):
1410 + if self.peek(k) not in '0123456789ABCDEFabcdef':
1411 + raise ScannerError("while scanning a %s" % name, start_mark,
1412 + "expected URI escape sequence of 2 hexdecimal numbers, but found %r"
1413 + % self.peek(k), self.get_mark())
1414 + codes.append(int(self.prefix(2), 16))
1415 + self.forward(2)
1416 + try:
1417 + value = bytes(codes).decode('utf-8')
1418 + except UnicodeDecodeError as exc:
1419 + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
1420 + return value
1421 +
1422 + def scan_line_break(self):
1423 + # Transforms:
1424 + # '\r\n' : '\n'
1425 + # '\r' : '\n'
1426 + # '\n' : '\n'
1427 + # '\x85' : '\n'
1428 + # '\u2028' : '\u2028'
1429 + # '\u2029 : '\u2029'
1430 + # default : ''
1431 + ch = self.peek()
1432 + if ch in '\r\n\x85':
1433 + if self.prefix(2) == '\r\n':
1434 + self.forward(2)
1435 + else:
1436 + self.forward()
1437 + return '\n'
1438 + elif ch in '\u2028\u2029':
1439 + self.forward()
1440 + return ch
1441 + return ''
1442 +
1443 +#try:
1444 +# import psyco
1445 +# psyco.bind(Scanner)
1446 +#except ImportError:
1447 +# pass
1448 +
python.d/python_modules/pyyaml3/serializer.py new
+111
@@ -0,0 +1,111 @@
1 +
2 +__all__ = ['Serializer', 'SerializerError']
3 +
4 +from .error import YAMLError
5 +from .events import *
6 +from .nodes import *
7 +
8 +class SerializerError(YAMLError):
9 + pass
10 +
11 +class Serializer:
12 +
13 + ANCHOR_TEMPLATE = 'id%03d'
14 +
15 + def __init__(self, encoding=None,
16 + explicit_start=None, explicit_end=None, version=None, tags=None):
17 + self.use_encoding = encoding
18 + self.use_explicit_start = explicit_start
19 + self.use_explicit_end = explicit_end
20 + self.use_version = version
21 + self.use_tags = tags
22 + self.serialized_nodes = {}
23 + self.anchors = {}
24 + self.last_anchor_id = 0
25 + self.closed = None
26 +
27 + def open(self):
28 + if self.closed is None:
29 + self.emit(StreamStartEvent(encoding=self.use_encoding))
30 + self.closed = False
31 + elif self.closed:
32 + raise SerializerError("serializer is closed")
33 + else:
34 + raise SerializerError("serializer is already opened")
35 +
36 + def close(self):
37 + if self.closed is None:
38 + raise SerializerError("serializer is not opened")
39 + elif not self.closed:
40 + self.emit(StreamEndEvent())
41 + self.closed = True
42 +
43 + #def __del__(self):
44 + # self.close()
45 +
46 + def serialize(self, node):
47 + if self.closed is None:
48 + raise SerializerError("serializer is not opened")
49 + elif self.closed:
50 + raise SerializerError("serializer is closed")
51 + self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
52 + version=self.use_version, tags=self.use_tags))
53 + self.anchor_node(node)
54 + self.serialize_node(node, None, None)
55 + self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
56 + self.serialized_nodes = {}
57 + self.anchors = {}
58 + self.last_anchor_id = 0
59 +
60 + def anchor_node(self, node):
61 + if node in self.anchors:
62 + if self.anchors[node] is None:
63 + self.anchors[node] = self.generate_anchor(node)
64 + else:
65 + self.anchors[node] = None
66 + if isinstance(node, SequenceNode):
67 + for item in node.value:
68 + self.anchor_node(item)
69 + elif isinstance(node, MappingNode):
70 + for key, value in node.value:
71 + self.anchor_node(key)
72 + self.anchor_node(value)
73 +
74 + def generate_anchor(self, node):
75 + self.last_anchor_id += 1
76 + return self.ANCHOR_TEMPLATE % self.last_anchor_id
77 +
78 + def serialize_node(self, node, parent, index):
79 + alias = self.anchors[node]
80 + if node in self.serialized_nodes:
81 + self.emit(AliasEvent(alias))
82 + else:
83 + self.serialized_nodes[node] = True
84 + self.descend_resolver(parent, index)
85 + if isinstance(node, ScalarNode):
86 + detected_tag = self.resolve(ScalarNode, node.value, (True, False))
87 + default_tag = self.resolve(ScalarNode, node.value, (False, True))
88 + implicit = (node.tag == detected_tag), (node.tag == default_tag)
89 + self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
90 + style=node.style))
91 + elif isinstance(node, SequenceNode):
92 + implicit = (node.tag
93 + == self.resolve(SequenceNode, node.value, True))
94 + self.emit(SequenceStartEvent(alias, node.tag, implicit,
95 + flow_style=node.flow_style))
96 + index = 0
97 + for item in node.value:
98 + self.serialize_node(item, node, index)
99 + index += 1
100 + self.emit(SequenceEndEvent())
101 + elif isinstance(node, MappingNode):
102 + implicit = (node.tag
103 + == self.resolve(MappingNode, node.value, True))
104 + self.emit(MappingStartEvent(alias, node.tag, implicit,
105 + flow_style=node.flow_style))
106 + for key, value in node.value:
107 + self.serialize_node(key, node, None)
108 + self.serialize_node(value, node, key)
109 + self.emit(MappingEndEvent())
110 + self.ascend_resolver()
111 +
python.d/python_modules/pyyaml3/tokens.py new
+104
@@ -0,0 +1,104 @@
1 +
2 +class Token(object):
3 + def __init__(self, start_mark, end_mark):
4 + self.start_mark = start_mark
5 + self.end_mark = end_mark
6 + def __repr__(self):
7 + attributes = [key for key in self.__dict__
8 + if not key.endswith('_mark')]
9 + attributes.sort()
10 + arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
11 + for key in attributes])
12 + return '%s(%s)' % (self.__class__.__name__, arguments)
13 +
14 +#class BOMToken(Token):
15 +# id = '<byte order mark>'
16 +
17 +class DirectiveToken(Token):
18 + id = '<directive>'
19 + def __init__(self, name, value, start_mark, end_mark):
20 + self.name = name
21 + self.value = value
22 + self.start_mark = start_mark
23 + self.end_mark = end_mark
24 +
25 +class DocumentStartToken(Token):
26 + id = '<document start>'
27 +
28 +class DocumentEndToken(Token):
29 + id = '<document end>'
30 +
31 +class StreamStartToken(Token):
32 + id = '<stream start>'
33 + def __init__(self, start_mark=None, end_mark=None,
34 + encoding=None):
35 + self.start_mark = start_mark
36 + self.end_mark = end_mark
37 + self.encoding = encoding
38 +
39 +class StreamEndToken(Token):
40 + id = '<stream end>'
41 +
42 +class BlockSequenceStartToken(Token):
43 + id = '<block sequence start>'
44 +
45 +class BlockMappingStartToken(Token):
46 + id = '<block mapping start>'
47 +
48 +class BlockEndToken(Token):
49 + id = '<block end>'
50 +
51 +class FlowSequenceStartToken(Token):
52 + id = '['
53 +
54 +class FlowMappingStartToken(Token):
55 + id = '{'
56 +
57 +class FlowSequenceEndToken(Token):
58 + id = ']'
59 +
60 +class FlowMappingEndToken(Token):
61 + id = '}'
62 +
63 +class KeyToken(Token):
64 + id = '?'
65 +
66 +class ValueToken(Token):
67 + id = ':'
68 +
69 +class BlockEntryToken(Token):
70 + id = '-'
71 +
72 +class FlowEntryToken(Token):
73 + id = ','
74 +
75 +class AliasToken(Token):
76 + id = '<alias>'
77 + def __init__(self, value, start_mark, end_mark):
78 + self.value = value
79 + self.start_mark = start_mark
80 + self.end_mark = end_mark
81 +
82 +class AnchorToken(Token):
83 + id = '<anchor>'
84 + def __init__(self, value, start_mark, end_mark):
85 + self.value = value
86 + self.start_mark = start_mark
87 + self.end_mark = end_mark
88 +
89 +class TagToken(Token):
90 + id = '<tag>'
91 + def __init__(self, value, start_mark, end_mark):
92 + self.value = value
93 + self.start_mark = start_mark
94 + self.end_mark = end_mark
95 +
96 +class ScalarToken(Token):
97 + id = '<scalar>'
98 + def __init__(self, value, plain, start_mark, end_mark, style=None):
99 + self.value = value
100 + self.plain = plain
101 + self.start_mark = start_mark
102 + self.end_mark = end_mark
103 + self.style = style
104 +