master
py 590 lines 24.9 KB
Raw
1 # SPDX-License-Identifier: MIT
2
3 # The following YAML grammar is LL(1) and is parsed by a recursive descent
4 # parser.
5 #
6 # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
7 # implicit_document ::= block_node DOCUMENT-END*
8 # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
9 # block_node_or_indentless_sequence ::=
10 # ALIAS
11 # | properties (block_content | indentless_block_sequence)?
12 # | block_content
13 # | indentless_block_sequence
14 # block_node ::= ALIAS
15 # | properties block_content?
16 # | block_content
17 # flow_node ::= ALIAS
18 # | properties flow_content?
19 # | flow_content
20 # properties ::= TAG ANCHOR? | ANCHOR TAG?
21 # block_content ::= block_collection | flow_collection | SCALAR
22 # flow_content ::= flow_collection | SCALAR
23 # block_collection ::= block_sequence | block_mapping
24 # flow_collection ::= flow_sequence | flow_mapping
25 # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
26 # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
27 # block_mapping ::= BLOCK-MAPPING_START
28 # ((KEY block_node_or_indentless_sequence?)?
29 # (VALUE block_node_or_indentless_sequence?)?)*
30 # BLOCK-END
31 # flow_sequence ::= FLOW-SEQUENCE-START
32 # (flow_sequence_entry FLOW-ENTRY)*
33 # flow_sequence_entry?
34 # FLOW-SEQUENCE-END
35 # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
36 # flow_mapping ::= FLOW-MAPPING-START
37 # (flow_mapping_entry FLOW-ENTRY)*
38 # flow_mapping_entry?
39 # FLOW-MAPPING-END
40 # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
41 #
42 # FIRST sets:
43 #
44 # stream: { STREAM-START }
45 # explicit_document: { DIRECTIVE DOCUMENT-START }
46 # implicit_document: FIRST(block_node)
47 # block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
48 # flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
49 # block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
50 # flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
51 # block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
52 # flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
53 # block_sequence: { BLOCK-SEQUENCE-START }
54 # block_mapping: { BLOCK-MAPPING-START }
55 # block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
56 # indentless_sequence: { ENTRY }
57 # flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
58 # flow_sequence: { FLOW-SEQUENCE-START }
59 # flow_mapping: { FLOW-MAPPING-START }
60 # flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
61 # flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
62
63 __all__ = ['Parser', 'ParserError']
64
65 from .error import MarkedYAMLError
66 from .tokens import *
67 from .events import *
68 from .scanner import *
69
70 class ParserError(MarkedYAMLError):
71 pass
72
73 class Parser:
74 # Since writing a recursive-descendant parser is a straightforward task, we
75 # do not give many comments here.
76
77 DEFAULT_TAGS = {
78 '!': '!',
79 '!!': 'tag:yaml.org,2002:',
80 }
81
82 def __init__(self):
83 self.current_event = None
84 self.yaml_version = None
85 self.tag_handles = {}
86 self.states = []
87 self.marks = []
88 self.state = self.parse_stream_start
89
90 def dispose(self):
91 # Reset the state attributes (to clear self-references)
92 self.states = []
93 self.state = None
94
95 def check_event(self, *choices):
96 # Check the type of the next event.
97 if self.current_event is None:
98 if self.state:
99 self.current_event = self.state()
100 if self.current_event is not None:
101 if not choices:
102 return True
103 for choice in choices:
104 if isinstance(self.current_event, choice):
105 return True
106 return False
107
108 def peek_event(self):
109 # Get the next event.
110 if self.current_event is None:
111 if self.state:
112 self.current_event = self.state()
113 return self.current_event
114
115 def get_event(self):
116 # Get the next event and proceed further.
117 if self.current_event is None:
118 if self.state:
119 self.current_event = self.state()
120 value = self.current_event
121 self.current_event = None
122 return value
123
124 # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
125 # implicit_document ::= block_node DOCUMENT-END*
126 # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
127
128 def parse_stream_start(self):
129
130 # Parse the stream start.
131 token = self.get_token()
132 event = StreamStartEvent(token.start_mark, token.end_mark,
133 encoding=token.encoding)
134
135 # Prepare the next state.
136 self.state = self.parse_implicit_document_start
137
138 return event
139
140 def parse_implicit_document_start(self):
141
142 # Parse an implicit document.
143 if not self.check_token(DirectiveToken, DocumentStartToken,
144 StreamEndToken):
145 self.tag_handles = self.DEFAULT_TAGS
146 token = self.peek_token()
147 start_mark = end_mark = token.start_mark
148 event = DocumentStartEvent(start_mark, end_mark,
149 explicit=False)
150
151 # Prepare the next state.
152 self.states.append(self.parse_document_end)
153 self.state = self.parse_block_node
154
155 return event
156
157 else:
158 return self.parse_document_start()
159
160 def parse_document_start(self):
161
162 # Parse any extra document end indicators.
163 while self.check_token(DocumentEndToken):
164 self.get_token()
165
166 # Parse an explicit document.
167 if not self.check_token(StreamEndToken):
168 token = self.peek_token()
169 start_mark = token.start_mark
170 version, tags = self.process_directives()
171 if not self.check_token(DocumentStartToken):
172 raise ParserError(None, None,
173 "expected '<document start>', but found %r"
174 % self.peek_token().id,
175 self.peek_token().start_mark)
176 token = self.get_token()
177 end_mark = token.end_mark
178 event = DocumentStartEvent(start_mark, end_mark,
179 explicit=True, version=version, tags=tags)
180 self.states.append(self.parse_document_end)
181 self.state = self.parse_document_content
182 else:
183 # Parse the end of the stream.
184 token = self.get_token()
185 event = StreamEndEvent(token.start_mark, token.end_mark)
186 assert not self.states
187 assert not self.marks
188 self.state = None
189 return event
190
191 def parse_document_end(self):
192
193 # Parse the document end.
194 token = self.peek_token()
195 start_mark = end_mark = token.start_mark
196 explicit = False
197 if self.check_token(DocumentEndToken):
198 token = self.get_token()
199 end_mark = token.end_mark
200 explicit = True
201 event = DocumentEndEvent(start_mark, end_mark,
202 explicit=explicit)
203
204 # Prepare the next state.
205 self.state = self.parse_document_start
206
207 return event
208
209 def parse_document_content(self):
210 if self.check_token(DirectiveToken,
211 DocumentStartToken, DocumentEndToken, StreamEndToken):
212 event = self.process_empty_scalar(self.peek_token().start_mark)
213 self.state = self.states.pop()
214 return event
215 else:
216 return self.parse_block_node()
217
218 def process_directives(self):
219 self.yaml_version = None
220 self.tag_handles = {}
221 while self.check_token(DirectiveToken):
222 token = self.get_token()
223 if token.name == 'YAML':
224 if self.yaml_version is not None:
225 raise ParserError(None, None,
226 "found duplicate YAML directive", token.start_mark)
227 major, minor = token.value
228 if major != 1:
229 raise ParserError(None, None,
230 "found incompatible YAML document (version 1.* is required)",
231 token.start_mark)
232 self.yaml_version = token.value
233 elif token.name == 'TAG':
234 handle, prefix = token.value
235 if handle in self.tag_handles:
236 raise ParserError(None, None,
237 "duplicate tag handle %r" % handle,
238 token.start_mark)
239 self.tag_handles[handle] = prefix
240 if self.tag_handles:
241 value = self.yaml_version, self.tag_handles.copy()
242 else:
243 value = self.yaml_version, None
244 for key in self.DEFAULT_TAGS:
245 if key not in self.tag_handles:
246 self.tag_handles[key] = self.DEFAULT_TAGS[key]
247 return value
248
249 # block_node_or_indentless_sequence ::= ALIAS
250 # | properties (block_content | indentless_block_sequence)?
251 # | block_content
252 # | indentless_block_sequence
253 # block_node ::= ALIAS
254 # | properties block_content?
255 # | block_content
256 # flow_node ::= ALIAS
257 # | properties flow_content?
258 # | flow_content
259 # properties ::= TAG ANCHOR? | ANCHOR TAG?
260 # block_content ::= block_collection | flow_collection | SCALAR
261 # flow_content ::= flow_collection | SCALAR
262 # block_collection ::= block_sequence | block_mapping
263 # flow_collection ::= flow_sequence | flow_mapping
264
265 def parse_block_node(self):
266 return self.parse_node(block=True)
267
268 def parse_flow_node(self):
269 return self.parse_node()
270
271 def parse_block_node_or_indentless_sequence(self):
272 return self.parse_node(block=True, indentless_sequence=True)
273
274 def parse_node(self, block=False, indentless_sequence=False):
275 if self.check_token(AliasToken):
276 token = self.get_token()
277 event = AliasEvent(token.value, token.start_mark, token.end_mark)
278 self.state = self.states.pop()
279 else:
280 anchor = None
281 tag = None
282 start_mark = end_mark = tag_mark = None
283 if self.check_token(AnchorToken):
284 token = self.get_token()
285 start_mark = token.start_mark
286 end_mark = token.end_mark
287 anchor = token.value
288 if self.check_token(TagToken):
289 token = self.get_token()
290 tag_mark = token.start_mark
291 end_mark = token.end_mark
292 tag = token.value
293 elif self.check_token(TagToken):
294 token = self.get_token()
295 start_mark = tag_mark = token.start_mark
296 end_mark = token.end_mark
297 tag = token.value
298 if self.check_token(AnchorToken):
299 token = self.get_token()
300 end_mark = token.end_mark
301 anchor = token.value
302 if tag is not None:
303 handle, suffix = tag
304 if handle is not None:
305 if handle not in self.tag_handles:
306 raise ParserError("while parsing a node", start_mark,
307 "found undefined tag handle %r" % handle,
308 tag_mark)
309 tag = self.tag_handles[handle]+suffix
310 else:
311 tag = suffix
312 #if tag == '!':
313 # raise ParserError("while parsing a node", start_mark,
314 # "found non-specific tag '!'", tag_mark,
315 # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
316 if start_mark is None:
317 start_mark = end_mark = self.peek_token().start_mark
318 event = None
319 implicit = (tag is None or tag == '!')
320 if indentless_sequence and self.check_token(BlockEntryToken):
321 end_mark = self.peek_token().end_mark
322 event = SequenceStartEvent(anchor, tag, implicit,
323 start_mark, end_mark)
324 self.state = self.parse_indentless_sequence_entry
325 else:
326 if self.check_token(ScalarToken):
327 token = self.get_token()
328 end_mark = token.end_mark
329 if (token.plain and tag is None) or tag == '!':
330 implicit = (True, False)
331 elif tag is None:
332 implicit = (False, True)
333 else:
334 implicit = (False, False)
335 event = ScalarEvent(anchor, tag, implicit, token.value,
336 start_mark, end_mark, style=token.style)
337 self.state = self.states.pop()
338 elif self.check_token(FlowSequenceStartToken):
339 end_mark = self.peek_token().end_mark
340 event = SequenceStartEvent(anchor, tag, implicit,
341 start_mark, end_mark, flow_style=True)
342 self.state = self.parse_flow_sequence_first_entry
343 elif self.check_token(FlowMappingStartToken):
344 end_mark = self.peek_token().end_mark
345 event = MappingStartEvent(anchor, tag, implicit,
346 start_mark, end_mark, flow_style=True)
347 self.state = self.parse_flow_mapping_first_key
348 elif block and self.check_token(BlockSequenceStartToken):
349 end_mark = self.peek_token().start_mark
350 event = SequenceStartEvent(anchor, tag, implicit,
351 start_mark, end_mark, flow_style=False)
352 self.state = self.parse_block_sequence_first_entry
353 elif block and self.check_token(BlockMappingStartToken):
354 end_mark = self.peek_token().start_mark
355 event = MappingStartEvent(anchor, tag, implicit,
356 start_mark, end_mark, flow_style=False)
357 self.state = self.parse_block_mapping_first_key
358 elif anchor is not None or tag is not None:
359 # Empty scalars are allowed even if a tag or an anchor is
360 # specified.
361 event = ScalarEvent(anchor, tag, (implicit, False), '',
362 start_mark, end_mark)
363 self.state = self.states.pop()
364 else:
365 if block:
366 node = 'block'
367 else:
368 node = 'flow'
369 token = self.peek_token()
370 raise ParserError("while parsing a %s node" % node, start_mark,
371 "expected the node content, but found %r" % token.id,
372 token.start_mark)
373 return event
374
375 # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
376
377 def parse_block_sequence_first_entry(self):
378 token = self.get_token()
379 self.marks.append(token.start_mark)
380 return self.parse_block_sequence_entry()
381
382 def parse_block_sequence_entry(self):
383 if self.check_token(BlockEntryToken):
384 token = self.get_token()
385 if not self.check_token(BlockEntryToken, BlockEndToken):
386 self.states.append(self.parse_block_sequence_entry)
387 return self.parse_block_node()
388 else:
389 self.state = self.parse_block_sequence_entry
390 return self.process_empty_scalar(token.end_mark)
391 if not self.check_token(BlockEndToken):
392 token = self.peek_token()
393 raise ParserError("while parsing a block collection", self.marks[-1],
394 "expected <block end>, but found %r" % token.id, token.start_mark)
395 token = self.get_token()
396 event = SequenceEndEvent(token.start_mark, token.end_mark)
397 self.state = self.states.pop()
398 self.marks.pop()
399 return event
400
401 # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
402
403 def parse_indentless_sequence_entry(self):
404 if self.check_token(BlockEntryToken):
405 token = self.get_token()
406 if not self.check_token(BlockEntryToken,
407 KeyToken, ValueToken, BlockEndToken):
408 self.states.append(self.parse_indentless_sequence_entry)
409 return self.parse_block_node()
410 else:
411 self.state = self.parse_indentless_sequence_entry
412 return self.process_empty_scalar(token.end_mark)
413 token = self.peek_token()
414 event = SequenceEndEvent(token.start_mark, token.start_mark)
415 self.state = self.states.pop()
416 return event
417
418 # block_mapping ::= BLOCK-MAPPING_START
419 # ((KEY block_node_or_indentless_sequence?)?
420 # (VALUE block_node_or_indentless_sequence?)?)*
421 # BLOCK-END
422
423 def parse_block_mapping_first_key(self):
424 token = self.get_token()
425 self.marks.append(token.start_mark)
426 return self.parse_block_mapping_key()
427
428 def parse_block_mapping_key(self):
429 if self.check_token(KeyToken):
430 token = self.get_token()
431 if not self.check_token(KeyToken, ValueToken, BlockEndToken):
432 self.states.append(self.parse_block_mapping_value)
433 return self.parse_block_node_or_indentless_sequence()
434 else:
435 self.state = self.parse_block_mapping_value
436 return self.process_empty_scalar(token.end_mark)
437 if not self.check_token(BlockEndToken):
438 token = self.peek_token()
439 raise ParserError("while parsing a block mapping", self.marks[-1],
440 "expected <block end>, but found %r" % token.id, token.start_mark)
441 token = self.get_token()
442 event = MappingEndEvent(token.start_mark, token.end_mark)
443 self.state = self.states.pop()
444 self.marks.pop()
445 return event
446
447 def parse_block_mapping_value(self):
448 if self.check_token(ValueToken):
449 token = self.get_token()
450 if not self.check_token(KeyToken, ValueToken, BlockEndToken):
451 self.states.append(self.parse_block_mapping_key)
452 return self.parse_block_node_or_indentless_sequence()
453 else:
454 self.state = self.parse_block_mapping_key
455 return self.process_empty_scalar(token.end_mark)
456 else:
457 self.state = self.parse_block_mapping_key
458 token = self.peek_token()
459 return self.process_empty_scalar(token.start_mark)
460
461 # flow_sequence ::= FLOW-SEQUENCE-START
462 # (flow_sequence_entry FLOW-ENTRY)*
463 # flow_sequence_entry?
464 # FLOW-SEQUENCE-END
465 # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
466 #
467 # Note that while production rules for both flow_sequence_entry and
468 # flow_mapping_entry are equal, their interpretations are different.
469 # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
470 # generate an inline mapping (set syntax).
471
472 def parse_flow_sequence_first_entry(self):
473 token = self.get_token()
474 self.marks.append(token.start_mark)
475 return self.parse_flow_sequence_entry(first=True)
476
477 def parse_flow_sequence_entry(self, first=False):
478 if not self.check_token(FlowSequenceEndToken):
479 if not first:
480 if self.check_token(FlowEntryToken):
481 self.get_token()
482 else:
483 token = self.peek_token()
484 raise ParserError("while parsing a flow sequence", self.marks[-1],
485 "expected ',' or ']', but got %r" % token.id, token.start_mark)
486
487 if self.check_token(KeyToken):
488 token = self.peek_token()
489 event = MappingStartEvent(None, None, True,
490 token.start_mark, token.end_mark,
491 flow_style=True)
492 self.state = self.parse_flow_sequence_entry_mapping_key
493 return event
494 elif not self.check_token(FlowSequenceEndToken):
495 self.states.append(self.parse_flow_sequence_entry)
496 return self.parse_flow_node()
497 token = self.get_token()
498 event = SequenceEndEvent(token.start_mark, token.end_mark)
499 self.state = self.states.pop()
500 self.marks.pop()
501 return event
502
503 def parse_flow_sequence_entry_mapping_key(self):
504 token = self.get_token()
505 if not self.check_token(ValueToken,
506 FlowEntryToken, FlowSequenceEndToken):
507 self.states.append(self.parse_flow_sequence_entry_mapping_value)
508 return self.parse_flow_node()
509 else:
510 self.state = self.parse_flow_sequence_entry_mapping_value
511 return self.process_empty_scalar(token.end_mark)
512
513 def parse_flow_sequence_entry_mapping_value(self):
514 if self.check_token(ValueToken):
515 token = self.get_token()
516 if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
517 self.states.append(self.parse_flow_sequence_entry_mapping_end)
518 return self.parse_flow_node()
519 else:
520 self.state = self.parse_flow_sequence_entry_mapping_end
521 return self.process_empty_scalar(token.end_mark)
522 else:
523 self.state = self.parse_flow_sequence_entry_mapping_end
524 token = self.peek_token()
525 return self.process_empty_scalar(token.start_mark)
526
527 def parse_flow_sequence_entry_mapping_end(self):
528 self.state = self.parse_flow_sequence_entry
529 token = self.peek_token()
530 return MappingEndEvent(token.start_mark, token.start_mark)
531
532 # flow_mapping ::= FLOW-MAPPING-START
533 # (flow_mapping_entry FLOW-ENTRY)*
534 # flow_mapping_entry?
535 # FLOW-MAPPING-END
536 # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
537
538 def parse_flow_mapping_first_key(self):
539 token = self.get_token()
540 self.marks.append(token.start_mark)
541 return self.parse_flow_mapping_key(first=True)
542
543 def parse_flow_mapping_key(self, first=False):
544 if not self.check_token(FlowMappingEndToken):
545 if not first:
546 if self.check_token(FlowEntryToken):
547 self.get_token()
548 else:
549 token = self.peek_token()
550 raise ParserError("while parsing a flow mapping", self.marks[-1],
551 "expected ',' or '}', but got %r" % token.id, token.start_mark)
552 if self.check_token(KeyToken):
553 token = self.get_token()
554 if not self.check_token(ValueToken,
555 FlowEntryToken, FlowMappingEndToken):
556 self.states.append(self.parse_flow_mapping_value)
557 return self.parse_flow_node()
558 else:
559 self.state = self.parse_flow_mapping_value
560 return self.process_empty_scalar(token.end_mark)
561 elif not self.check_token(FlowMappingEndToken):
562 self.states.append(self.parse_flow_mapping_empty_value)
563 return self.parse_flow_node()
564 token = self.get_token()
565 event = MappingEndEvent(token.start_mark, token.end_mark)
566 self.state = self.states.pop()
567 self.marks.pop()
568 return event
569
570 def parse_flow_mapping_value(self):
571 if self.check_token(ValueToken):
572 token = self.get_token()
573 if not self.check_token(FlowEntryToken, FlowMappingEndToken):
574 self.states.append(self.parse_flow_mapping_key)
575 return self.parse_flow_node()
576 else:
577 self.state = self.parse_flow_mapping_key
578 return self.process_empty_scalar(token.end_mark)
579 else:
580 self.state = self.parse_flow_mapping_key
581 token = self.peek_token()
582 return self.process_empty_scalar(token.start_mark)
583
584 def parse_flow_mapping_empty_value(self):
585 self.state = self.parse_flow_mapping_key
586 return self.process_empty_scalar(self.peek_token().start_mark)
587
588 def process_empty_scalar(self, mark):
589 return ScalarEvent(None, None, (True, False), '', mark, mark)
590