master
py 677 lines 23 KB
Raw
1 # Copyright IBM, Corp. 2011
2 # Copyright (c) 2013-2021 Red Hat Inc.
3 #
4 # Authors:
5 # Anthony Liguori <aliguori@us.ibm.com>
6 # Markus Armbruster <armbru@redhat.com>
7 # Eric Blake <eblake@redhat.com>
8 # Marc-André Lureau <marcandre.lureau@redhat.com>
9 # John Snow <jsnow@redhat.com>
10 #
11 # This work is licensed under the terms of the GNU GPL, version 2.
12 # See the COPYING file in the top-level directory.
13
14 """
15 Normalize and validate (context-free) QAPI schema expression structures.
16
17 `QAPISchemaParser` parses a QAPI schema into abstract syntax trees
18 consisting of dict, list, str, bool, and int nodes. This module ensures
19 that these nested structures have the correct type(s) and key(s) where
20 appropriate for the QAPI context-free grammar.
21
22 The QAPI schema expression language allows for certain syntactic sugar;
23 this module also handles the normalization process of these nested
24 structures.
25
26 See `check_exprs` for the main entry point.
27
28 See `schema.QAPISchema` for processing into native Python data
29 structures and contextual semantic validation.
30 """
31
32 import re
33 from typing import (
34 Dict,
35 Iterable,
36 List,
37 Optional,
38 Union,
39 cast,
40 )
41
42 from .common import c_name
43 from .error import QAPISemError
44 from .parser import QAPIExpression
45 from .source import QAPISourceInfo
46
47
48 # See check_name_str(), below.
49 valid_name = re.compile(r'(__[a-z0-9.-]+_)?'
50 r'(x-)?'
51 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE)
52
53
54 def check_name_is_str(name: object,
55 info: QAPISourceInfo,
56 source: str) -> None:
57 """
58 Ensure that ``name`` is a ``str``.
59
60 :raise QAPISemError: When ``name`` fails validation.
61 """
62 if not isinstance(name, str):
63 raise QAPISemError(info, "%s requires a string name" % source)
64
65
66 def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str:
67 """
68 Ensure that ``name`` is a valid QAPI name.
69
70 A valid name consists of ASCII letters, digits, ``-``, and ``_``,
71 starting with a letter. It may be prefixed by a downstream prefix
72 of the form __RFQDN_, or the experimental prefix ``x-``. If both
73 prefixes are present, the __RFDQN_ prefix goes first.
74
75 A valid name cannot start with ``q_``, which is reserved.
76
77 :param name: Name to check.
78 :param info: QAPI schema source file information.
79 :param source: Error string describing what ``name`` belongs to.
80
81 :raise QAPISemError: When ``name`` fails validation.
82 :return: The stem of the valid name, with no prefixes.
83 """
84 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
85 # and 'q_obj_*' implicit type names.
86 match = valid_name.match(name)
87 if not match or c_name(name, False).startswith('q_'):
88 raise QAPISemError(info, "%s has an invalid name" % source)
89 return match.group(3)
90
91
92 def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None:
93 """
94 Ensure that ``name`` is a valid event name.
95
96 This means it must be a valid QAPI name as checked by
97 `check_name_str()`, but where the stem prohibits lowercase
98 characters and ``-``.
99
100 :param name: Name to check.
101 :param info: QAPI schema source file information.
102 :param source: Error string describing what ``name`` belongs to.
103
104 :raise QAPISemError: When ``name`` fails validation.
105 """
106 stem = check_name_str(name, info, source)
107 if re.search(r'[a-z-]', stem):
108 raise QAPISemError(
109 info, "name of %s must not use lowercase or '-'" % source)
110
111
112 def check_name_lower(name: str, info: QAPISourceInfo, source: str,
113 permit_upper: bool = False,
114 permit_underscore: bool = False) -> None:
115 """
116 Ensure that ``name`` is a valid command or member name.
117
118 This means it must be a valid QAPI name as checked by
119 `check_name_str()`, but where the stem prohibits uppercase
120 characters and ``_``.
121
122 :param name: Name to check.
123 :param info: QAPI schema source file information.
124 :param source: Error string describing what ``name`` belongs to.
125 :param permit_upper: Additionally permit uppercase.
126 :param permit_underscore: Additionally permit ``_``.
127
128 :raise QAPISemError: When ``name`` fails validation.
129 """
130 stem = check_name_str(name, info, source)
131 if ((not permit_upper and re.search(r'[A-Z]', stem))
132 or (not permit_underscore and '_' in stem)):
133 raise QAPISemError(
134 info, "name of %s must not use uppercase or '_'" % source)
135
136
137 def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None:
138 """
139 Ensure that ``name`` is a valid user-defined type name.
140
141 This means it must be a valid QAPI name as checked by
142 `check_name_str()`, but where the stem must be in CamelCase.
143
144 :param name: Name to check.
145 :param info: QAPI schema source file information.
146 :param source: Error string describing what ``name`` belongs to.
147
148 :raise QAPISemError: When ``name`` fails validation.
149 """
150 stem = check_name_str(name, info, source)
151 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem):
152 raise QAPISemError(info, "name of %s must use CamelCase" % source)
153
154
155 def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None:
156 """
157 Ensure that ``name`` is a valid definition name.
158
159 Based on the value of ``meta``, this means that:
160 - 'event' names adhere to `check_name_upper()`.
161 - 'command' names adhere to `check_name_lower()`.
162 - Else, meta is a type, and must pass `check_name_camel()`.
163 These names must not end with ``List``.
164
165 :param name: Name to check.
166 :param info: QAPI schema source file information.
167 :param meta: Meta-type name of the QAPI expression.
168
169 :raise QAPISemError: When ``name`` fails validation.
170 """
171 if meta == 'event':
172 check_name_upper(name, info, meta)
173 elif meta == 'command':
174 check_name_lower(
175 name, info, meta,
176 permit_underscore=name in info.pragma.command_name_exceptions)
177 else:
178 check_name_camel(name, info, meta)
179 if name.endswith('List'):
180 raise QAPISemError(
181 info, "%s name should not end in 'List'" % meta)
182
183
184 def check_keys(value: Dict[str, object],
185 info: QAPISourceInfo,
186 source: str,
187 required: List[str],
188 optional: List[str]) -> None:
189 """
190 Ensure that a dict has a specific set of keys.
191
192 :param value: The dict to check.
193 :param info: QAPI schema source file information.
194 :param source: Error string describing this ``value``.
195 :param required: Keys that *must* be present.
196 :param optional: Keys that *may* be present.
197
198 :raise QAPISemError: When unknown keys are present.
199 """
200
201 def pprint(elems: Iterable[str]) -> str:
202 return ', '.join("'" + e + "'" for e in sorted(elems))
203
204 missing = set(required) - set(value)
205 if missing:
206 raise QAPISemError(
207 info,
208 "%s misses key%s %s"
209 % (source, 's' if len(missing) > 1 else '',
210 pprint(missing)))
211 allowed = set(required) | set(optional)
212 unknown = set(value) - allowed
213 if unknown:
214 raise QAPISemError(
215 info,
216 "%s has unknown key%s %s\nValid keys are %s."
217 % (source, 's' if len(unknown) > 1 else '',
218 pprint(unknown), pprint(allowed)))
219
220
221 def check_flags(expr: QAPIExpression) -> None:
222 """
223 Ensure flag members (if present) have valid values.
224
225 :param expr: The expression to validate.
226
227 :raise QAPISemError:
228 When certain flags have an invalid value, or when
229 incompatible flags are present.
230 """
231 for key in ('gen', 'success-response'):
232 if key in expr and expr[key] is not False:
233 raise QAPISemError(
234 expr.info, "flag '%s' may only use false value" % key)
235 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'):
236 if key in expr and expr[key] is not True:
237 raise QAPISemError(
238 expr.info, "flag '%s' may only use true value" % key)
239 if 'allow-oob' in expr and 'coroutine' in expr:
240 # This is not necessarily a fundamental incompatibility, but
241 # we don't have a use case and the desired semantics isn't
242 # obvious. The simplest solution is to forbid it until we get
243 # a use case for it.
244 raise QAPISemError(
245 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible")
246
247
248 def check_if(expr: Dict[str, object],
249 info: QAPISourceInfo, source: str) -> None:
250 """
251 Validate the ``if`` member of an object.
252
253 The ``if`` member may be either a ``str`` or a dict.
254
255 :param expr: The expression containing the ``if`` member to validate.
256 :param info: QAPI schema source file information.
257 :param source: Error string describing ``expr``.
258
259 :raise QAPISemError:
260 When the "if" member fails validation, or when there are no
261 non-empty conditions.
262 :return: None
263 """
264
265 def _check_if(cond: Union[str, object]) -> None:
266 if isinstance(cond, str):
267 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond):
268 raise QAPISemError(
269 info,
270 "'if' condition '%s' of %s is not a valid identifier"
271 % (cond, source))
272 return
273
274 if not isinstance(cond, dict):
275 raise QAPISemError(
276 info,
277 "'if' condition of %s must be a string or an object" % source)
278 check_keys(cond, info, "'if' condition of %s" % source, [],
279 ["all", "any", "not"])
280 if len(cond) != 1:
281 raise QAPISemError(
282 info,
283 "'if' condition of %s has conflicting keys" % source)
284
285 if 'not' in cond:
286 _check_if(cond['not'])
287 elif 'all' in cond:
288 _check_infix('all', cond['all'])
289 else:
290 _check_infix('any', cond['any'])
291
292 def _check_infix(operator: str, operands: object) -> None:
293 if not isinstance(operands, list):
294 raise QAPISemError(
295 info,
296 "'%s' condition of %s must be an array"
297 % (operator, source))
298 if not operands:
299 raise QAPISemError(
300 info, "'if' condition [] of %s is useless" % source)
301 for operand in operands:
302 _check_if(operand)
303
304 ifcond = expr.get('if')
305 if ifcond is None:
306 return
307
308 _check_if(ifcond)
309
310
311 def normalize_members(members: object) -> None:
312 """
313 Normalize a "members" value.
314
315 If ``members`` is a dict, for every value in that dict, if that
316 value is not itself already a dict, normalize it to
317 ``{'type': value}``.
318
319 :forms:
320 :sugared: ``Dict[str, Union[str, TypeRef]]``
321 :canonical: ``Dict[str, TypeRef]``
322
323 :param members: The members value to normalize.
324
325 :return: None, ``members`` is normalized in-place as needed.
326 """
327 if isinstance(members, dict):
328 for key, arg in members.items():
329 if isinstance(arg, dict):
330 continue
331 members[key] = {'type': arg}
332
333
334 def check_type_name(value: Optional[object],
335 info: QAPISourceInfo, source: str) -> None:
336 if value is not None and not isinstance(value, str):
337 raise QAPISemError(info, "%s should be a type name" % source)
338
339
340 def check_type_name_or_array(value: Optional[object],
341 info: QAPISourceInfo, source: str) -> None:
342 if value is None or isinstance(value, str):
343 return
344
345 if not isinstance(value, list):
346 raise QAPISemError(info,
347 "%s should be a type name or array" % source)
348
349 if len(value) != 1 or not isinstance(value[0], str):
350 raise QAPISemError(info,
351 "%s: array type must contain single type name" %
352 source)
353
354
355 def check_type_implicit(value: Optional[object],
356 info: QAPISourceInfo, source: str,
357 parent_name: Optional[str]) -> None:
358 """
359 Normalize and validate an optional implicit struct type.
360
361 Accept ``None`` or a ``dict`` defining an implicit struct type.
362 The latter is normalized in place.
363
364 :param value: The value to check.
365 :param info: QAPI schema source file information.
366 :param source: Error string describing this ``value``.
367 :param parent_name:
368 When the value of ``parent_name`` is in pragma
369 ``member-name-exceptions``, an implicit struct type may
370 violate the member naming rules.
371
372 :raise QAPISemError: When ``value`` fails validation.
373 :return: None
374 """
375 if value is None:
376 return
377
378 if not isinstance(value, dict):
379 raise QAPISemError(info,
380 "%s should be an object or type name" % source)
381
382 permissive = parent_name in info.pragma.member_name_exceptions
383
384 for (key, arg) in value.items():
385 key_source = "%s member '%s'" % (source, key)
386 if key.startswith('*'):
387 key = key[1:]
388 check_name_lower(key, info, key_source,
389 permit_upper=permissive,
390 permit_underscore=permissive)
391 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
392 raise QAPISemError(info, "%s uses reserved name" % key_source)
393 check_keys(arg, info, key_source, ['type'], ['if', 'features'])
394 check_if(arg, info, key_source)
395 check_features(arg.get('features'), info)
396 check_type_name_or_array(arg['type'], info, key_source)
397
398
399 def check_type_name_or_implicit(value: Optional[object],
400 info: QAPISourceInfo, source: str,
401 parent_name: Optional[str]) -> None:
402 if value is None or isinstance(value, str):
403 return
404
405 check_type_implicit(value, info, source, parent_name)
406
407
408 def check_features(features: Optional[object],
409 info: QAPISourceInfo) -> None:
410 """
411 Normalize and validate the ``features`` member.
412
413 ``features`` may be a ``list`` of either ``str`` or ``dict``.
414 Any ``str`` element will be normalized to ``{'name': element}``.
415
416 :forms:
417 :sugared: ``List[Union[str, Feature]]``
418 :canonical: ``List[Feature]``
419
420 :param features: The features member value to validate.
421 :param info: QAPI schema source file information.
422
423 :raise QAPISemError: When ``features`` fails validation.
424 :return: None, ``features`` is normalized in-place as needed.
425 """
426 if features is None:
427 return
428 if not isinstance(features, list):
429 raise QAPISemError(info, "'features' must be an array")
430 features[:] = [f if isinstance(f, dict) else {'name': f}
431 for f in features]
432 for feat in features:
433 source = "'features' member"
434 assert isinstance(feat, dict)
435 check_keys(feat, info, source, ['name'], ['if'])
436 check_name_is_str(feat['name'], info, source)
437 source = "%s '%s'" % (source, feat['name'])
438 check_name_lower(feat['name'], info, source)
439 check_if(feat, info, source)
440
441
442 def check_enum(expr: QAPIExpression) -> None:
443 """
444 Normalize and validate this expression as an ``enum`` definition.
445
446 :param expr: The expression to validate.
447
448 :raise QAPISemError: When ``expr`` is not a valid ``enum``.
449 :return: None, ``expr`` is normalized in-place as needed.
450 """
451 name = expr['enum']
452 members = expr['data']
453 prefix = expr.get('prefix')
454 info = expr.info
455
456 if not isinstance(members, list):
457 raise QAPISemError(info, "'data' must be an array")
458 if prefix is not None and not isinstance(prefix, str):
459 raise QAPISemError(info, "'prefix' must be a string")
460
461 permissive = name in info.pragma.member_name_exceptions
462
463 members[:] = [m if isinstance(m, dict) else {'name': m}
464 for m in members]
465 for member in members:
466 source = "'data' member"
467 check_keys(member, info, source, ['name'], ['if', 'features'])
468 member_name = member['name']
469 check_name_is_str(member_name, info, source)
470 source = "%s '%s'" % (source, member_name)
471 # Enum members may start with a digit
472 if member_name[0].isdigit():
473 member_name = 'd' + member_name # Hack: hide the digit
474 check_name_lower(member_name, info, source,
475 permit_upper=permissive,
476 permit_underscore=permissive)
477 check_if(member, info, source)
478 check_features(member.get('features'), info)
479
480
481 def check_struct(expr: QAPIExpression) -> None:
482 """
483 Normalize and validate this expression as a ``struct`` definition.
484
485 :param expr: The expression to validate.
486
487 :raise QAPISemError: When ``expr`` is not a valid ``struct``.
488 :return: None, ``expr`` is normalized in-place as needed.
489 """
490 name = cast(str, expr['struct']) # Checked in check_exprs
491 members = expr['data']
492
493 check_type_implicit(members, expr.info, "'data'", name)
494 check_type_name(expr.get('base'), expr.info, "'base'")
495
496
497 def check_union(expr: QAPIExpression) -> None:
498 """
499 Normalize and validate this expression as a ``union`` definition.
500
501 :param expr: The expression to validate.
502
503 :raise QAPISemError: when ``expr`` is not a valid ``union``.
504 :return: None, ``expr`` is normalized in-place as needed.
505 """
506 name = cast(str, expr['union']) # Checked in check_exprs
507 base = expr['base']
508 discriminator = expr['discriminator']
509 members = expr['data']
510 info = expr.info
511
512 check_type_name_or_implicit(base, info, "'base'", name)
513 check_name_is_str(discriminator, info, "'discriminator'")
514
515 if not isinstance(members, dict):
516 raise QAPISemError(info, "'data' must be an object")
517
518 for (key, value) in members.items():
519 source = "'data' member '%s'" % key
520 check_keys(value, info, source, ['type'], ['if'])
521 check_if(value, info, source)
522 check_type_name(value['type'], info, source)
523
524
525 def check_alternate(expr: QAPIExpression) -> None:
526 """
527 Normalize and validate this expression as an ``alternate`` definition.
528
529 :param expr: The expression to validate.
530
531 :raise QAPISemError: When ``expr`` is not a valid ``alternate``.
532 :return: None, ``expr`` is normalized in-place as needed.
533 """
534 members = expr['data']
535 info = expr.info
536
537 if not members:
538 raise QAPISemError(info, "'data' must not be empty")
539
540 if not isinstance(members, dict):
541 raise QAPISemError(info, "'data' must be an object")
542
543 for (key, value) in members.items():
544 source = "'data' member '%s'" % key
545 check_name_lower(key, info, source)
546 check_keys(value, info, source, ['type'], ['if'])
547 check_if(value, info, source)
548 check_type_name_or_array(value['type'], info, source)
549
550
551 def check_command(expr: QAPIExpression) -> None:
552 """
553 Normalize and validate this expression as a ``command`` definition.
554
555 :param expr: The expression to validate.
556
557 :raise QAPISemError: When ``expr`` is not a valid ``command``.
558 :return: None, ``expr`` is normalized in-place as needed.
559 """
560 args = expr.get('data')
561 rets = expr.get('returns')
562 boxed = expr.get('boxed', False)
563
564 if boxed:
565 if args is None:
566 raise QAPISemError(expr.info, "'boxed': true requires 'data'")
567 check_type_name(args, expr.info, "'data'")
568 else:
569 check_type_name_or_implicit(args, expr.info, "'data'", None)
570 check_type_name_or_array(rets, expr.info, "'returns'")
571
572
573 def check_event(expr: QAPIExpression) -> None:
574 """
575 Normalize and validate this expression as an ``event`` definition.
576
577 :param expr: The expression to validate.
578
579 :raise QAPISemError: When ``expr`` is not a valid ``event``.
580 :return: None, ``expr`` is normalized in-place as needed.
581 """
582 args = expr.get('data')
583 boxed = expr.get('boxed', False)
584
585 if boxed:
586 if args is None:
587 raise QAPISemError(expr.info, "'boxed': true requires 'data'")
588 check_type_name(args, expr.info, "'data'")
589 else:
590 check_type_name_or_implicit(args, expr.info, "'data'", None)
591
592
593 def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]:
594 """
595 Validate and normalize a list of parsed QAPI schema expressions.
596
597 This function accepts a list of expressions and metadata as returned
598 by the parser. It destructively normalizes the expressions in-place.
599
600 :param exprs: The list of expressions to normalize and validate.
601
602 :raise QAPISemError: When any expression fails validation.
603 :return: The same list of expressions (now modified).
604 """
605 for expr in exprs:
606 info = expr.info
607 doc = expr.doc
608
609 if 'include' in expr:
610 continue
611
612 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate',
613 'command', 'event'}
614 if len(metas) != 1:
615 raise QAPISemError(
616 info,
617 "expression must have exactly one key"
618 " 'enum', 'struct', 'union', 'alternate',"
619 " 'command', 'event'")
620 meta = metas.pop()
621
622 check_name_is_str(expr[meta], info, "'%s'" % meta)
623 name = cast(str, expr[meta])
624 info.set_defn(meta, name)
625 check_defn_name_str(name, info, meta)
626
627 if doc:
628 if doc.symbol != name:
629 raise QAPISemError(
630 info, "documentation comment is for '%s'" % doc.symbol)
631 doc.check_expr(expr)
632 elif info.pragma.doc_required:
633 raise QAPISemError(info,
634 "documentation comment required")
635
636 if meta == 'enum':
637 check_keys(expr, info, meta,
638 ['enum', 'data'], ['if', 'features', 'prefix'])
639 check_enum(expr)
640 elif meta == 'union':
641 check_keys(expr, info, meta,
642 ['union', 'base', 'discriminator', 'data'],
643 ['if', 'features'])
644 normalize_members(expr.get('base'))
645 normalize_members(expr['data'])
646 check_union(expr)
647 elif meta == 'alternate':
648 check_keys(expr, info, meta,
649 ['alternate', 'data'], ['if', 'features'])
650 normalize_members(expr['data'])
651 check_alternate(expr)
652 elif meta == 'struct':
653 check_keys(expr, info, meta,
654 ['struct', 'data'], ['base', 'if', 'features'])
655 normalize_members(expr['data'])
656 check_struct(expr)
657 elif meta == 'command':
658 check_keys(expr, info, meta,
659 ['command'],
660 ['data', 'returns', 'boxed', 'if', 'features',
661 'gen', 'success-response', 'allow-oob',
662 'allow-preconfig', 'coroutine'])
663 normalize_members(expr.get('data'))
664 check_command(expr)
665 elif meta == 'event':
666 check_keys(expr, info, meta,
667 ['event'], ['data', 'boxed', 'if', 'features'])
668 normalize_members(expr.get('data'))
669 check_event(expr)
670 else:
671 assert False, 'unexpected meta type'
672
673 check_if(expr, info, meta)
674 check_features(expr.get('features'), info)
675 check_flags(expr)
676
677 return exprs