| 1 | # QAPI schema parser |
| 2 | # |
| 3 | # Copyright IBM, Corp. 2011 |
| 4 | # Copyright (c) 2013-2019 Red Hat Inc. |
| 5 | # |
| 6 | # Authors: |
| 7 | # Anthony Liguori <aliguori@us.ibm.com> |
| 8 | # Markus Armbruster <armbru@redhat.com> |
| 9 | # Marc-André Lureau <marcandre.lureau@redhat.com> |
| 10 | # Kevin Wolf <kwolf@redhat.com> |
| 11 | # |
| 12 | # This work is licensed under the terms of the GNU GPL, version 2. |
| 13 | # See the COPYING file in the top-level directory. |
| 14 | |
| 15 | import enum |
| 16 | import os |
| 17 | import re |
| 18 | from typing import ( |
| 19 | TYPE_CHECKING, |
| 20 | Any, |
| 21 | Dict, |
| 22 | List, |
| 23 | Mapping, |
| 24 | Match, |
| 25 | Optional, |
| 26 | Set, |
| 27 | Union, |
| 28 | ) |
| 29 | |
| 30 | from .common import must_match |
| 31 | from .error import QAPISemError, QAPISourceError |
| 32 | from .source import QAPISourceInfo |
| 33 | |
| 34 | |
| 35 | if TYPE_CHECKING: |
| 36 | # pylint: disable=cyclic-import |
| 37 | # TODO: Remove cycle. [schema -> expr -> parser -> schema] |
| 38 | from .schema import QAPISchemaFeature, QAPISchemaMember |
| 39 | |
| 40 | |
| 41 | # Return value alias for get_expr(). |
| 42 | _ExprValue = Union[List[object], Dict[str, object], str, bool] |
| 43 | |
| 44 | |
| 45 | class QAPIExpression(Dict[str, Any]): |
| 46 | # pylint: disable=too-few-public-methods |
| 47 | def __init__(self, |
| 48 | data: Mapping[str, object], |
| 49 | info: QAPISourceInfo, |
| 50 | doc: Optional['QAPIDoc'] = None): |
| 51 | super().__init__(data) |
| 52 | self.info = info |
| 53 | self.doc: Optional['QAPIDoc'] = doc |
| 54 | |
| 55 | |
| 56 | class QAPIParseError(QAPISourceError): |
| 57 | """Error class for all QAPI schema parsing errors.""" |
| 58 | def __init__(self, parser: 'QAPISchemaParser', msg: str): |
| 59 | col = 1 |
| 60 | for ch in parser.src[parser.line_pos:parser.pos]: |
| 61 | if ch == '\t': |
| 62 | col = (col + 7) % 8 + 1 |
| 63 | else: |
| 64 | col += 1 |
| 65 | super().__init__(parser.info, msg, col) |
| 66 | |
| 67 | |
| 68 | class QAPISchemaParser: |
| 69 | """ |
| 70 | Parse QAPI schema source. |
| 71 | |
| 72 | Parse a JSON-esque schema file and process directives. See |
| 73 | qapi-code-gen.rst section "Schema Syntax" for the exact syntax. |
| 74 | Grammatical validation is handled later by `expr.check_exprs()`. |
| 75 | |
| 76 | :param fname: Source file name. |
| 77 | :param previously_included: |
| 78 | The absolute names of previously included source files, |
| 79 | if being invoked from another parser. |
| 80 | :param incl_info: |
| 81 | `QAPISourceInfo` belonging to the parent module. |
| 82 | ``None`` implies this is the root module. |
| 83 | |
| 84 | :ivar exprs: Resulting parsed expressions. |
| 85 | :ivar docs: Resulting parsed documentation blocks. |
| 86 | |
| 87 | :raise OSError: For problems reading the root schema document. |
| 88 | :raise QAPIError: For errors in the schema source. |
| 89 | """ |
| 90 | def __init__(self, |
| 91 | fname: str, |
| 92 | previously_included: Optional[Set[str]] = None, |
| 93 | incl_info: Optional[QAPISourceInfo] = None): |
| 94 | self._fname = fname |
| 95 | self._included = previously_included or set() |
| 96 | self._included.add(os.path.abspath(self._fname)) |
| 97 | self.src = '' |
| 98 | |
| 99 | # Lexer state (see `accept` for details): |
| 100 | self.info = QAPISourceInfo(self._fname, incl_info) |
| 101 | self.tok: Union[None, str] = None |
| 102 | self.pos = 0 |
| 103 | self.cursor = 0 |
| 104 | self.val: Optional[Union[bool, str]] = None |
| 105 | self.line_pos = 0 |
| 106 | |
| 107 | # Parser output: |
| 108 | self.exprs: List[QAPIExpression] = [] |
| 109 | self.docs: List[QAPIDoc] = [] |
| 110 | |
| 111 | # State for tracking qmp-example blocks and simple |
| 112 | # :: literal blocks. |
| 113 | self._literal_mode = False |
| 114 | self._literal_mode_indent = 0 |
| 115 | |
| 116 | # Showtime! |
| 117 | self._parse() |
| 118 | |
| 119 | def _parse(self) -> None: |
| 120 | """ |
| 121 | Parse the QAPI schema document. |
| 122 | |
| 123 | :return: None. Results are stored in ``.exprs`` and ``.docs``. |
| 124 | """ |
| 125 | cur_doc = None |
| 126 | |
| 127 | # May raise OSError; allow the caller to handle it. |
| 128 | with open(self._fname, 'r', encoding='utf-8') as fp: |
| 129 | self.src = fp.read() |
| 130 | if self.src == '' or self.src[-1] != '\n': |
| 131 | self.src += '\n' |
| 132 | |
| 133 | # Prime the lexer: |
| 134 | self.accept() |
| 135 | |
| 136 | # Parse until done: |
| 137 | while self.tok is not None: |
| 138 | info = self.info |
| 139 | if self.tok == '#': |
| 140 | self.reject_expr_doc(cur_doc) |
| 141 | cur_doc = self.get_doc() |
| 142 | self.docs.append(cur_doc) |
| 143 | continue |
| 144 | |
| 145 | expr = self.get_expr() |
| 146 | if not isinstance(expr, dict): |
| 147 | raise QAPISemError( |
| 148 | info, "top-level expression must be an object") |
| 149 | |
| 150 | if 'include' in expr: |
| 151 | self.reject_expr_doc(cur_doc) |
| 152 | if len(expr) != 1: |
| 153 | raise QAPISemError(info, "invalid 'include' directive") |
| 154 | include = expr['include'] |
| 155 | if not isinstance(include, str): |
| 156 | raise QAPISemError(info, |
| 157 | "value of 'include' must be a string") |
| 158 | incl_fname = os.path.join(os.path.dirname(self._fname), |
| 159 | include) |
| 160 | self._add_expr({'include': incl_fname}, info) |
| 161 | exprs_include = self._include(include, info, incl_fname, |
| 162 | self._included) |
| 163 | if exprs_include: |
| 164 | self.exprs.extend(exprs_include.exprs) |
| 165 | self.docs.extend(exprs_include.docs) |
| 166 | elif "pragma" in expr: |
| 167 | self.reject_expr_doc(cur_doc) |
| 168 | if len(expr) != 1: |
| 169 | raise QAPISemError(info, "invalid 'pragma' directive") |
| 170 | pragma = expr['pragma'] |
| 171 | if not isinstance(pragma, dict): |
| 172 | raise QAPISemError( |
| 173 | info, "value of 'pragma' must be an object") |
| 174 | for name, value in pragma.items(): |
| 175 | self._pragma(name, value, info) |
| 176 | else: |
| 177 | if cur_doc and not cur_doc.symbol: |
| 178 | raise QAPISemError( |
| 179 | cur_doc.info, "definition documentation required") |
| 180 | self._add_expr(expr, info, cur_doc) |
| 181 | cur_doc = None |
| 182 | self.reject_expr_doc(cur_doc) |
| 183 | |
| 184 | def _add_expr(self, expr: Mapping[str, object], |
| 185 | info: QAPISourceInfo, |
| 186 | doc: Optional['QAPIDoc'] = None) -> None: |
| 187 | self.exprs.append(QAPIExpression(expr, info, doc)) |
| 188 | |
| 189 | @staticmethod |
| 190 | def reject_expr_doc(doc: Optional['QAPIDoc']) -> None: |
| 191 | if doc and doc.symbol: |
| 192 | raise QAPISemError( |
| 193 | doc.info, |
| 194 | "documentation for '%s' is not followed by the definition" |
| 195 | % doc.symbol) |
| 196 | |
| 197 | @staticmethod |
| 198 | def _include(include: str, |
| 199 | info: QAPISourceInfo, |
| 200 | incl_fname: str, |
| 201 | previously_included: Set[str] |
| 202 | ) -> Optional['QAPISchemaParser']: |
| 203 | incl_abs_fname = os.path.abspath(incl_fname) |
| 204 | # catch inclusion cycle |
| 205 | inf: Optional[QAPISourceInfo] = info |
| 206 | while inf: |
| 207 | if incl_abs_fname == os.path.abspath(inf.fname): |
| 208 | raise QAPISemError(info, "inclusion loop for %s" % include) |
| 209 | inf = inf.parent |
| 210 | |
| 211 | # skip multiple include of the same file |
| 212 | if incl_abs_fname in previously_included: |
| 213 | return None |
| 214 | |
| 215 | try: |
| 216 | return QAPISchemaParser(incl_fname, previously_included, info) |
| 217 | except OSError as err: |
| 218 | raise QAPISemError( |
| 219 | info, |
| 220 | f"can't read include file '{incl_fname}': {err.strerror}" |
| 221 | ) from err |
| 222 | |
| 223 | @staticmethod |
| 224 | def _pragma(name: str, value: object, info: QAPISourceInfo) -> None: |
| 225 | |
| 226 | def check_list_str(name: str, value: object) -> List[str]: |
| 227 | if (not isinstance(value, list) or |
| 228 | any(not isinstance(elt, str) for elt in value)): |
| 229 | raise QAPISemError( |
| 230 | info, |
| 231 | "pragma %s must be a list of strings" % name) |
| 232 | return value |
| 233 | |
| 234 | pragma = info.pragma |
| 235 | |
| 236 | if name == 'doc-required': |
| 237 | if not isinstance(value, bool): |
| 238 | raise QAPISemError(info, |
| 239 | "pragma 'doc-required' must be boolean") |
| 240 | pragma.doc_required = value |
| 241 | elif name == 'command-name-exceptions': |
| 242 | pragma.command_name_exceptions = check_list_str(name, value) |
| 243 | elif name == 'command-returns-exceptions': |
| 244 | pragma.command_returns_exceptions = check_list_str(name, value) |
| 245 | elif name == 'documentation-exceptions': |
| 246 | pragma.documentation_exceptions = check_list_str(name, value) |
| 247 | elif name == 'member-name-exceptions': |
| 248 | pragma.member_name_exceptions = check_list_str(name, value) |
| 249 | else: |
| 250 | raise QAPISemError(info, "unknown pragma '%s'" % name) |
| 251 | |
| 252 | def accept(self, skip_comment: bool = True) -> None: |
| 253 | """ |
| 254 | Read and store the next token. |
| 255 | |
| 256 | :param skip_comment: |
| 257 | When false, return COMMENT tokens ("#"). |
| 258 | This is used when reading documentation blocks. |
| 259 | |
| 260 | :return: |
| 261 | None. Several instance attributes are updated instead: |
| 262 | |
| 263 | - ``.tok`` represents the token type. See below for values. |
| 264 | - ``.info`` describes the token's source location. |
| 265 | - ``.val`` is the token's value, if any. See below. |
| 266 | - ``.pos`` is the buffer index of the first character of |
| 267 | the token. |
| 268 | |
| 269 | * Single-character tokens: |
| 270 | |
| 271 | These are "{", "}", ":", ",", "[", and "]". |
| 272 | ``.tok`` holds the single character and ``.val`` is None. |
| 273 | |
| 274 | * Multi-character tokens: |
| 275 | |
| 276 | * COMMENT: |
| 277 | |
| 278 | This token is not normally returned by the lexer, but it can |
| 279 | be when ``skip_comment`` is False. ``.tok`` is "#", and |
| 280 | ``.val`` is a string including all chars until end-of-line, |
| 281 | including the "#" itself. |
| 282 | |
| 283 | * STRING: |
| 284 | |
| 285 | ``.tok`` is "'", the single quote. ``.val`` contains the |
| 286 | string, excluding the surrounding quotes. |
| 287 | |
| 288 | * TRUE and FALSE: |
| 289 | |
| 290 | ``.tok`` is either "t" or "f", ``.val`` will be the |
| 291 | corresponding bool value. |
| 292 | |
| 293 | * EOF: |
| 294 | |
| 295 | ``.tok`` and ``.val`` will both be None at EOF. |
| 296 | """ |
| 297 | while True: |
| 298 | self.tok = self.src[self.cursor] |
| 299 | self.pos = self.cursor |
| 300 | self.cursor += 1 |
| 301 | self.val = None |
| 302 | |
| 303 | if self.tok == '#': |
| 304 | if self.src[self.cursor] == '#': |
| 305 | # Start of doc comment |
| 306 | skip_comment = False |
| 307 | self.cursor = self.src.find('\n', self.cursor) |
| 308 | if not skip_comment: |
| 309 | self.val = self.src[self.pos:self.cursor] |
| 310 | return |
| 311 | elif self.tok in '{}:,[]': |
| 312 | return |
| 313 | elif self.tok == "'": |
| 314 | # Note: we accept only printable ASCII |
| 315 | string = '' |
| 316 | esc = False |
| 317 | while True: |
| 318 | ch = self.src[self.cursor] |
| 319 | self.cursor += 1 |
| 320 | if ch == '\n': |
| 321 | raise QAPIParseError(self, "missing terminating \"'\"") |
| 322 | if esc: |
| 323 | # Note: we recognize only \\ because we have |
| 324 | # no use for funny characters in strings |
| 325 | if ch != '\\': |
| 326 | raise QAPIParseError(self, |
| 327 | "unknown escape \\%s" % ch) |
| 328 | esc = False |
| 329 | elif ch == '\\': |
| 330 | esc = True |
| 331 | continue |
| 332 | elif ch == "'": |
| 333 | self.val = string |
| 334 | return |
| 335 | if ord(ch) < 32 or ord(ch) >= 127: |
| 336 | raise QAPIParseError( |
| 337 | self, "funny character in string") |
| 338 | string += ch |
| 339 | elif self.src.startswith('true', self.pos): |
| 340 | self.val = True |
| 341 | self.cursor += 3 |
| 342 | return |
| 343 | elif self.src.startswith('false', self.pos): |
| 344 | self.val = False |
| 345 | self.cursor += 4 |
| 346 | return |
| 347 | elif self.tok == '\n': |
| 348 | if self.cursor == len(self.src): |
| 349 | self.tok = None |
| 350 | return |
| 351 | self.info = self.info.next_line() |
| 352 | self.line_pos = self.cursor |
| 353 | elif not self.tok.isspace(): |
| 354 | # Show up to next structural, whitespace or quote |
| 355 | # character |
| 356 | match = must_match('[^[\\]{}:,\\s\']+', |
| 357 | self.src[self.cursor-1:]) |
| 358 | raise QAPIParseError(self, "stray '%s'" % match.group(0)) |
| 359 | |
| 360 | def get_members(self) -> Dict[str, object]: |
| 361 | expr: Dict[str, object] = {} |
| 362 | if self.tok == '}': |
| 363 | self.accept() |
| 364 | return expr |
| 365 | if self.tok != "'": |
| 366 | raise QAPIParseError(self, "expected string or '}'") |
| 367 | while True: |
| 368 | key = self.val |
| 369 | assert isinstance(key, str) # Guaranteed by tok == "'" |
| 370 | |
| 371 | self.accept() |
| 372 | if self.tok != ':': |
| 373 | raise QAPIParseError(self, "expected ':'") |
| 374 | self.accept() |
| 375 | if key in expr: |
| 376 | raise QAPIParseError(self, "duplicate key '%s'" % key) |
| 377 | expr[key] = self.get_expr() |
| 378 | if self.tok == '}': |
| 379 | self.accept() |
| 380 | return expr |
| 381 | if self.tok != ',': |
| 382 | raise QAPIParseError(self, "expected ',' or '}'") |
| 383 | self.accept() |
| 384 | if self.tok != "'": |
| 385 | raise QAPIParseError(self, "expected string") |
| 386 | |
| 387 | def get_values(self) -> List[object]: |
| 388 | expr: List[object] = [] |
| 389 | if self.tok == ']': |
| 390 | self.accept() |
| 391 | return expr |
| 392 | if self.tok not in tuple("{['tf"): |
| 393 | raise QAPIParseError( |
| 394 | self, "expected '{', '[', ']', string, or boolean") |
| 395 | while True: |
| 396 | expr.append(self.get_expr()) |
| 397 | if self.tok == ']': |
| 398 | self.accept() |
| 399 | return expr |
| 400 | if self.tok != ',': |
| 401 | raise QAPIParseError(self, "expected ',' or ']'") |
| 402 | self.accept() |
| 403 | |
| 404 | def get_expr(self) -> _ExprValue: |
| 405 | expr: _ExprValue |
| 406 | if self.tok == '{': |
| 407 | self.accept() |
| 408 | expr = self.get_members() |
| 409 | elif self.tok == '[': |
| 410 | self.accept() |
| 411 | expr = self.get_values() |
| 412 | elif self.tok in tuple("'tf"): |
| 413 | assert isinstance(self.val, (str, bool)) |
| 414 | expr = self.val |
| 415 | self.accept() |
| 416 | else: |
| 417 | raise QAPIParseError( |
| 418 | self, "expected '{', '[', string, or boolean") |
| 419 | return expr |
| 420 | |
| 421 | def get_doc_line(self) -> Optional[str]: |
| 422 | if self.tok != '#': |
| 423 | raise QAPIParseError( |
| 424 | self, "documentation comment must end with '##'") |
| 425 | assert isinstance(self.val, str) |
| 426 | if self.val.startswith('##'): |
| 427 | # End of doc comment |
| 428 | if self.val != '##': |
| 429 | raise QAPIParseError( |
| 430 | self, "junk after '##' at end of documentation comment") |
| 431 | self._literal_mode = False |
| 432 | return None |
| 433 | if self.val == '#': |
| 434 | return '' |
| 435 | if self.val[1] != ' ': |
| 436 | raise QAPIParseError(self, "missing space after #") |
| 437 | |
| 438 | line = self.val[2:].rstrip() |
| 439 | |
| 440 | if re.match(r'(\.\. +qmp-example)? *::$', line): |
| 441 | self._literal_mode = True |
| 442 | self._literal_mode_indent = 0 |
| 443 | elif self._literal_mode and line: |
| 444 | indent = must_match(r'\s*', line).end() |
| 445 | if self._literal_mode_indent == 0: |
| 446 | self._literal_mode_indent = indent |
| 447 | elif indent < self._literal_mode_indent: |
| 448 | # ReST directives stop at decreasing indentation |
| 449 | self._literal_mode = False |
| 450 | |
| 451 | if not self._literal_mode: |
| 452 | self._validate_doc_line_format(line) |
| 453 | |
| 454 | return line |
| 455 | |
| 456 | def _validate_doc_line_format(self, line: str) -> None: |
| 457 | """ |
| 458 | Validate documentation format rules for a single line: |
| 459 | 1. Lines should not exceed 70 characters |
| 460 | 2. Sentences should be separated by two spaces |
| 461 | """ |
| 462 | full_line_length = len(line) + 2 # "# " = 2 characters |
| 463 | if full_line_length > 70: |
| 464 | # Skip URL lines - they can't be broken |
| 465 | if re.match(r' *(https?|ftp)://[^ ]*$', line): |
| 466 | pass |
| 467 | else: |
| 468 | raise QAPIParseError( |
| 469 | self, "documentation line longer than 70 characters") |
| 470 | |
| 471 | single_space_pattern = r'(\be\.g\.|^ *\d\.|([.!?])) [A-Z0-9(]' |
| 472 | for m in list(re.finditer(single_space_pattern, line)): |
| 473 | if not m.group(2): |
| 474 | continue |
| 475 | # HACK so the error message points to the offending spot |
| 476 | self.pos = self.line_pos + 2 + m.start(2) + 1 |
| 477 | raise QAPIParseError( |
| 478 | self, "Use two spaces between sentences\n" |
| 479 | "If this not the end of a sentence, please report a bug.") |
| 480 | |
| 481 | @staticmethod |
| 482 | def _match_at_name_colon(string: str) -> Optional[Match[str]]: |
| 483 | return re.match(r'@([^:]*): *', string) |
| 484 | |
| 485 | def get_doc_indented(self, doc: 'QAPIDoc') -> Optional[str]: |
| 486 | self.accept(False) |
| 487 | line = self.get_doc_line() |
| 488 | while line == '': |
| 489 | doc.append_line(line) |
| 490 | self.accept(False) |
| 491 | line = self.get_doc_line() |
| 492 | if line is None: |
| 493 | return line |
| 494 | indent = must_match(r'\s*', line).end() |
| 495 | if not indent: |
| 496 | return line |
| 497 | doc.append_line(line) |
| 498 | prev_line_blank = False |
| 499 | while True: |
| 500 | self.accept(False) |
| 501 | line = self.get_doc_line() |
| 502 | if line is None: |
| 503 | return line |
| 504 | if self._match_at_name_colon(line): |
| 505 | return line |
| 506 | cur_indent = must_match(r'\s*', line).end() |
| 507 | if line != '' and cur_indent < indent: |
| 508 | if prev_line_blank: |
| 509 | return line |
| 510 | raise QAPIParseError( |
| 511 | self, |
| 512 | "unexpected de-indent (expected at least %d spaces)" % |
| 513 | indent) |
| 514 | doc.append_line(line) |
| 515 | prev_line_blank = True |
| 516 | |
| 517 | def get_doc_paragraph(self, doc: 'QAPIDoc') -> Optional[str]: |
| 518 | while True: |
| 519 | self.accept(False) |
| 520 | line = self.get_doc_line() |
| 521 | if line is None: |
| 522 | return line |
| 523 | if line == '': |
| 524 | return line |
| 525 | doc.append_line(line) |
| 526 | |
| 527 | def get_doc(self) -> 'QAPIDoc': |
| 528 | if self.val != '##': |
| 529 | raise QAPIParseError( |
| 530 | self, "junk after '##' at start of documentation comment") |
| 531 | info = self.info |
| 532 | self.accept(False) |
| 533 | line = self.get_doc_line() |
| 534 | if line is not None and line.startswith('@'): |
| 535 | # Definition documentation |
| 536 | if not line.endswith(':'): |
| 537 | raise QAPIParseError(self, "line should end with ':'") |
| 538 | # Invalid names are not checked here, but the name |
| 539 | # provided *must* match the following definition, |
| 540 | # which *is* validated in expr.py. |
| 541 | symbol = line[1:-1] |
| 542 | if not symbol: |
| 543 | raise QAPIParseError(self, "name required after '@'") |
| 544 | doc = QAPIDoc(info, symbol) |
| 545 | doc.all_sections.append(QAPIDoc.Section(info, QAPIDoc.Kind.INTRO)) |
| 546 | line = self.get_doc_indented(doc) |
| 547 | no_more_args = False |
| 548 | |
| 549 | while line is not None: |
| 550 | # Blank lines |
| 551 | while line == '': |
| 552 | self.accept(False) |
| 553 | line = self.get_doc_line() |
| 554 | if line is None: |
| 555 | break |
| 556 | # Non-blank line, first of a section |
| 557 | if line == 'Features:': |
| 558 | if doc.has_features: |
| 559 | raise QAPIParseError( |
| 560 | self, "duplicated 'Features:' line") |
| 561 | self.accept(False) |
| 562 | line = self.get_doc_line() |
| 563 | while line == '': |
| 564 | self.accept(False) |
| 565 | line = self.get_doc_line() |
| 566 | while (line is not None |
| 567 | and (match := self._match_at_name_colon(line))): |
| 568 | doc.new_feature(self.info, match.group(1)) |
| 569 | text = line[match.end():] |
| 570 | if text: |
| 571 | doc.append_line(text) |
| 572 | line = self.get_doc_indented(doc) |
| 573 | if not doc.has_features: |
| 574 | raise QAPIParseError( |
| 575 | self, 'feature descriptions expected') |
| 576 | no_more_args = True |
| 577 | elif match := self._match_at_name_colon(line): |
| 578 | # description |
| 579 | if no_more_args: |
| 580 | raise QAPIParseError( |
| 581 | self, |
| 582 | "description of '@%s:' follows a section" |
| 583 | % match.group(1)) |
| 584 | while (line is not None |
| 585 | and (match := self._match_at_name_colon(line))): |
| 586 | doc.new_argument(self.info, match.group(1)) |
| 587 | text = line[match.end():] |
| 588 | if text: |
| 589 | doc.append_line(text) |
| 590 | line = self.get_doc_indented(doc) |
| 591 | no_more_args = True |
| 592 | elif match := re.match( |
| 593 | r'(Returns|Errors|Since|Notes?|Examples?|TODO)' |
| 594 | r'(?!::): *', |
| 595 | line, |
| 596 | ): |
| 597 | # tagged section |
| 598 | |
| 599 | # Note: "sections" with two colons are left alone as |
| 600 | # rST markup and not interpreted as a section heading. |
| 601 | |
| 602 | # TODO: Remove these errors sometime in 2025 or so |
| 603 | # after we've fully transitioned to the new qapidoc |
| 604 | # generator. |
| 605 | |
| 606 | # See commit message for more markup suggestions O:-) |
| 607 | if 'Note' in match.group(1): |
| 608 | emsg = ( |
| 609 | f"The '{match.group(1)}' section is no longer " |
| 610 | "supported. Please use rST's '.. note::' or " |
| 611 | "'.. admonition:: notes' directives, or another " |
| 612 | "suitable admonition instead." |
| 613 | ) |
| 614 | raise QAPIParseError(self, emsg) |
| 615 | |
| 616 | if 'Example' in match.group(1): |
| 617 | emsg = ( |
| 618 | f"The '{match.group(1)}' section is no longer " |
| 619 | "supported. Please use the '.. qmp-example::' " |
| 620 | "directive, or other suitable markup instead." |
| 621 | ) |
| 622 | raise QAPIParseError(self, emsg) |
| 623 | |
| 624 | doc.new_tagged_section( |
| 625 | self.info, |
| 626 | QAPIDoc.Kind.from_string(match.group(1)) |
| 627 | ) |
| 628 | text = line[match.end():] |
| 629 | if text: |
| 630 | doc.append_line(text) |
| 631 | line = self.get_doc_indented(doc) |
| 632 | no_more_args = True |
| 633 | else: |
| 634 | # plain paragraph(s) |
| 635 | doc.ensure_untagged_section(self.info) |
| 636 | doc.append_line(line) |
| 637 | line = self.get_doc_paragraph(doc) |
| 638 | else: |
| 639 | # Free-form documentation |
| 640 | doc = QAPIDoc(info) |
| 641 | doc.ensure_untagged_section(self.info) |
| 642 | while line is not None: |
| 643 | if match := self._match_at_name_colon(line): |
| 644 | raise QAPIParseError( |
| 645 | self, |
| 646 | "'@%s:' not allowed in free-form documentation" |
| 647 | % match.group(1)) |
| 648 | doc.append_line(line) |
| 649 | self.accept(False) |
| 650 | line = self.get_doc_line() |
| 651 | |
| 652 | self.accept() |
| 653 | doc.end() |
| 654 | return doc |
| 655 | |
| 656 | |
| 657 | class QAPIDoc: |
| 658 | """ |
| 659 | A documentation comment block, either definition or free-form |
| 660 | |
| 661 | Definition documentation blocks consist of |
| 662 | |
| 663 | * a body section: one line naming the definition, followed by an |
| 664 | overview (any number of lines) |
| 665 | |
| 666 | * argument sections: a description of each argument (for commands |
| 667 | and events) or member (for structs, unions and alternates) |
| 668 | |
| 669 | * features sections: a description of each feature flag |
| 670 | |
| 671 | * additional (non-argument) sections, possibly tagged |
| 672 | |
| 673 | Free-form documentation blocks consist only of a body section. |
| 674 | """ |
| 675 | |
| 676 | class Kind(enum.Enum): |
| 677 | PLAIN = 0 |
| 678 | MEMBER = 1 |
| 679 | FEATURE = 2 |
| 680 | RETURNS = 3 |
| 681 | ERRORS = 4 |
| 682 | SINCE = 5 |
| 683 | TODO = 6 |
| 684 | INTRO = 7 |
| 685 | |
| 686 | @staticmethod |
| 687 | def from_string(kind: str) -> 'QAPIDoc.Kind': |
| 688 | return QAPIDoc.Kind[kind.upper()] |
| 689 | |
| 690 | def __str__(self) -> str: |
| 691 | return self.name.title() |
| 692 | |
| 693 | class Section: |
| 694 | # pylint: disable=too-few-public-methods |
| 695 | def __init__( |
| 696 | self, |
| 697 | info: QAPISourceInfo, |
| 698 | kind: 'QAPIDoc.Kind', |
| 699 | ): |
| 700 | # section source info, i.e. where it begins |
| 701 | self.info = info |
| 702 | # section kind |
| 703 | self.kind = kind |
| 704 | # section text without tag |
| 705 | self.text = '' |
| 706 | |
| 707 | def __repr__(self) -> str: |
| 708 | return f"<QAPIDoc.Section kind={self.kind!r} text={self.text!r}>" |
| 709 | |
| 710 | def append_line(self, line: str) -> None: |
| 711 | self.text += line + '\n' |
| 712 | |
| 713 | class ArgSection(Section): |
| 714 | def __init__( |
| 715 | self, |
| 716 | info: QAPISourceInfo, |
| 717 | kind: 'QAPIDoc.Kind', |
| 718 | name: str |
| 719 | ): |
| 720 | super().__init__(info, kind) |
| 721 | self.name = name |
| 722 | self.member: Optional['QAPISchemaMember'] = None |
| 723 | |
| 724 | def connect(self, member: 'QAPISchemaMember') -> None: |
| 725 | self.member = member |
| 726 | |
| 727 | def __init__(self, info: QAPISourceInfo, symbol: Optional[str] = None): |
| 728 | # info points to the doc comment block's first line |
| 729 | self.info = info |
| 730 | # definition doc's symbol, None for free-form doc |
| 731 | self.symbol: Optional[str] = symbol |
| 732 | # the sections in textual order |
| 733 | self.all_sections: List[QAPIDoc.Section] = [] |
| 734 | # dicts mapping parameter/feature names to their description |
| 735 | self._args: Dict[str, QAPIDoc.ArgSection] = {} |
| 736 | self._features: Dict[str, QAPIDoc.ArgSection] = {} |
| 737 | # a command's "Returns" and "Errors" section |
| 738 | self._returns: Optional[QAPIDoc.Section] = None |
| 739 | self._errors: Optional[QAPIDoc.Section] = None |
| 740 | # "Since" section |
| 741 | self.since: Optional[QAPIDoc.Section] = None |
| 742 | |
| 743 | @property |
| 744 | def has_features(self) -> bool: |
| 745 | return bool(self._features) |
| 746 | |
| 747 | def end(self) -> None: |
| 748 | for section in self.all_sections: |
| 749 | section.text = section.text.strip('\n') |
| 750 | if not (section.kind.name in ("INTRO", "PLAIN") or section.text): |
| 751 | raise QAPISemError( |
| 752 | section.info, "text required after '%s:'" % section.kind) |
| 753 | |
| 754 | def ensure_untagged_section(self, info: QAPISourceInfo) -> None: |
| 755 | kind = QAPIDoc.Kind.PLAIN |
| 756 | |
| 757 | if self.all_sections and self.all_sections[-1].kind == kind: |
| 758 | # extend current section |
| 759 | section = self.all_sections[-1] |
| 760 | if not section.text: |
| 761 | # Section is empty so far; update info to start *here*. |
| 762 | section.info = info |
| 763 | section.text += '\n' |
| 764 | return |
| 765 | |
| 766 | # start new section |
| 767 | section = self.Section(info, kind) |
| 768 | self.all_sections.append(section) |
| 769 | |
| 770 | def new_tagged_section( |
| 771 | self, |
| 772 | info: QAPISourceInfo, |
| 773 | kind: 'QAPIDoc.Kind', |
| 774 | ) -> None: |
| 775 | section = self.Section(info, kind) |
| 776 | if kind == QAPIDoc.Kind.RETURNS: |
| 777 | if self._returns: |
| 778 | raise QAPISemError( |
| 779 | info, "duplicated '%s' section" % kind) |
| 780 | self._returns = section |
| 781 | elif kind == QAPIDoc.Kind.ERRORS: |
| 782 | if self._errors: |
| 783 | raise QAPISemError( |
| 784 | info, "duplicated '%s' section" % kind) |
| 785 | self._errors = section |
| 786 | elif kind == QAPIDoc.Kind.SINCE: |
| 787 | if self.since: |
| 788 | raise QAPISemError( |
| 789 | info, "duplicated '%s' section" % kind) |
| 790 | self.since = section |
| 791 | self.all_sections.append(section) |
| 792 | |
| 793 | def _new_description( |
| 794 | self, |
| 795 | info: QAPISourceInfo, |
| 796 | name: str, |
| 797 | kind: 'QAPIDoc.Kind', |
| 798 | desc: Dict[str, ArgSection] |
| 799 | ) -> None: |
| 800 | if not name: |
| 801 | raise QAPISemError(info, "invalid parameter name") |
| 802 | if name in desc: |
| 803 | raise QAPISemError(info, "'%s' parameter name duplicated" % name) |
| 804 | section = self.ArgSection(info, kind, name) |
| 805 | self.all_sections.append(section) |
| 806 | desc[name] = section |
| 807 | |
| 808 | def new_argument(self, info: QAPISourceInfo, name: str) -> None: |
| 809 | self._new_description(info, name, QAPIDoc.Kind.MEMBER, self._args) |
| 810 | |
| 811 | def new_feature(self, info: QAPISourceInfo, name: str) -> None: |
| 812 | self._new_description(info, name, QAPIDoc.Kind.FEATURE, self._features) |
| 813 | |
| 814 | def append_line(self, line: str) -> None: |
| 815 | self.all_sections[-1].append_line(line) |
| 816 | |
| 817 | def _insert_near_kind( |
| 818 | self, |
| 819 | kind: 'QAPIDoc.Kind', |
| 820 | new_sect: 'QAPIDoc.Section', |
| 821 | after: bool = False, |
| 822 | ) -> bool: |
| 823 | """Insert or append a new doc section at a specific point.""" |
| 824 | for idx, sect in enumerate(reversed(self.all_sections)): |
| 825 | if sect.kind == kind: |
| 826 | pos = len(self.all_sections) - idx - 1 |
| 827 | if after: |
| 828 | pos += 1 |
| 829 | self.all_sections.insert(pos, new_sect) |
| 830 | return True |
| 831 | return False |
| 832 | |
| 833 | def _insert_after_intro( |
| 834 | self, |
| 835 | section: 'QAPIDoc.Section', |
| 836 | ) -> None: |
| 837 | """ |
| 838 | Insert a section immediately after the intro section. |
| 839 | |
| 840 | While we convert PLAIN sections to INTRO sections, all |
| 841 | contiguous INTRO/PLAIN sections at the start of a QAPIDoc |
| 842 | section list are treated as "the intro". |
| 843 | |
| 844 | Once INTRO conversion is complete, this helper will no longer be |
| 845 | needed and ``_insert_near_kind(QAPIDoc.Kind.INTRO, ...)`` will |
| 846 | be sufficient. |
| 847 | """ |
| 848 | index = 0 |
| 849 | for index, ref_section in enumerate(self.all_sections): |
| 850 | if ref_section.kind.name in ("PLAIN", "INTRO"): |
| 851 | continue |
| 852 | break |
| 853 | else: |
| 854 | index += 1 |
| 855 | |
| 856 | self.all_sections.insert(index, section) |
| 857 | |
| 858 | def append_member_stub(self, stub: 'QAPIDoc.Section') -> None: |
| 859 | |
| 860 | """ |
| 861 | Append a stub section after any Member sections. |
| 862 | """ |
| 863 | if self._insert_near_kind(QAPIDoc.Kind.MEMBER, stub, True): |
| 864 | return |
| 865 | |
| 866 | # No MEMBER sections present. Insert after INTRO/PLAIN sections. |
| 867 | self._insert_after_intro(stub) |
| 868 | |
| 869 | def connect_member(self, member: 'QAPISchemaMember') -> None: |
| 870 | if member.name not in self._args: |
| 871 | assert member.info |
| 872 | if self.symbol not in member.info.pragma.documentation_exceptions: |
| 873 | raise QAPISemError(member.info, |
| 874 | "%s '%s' lacks documentation" |
| 875 | % (member.role, member.name)) |
| 876 | # Insert stub documentation section for missing member docs. |
| 877 | # TODO: drop when undocumented members are outlawed |
| 878 | stub_section = QAPIDoc.ArgSection( |
| 879 | self.info, QAPIDoc.Kind.MEMBER, member.name) |
| 880 | self._args[member.name] = stub_section |
| 881 | self.append_member_stub(stub_section) |
| 882 | |
| 883 | self._args[member.name].connect(member) |
| 884 | |
| 885 | def connect_feature(self, feature: 'QAPISchemaFeature') -> None: |
| 886 | if feature.name not in self._features: |
| 887 | raise QAPISemError(feature.info, |
| 888 | "feature '%s' lacks documentation" |
| 889 | % feature.name) |
| 890 | self._features[feature.name].connect(feature) |
| 891 | |
| 892 | def ensure_returns(self, info: QAPISourceInfo) -> None: |
| 893 | # This is more complicated than it ought to be. The doc |
| 894 | # parser should already know where a generated RETURNS section |
| 895 | # should go. It currently doesn't, mostly because it accepts |
| 896 | # tagged sections in any order. |
| 897 | # |
| 898 | # TODO: Tighten doc syntax and simplify. |
| 899 | |
| 900 | if any(s.kind == QAPIDoc.Kind.RETURNS for s in self.all_sections): |
| 901 | return |
| 902 | |
| 903 | # Stub "Returns" section for undocumented returns value |
| 904 | stub = QAPIDoc.Section(info, QAPIDoc.Kind.RETURNS) |
| 905 | |
| 906 | if any(self._insert_near_kind(kind, stub, after) for kind, after in ( |
| 907 | # 1. If arguments, right after those. |
| 908 | (QAPIDoc.Kind.MEMBER, True), |
| 909 | # 2. Elif errors, right *before* those. |
| 910 | (QAPIDoc.Kind.ERRORS, False), |
| 911 | # 3. Elif features, right *before* those. |
| 912 | (QAPIDoc.Kind.FEATURE, False), |
| 913 | )): |
| 914 | return |
| 915 | |
| 916 | # Otherwise, it should go right after the intro. |
| 917 | self._insert_after_intro(stub) |
| 918 | |
| 919 | def check_expr(self, expr: QAPIExpression) -> None: |
| 920 | if 'command' in expr: |
| 921 | if self._returns and 'returns' not in expr: |
| 922 | raise QAPISemError( |
| 923 | self._returns.info, |
| 924 | "'Returns' section, but command doesn't return anything") |
| 925 | else: |
| 926 | if self._returns: |
| 927 | raise QAPISemError( |
| 928 | self._returns.info, |
| 929 | "'Returns' section is only valid for commands") |
| 930 | if self._errors: |
| 931 | raise QAPISemError( |
| 932 | self._errors.info, |
| 933 | "'Errors' section is only valid for commands") |
| 934 | |
| 935 | def check(self) -> None: |
| 936 | |
| 937 | def check_args_section( |
| 938 | args: Dict[str, QAPIDoc.ArgSection], what: str |
| 939 | ) -> None: |
| 940 | bogus = [name for name, section in args.items() |
| 941 | if not section.member] |
| 942 | if bogus: |
| 943 | raise QAPISemError( |
| 944 | args[bogus[0]].info, |
| 945 | "documented %s%s '%s' %s not exist" % ( |
| 946 | what, |
| 947 | "s" if len(bogus) > 1 else "", |
| 948 | "', '".join(bogus), |
| 949 | "do" if len(bogus) > 1 else "does" |
| 950 | )) |
| 951 | |
| 952 | check_args_section(self._args, 'member') |
| 953 | check_args_section(self._features, 'feature') |