master
py 701 lines 23.3 KB
Raw
1 # coding=utf-8
2 #
3 # QEMU qapidoc QAPI file parsing extension
4 #
5 # Copyright (c) 2024-2025 Red Hat
6 # Copyright (c) 2020 Linaro
7 #
8 # This work is licensed under the terms of the GNU GPLv2 or later.
9 # See the COPYING file in the top-level directory.
10
11 """
12 qapidoc is a Sphinx extension that implements the qapi-doc directive
13
14 The purpose of this extension is to read the documentation comments
15 in QAPI schema files, and insert them all into the current document.
16
17 It implements one new rST directive, "qapi-doc::".
18 Each qapi-doc:: directive takes one argument, which is the
19 pathname of the schema file to process, relative to the source tree.
20
21 The docs/conf.py file must set the qapidoc_srctree config value to
22 the root of the QEMU source tree.
23
24 The Sphinx documentation on writing extensions is at:
25 https://www.sphinx-doc.org/en/master/development/index.html
26 """
27
28 from __future__ import annotations
29
30
31 __version__ = "2.0"
32
33 from contextlib import contextmanager
34 import os
35 from pathlib import Path
36 import re
37 import sys
38 import textwrap
39 from typing import TYPE_CHECKING
40
41 from docutils import nodes
42 from docutils.parsers.rst import directives
43 from docutils.statemachine import StringList
44 from qapi.error import QAPIError
45 from qapi.parser import QAPIDoc
46 from qapi.schema import (
47 QAPISchema,
48 QAPISchemaArrayType,
49 QAPISchemaCommand,
50 QAPISchemaDefinition,
51 QAPISchemaEnumMember,
52 QAPISchemaEvent,
53 QAPISchemaFeature,
54 QAPISchemaMember,
55 QAPISchemaObjectType,
56 QAPISchemaObjectTypeMember,
57 QAPISchemaType,
58 QAPISchemaVisitor,
59 )
60 from qapi.source import QAPISourceInfo
61 from sphinx import addnodes
62 from sphinx.directives.code import CodeBlock
63 from sphinx.errors import ExtensionError
64 from sphinx.util import logging
65 from sphinx.util.docutils import SphinxDirective, switch_source_input
66 from sphinx.util.nodes import nested_parse_with_titles
67
68
69 if TYPE_CHECKING:
70 from typing import (
71 Any,
72 Generator,
73 List,
74 Optional,
75 Sequence,
76 Union,
77 )
78
79 from sphinx.application import Sphinx
80 from sphinx.util.typing import ExtensionMetadata
81
82
83 logger = logging.getLogger(__name__)
84
85
86 class Transmogrifier:
87 # pylint: disable=too-many-public-methods
88
89 # Field names used for different entity types:
90 field_types = {
91 "enum": "value",
92 "struct": "memb",
93 "union": "memb",
94 "event": "memb",
95 "command": "arg",
96 "alternate": "alt",
97 }
98
99 def __init__(self) -> None:
100 self._curr_ent: Optional[QAPISchemaDefinition] = None
101 self._result = StringList()
102 self.indent = 0
103
104 @property
105 def result(self) -> StringList:
106 return self._result
107
108 @property
109 def entity(self) -> QAPISchemaDefinition:
110 assert self._curr_ent is not None
111 return self._curr_ent
112
113 @property
114 def member_field_type(self) -> str:
115 return self.field_types[self.entity.meta]
116
117 # General-purpose rST generation functions
118
119 def get_indent(self) -> str:
120 return " " * self.indent
121
122 @contextmanager
123 def indented(self) -> Generator[None]:
124 self.indent += 1
125 try:
126 yield
127 finally:
128 self.indent -= 1
129
130 def add_line_raw(self, line: str, source: str, *lineno: int) -> None:
131 """Append one line of generated reST to the output."""
132
133 # NB: Sphinx uses zero-indexed lines; subtract one.
134 lineno = tuple((n - 1 for n in lineno))
135
136 if line.strip():
137 # not a blank line
138 self._result.append(
139 self.get_indent() + line.rstrip("\n"), source, *lineno
140 )
141 else:
142 self._result.append("", source, *lineno)
143
144 def add_line(self, content: str, info: QAPISourceInfo) -> None:
145 # NB: We *require* an info object; this works out OK because we
146 # don't document built-in objects that don't have
147 # one. Everything else should.
148 self.add_line_raw(content, info.fname, info.line)
149
150 def add_lines(
151 self,
152 content: str,
153 info: QAPISourceInfo,
154 dedent: bool = False,
155 ) -> None:
156 lines = content.splitlines(True)
157
158 if dedent:
159 lines = textwrap.dedent(content).splitlines(True)
160 else:
161 lines = content.splitlines(True)
162
163 for i, line in enumerate(lines):
164 self.add_line_raw(line, info.fname, info.line + i)
165
166 def ensure_blank_line(self) -> None:
167 # Empty document -- no blank line required.
168 if not self._result:
169 return
170
171 # Last line isn't blank, add one.
172 if self._result[-1].strip(): # pylint: disable=no-member
173 fname, line = self._result.info(-1)
174 assert isinstance(line, int)
175 # New blank line is credited to one-after the current last line.
176 # +2: correct for zero/one index, then increment by one.
177 self.add_line_raw("", fname, line + 2)
178
179 def add_field(
180 self,
181 kind: str,
182 name: str,
183 body: str,
184 info: QAPISourceInfo,
185 typ: Optional[str] = None,
186 ) -> None:
187 if typ:
188 text = f":{kind} {typ} {name}: {body}"
189 else:
190 text = f":{kind} {name}: {body}"
191 self.add_lines(text, info)
192
193 def format_type(
194 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember]
195 ) -> Optional[str]:
196 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)):
197 return None
198
199 qapi_type = ent
200 optional = False
201 if isinstance(ent, QAPISchemaObjectTypeMember):
202 qapi_type = ent.type
203 optional = ent.optional
204
205 if isinstance(qapi_type, QAPISchemaArrayType):
206 ret = f"[{qapi_type.element_type.doc_type()}]"
207 else:
208 assert isinstance(qapi_type, QAPISchemaType)
209 tmp = qapi_type.doc_type()
210 assert tmp
211 ret = tmp
212 if optional:
213 ret += "?"
214
215 return ret
216
217 def generate_field(
218 self,
219 kind: str,
220 member: QAPISchemaMember,
221 body: str,
222 info: QAPISourceInfo,
223 ) -> None:
224 typ = self.format_type(member)
225 self.add_field(kind, member.name, body, info, typ)
226
227 @staticmethod
228 def reformat_arobase(text: str) -> str:
229 """ reformats @var to ``var`` """
230 return re.sub(r"@([\w-]+)", r"``\1``", text)
231
232 # Transmogrification helpers
233
234 def visit_plaintext(self, section: QAPIDoc.Section) -> None:
235 # Squelch empty paragraphs.
236 if not section.text:
237 return
238
239 # Intro sections, which are indented in QAPI source, need to
240 # be dedented to avoid accidental block quotes in ReST syntax.
241 dedent = bool(section.kind == QAPIDoc.Kind.INTRO)
242 self.ensure_blank_line()
243 self.add_lines(section.text, section.info, dedent)
244 self.ensure_blank_line()
245
246 def visit_member(self, section: QAPIDoc.ArgSection) -> None:
247 # FIXME: ifcond for members
248 # TODO: features for members (documented at entity-level,
249 # but sometimes defined per-member. Should we add such
250 # information to member descriptions when we can?)
251 assert section.member
252 self.generate_field(
253 self.member_field_type,
254 section.member,
255 # TODO drop fallbacks when undocumented members are outlawed
256 section.text if section.text else "Not documented",
257 section.info,
258 )
259
260 def visit_feature(self, section: QAPIDoc.ArgSection) -> None:
261 # FIXME - ifcond for features is not handled at all yet!
262 # Proposal: decorate the right-hand column with some graphical
263 # element to indicate conditional availability?
264 assert section.text # Guaranteed by parser.py
265 assert section.member
266
267 self.generate_field("feat", section.member, section.text, section.info)
268
269 def visit_returns(self, section: QAPIDoc.Section) -> None:
270 assert isinstance(self.entity, QAPISchemaCommand)
271 rtype = self.entity.ret_type
272 # return statements will not be present (and won't be
273 # autogenerated) for any command that doesn't return
274 # *something*, so rtype will always be defined here.
275 assert rtype
276
277 typ = self.format_type(rtype)
278 assert typ
279
280 if section.text:
281 self.add_field("return", typ, section.text, section.info)
282 else:
283 self.add_lines(f":return-nodesc: {typ}", section.info)
284
285 def visit_errors(self, section: QAPIDoc.Section) -> None:
286 # If the section text does not start with a space, it means text
287 # began on the same line as the "Error:" string and we should
288 # not insert a newline in this case.
289 if section.text[0].isspace():
290 text = f":error:\n{section.text}"
291 else:
292 text = f":error: {section.text}"
293 self.add_lines(text, section.info)
294
295 def preamble(self, ent: QAPISchemaDefinition) -> None:
296 """
297 Generate option lines for QAPI entity directives.
298 """
299 if ent.doc and ent.doc.since:
300 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE
301 # Generated from the entity's docblock; info location is exact.
302 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info)
303
304 if ent.ifcond.is_present():
305 doc = ent.ifcond.docgen()
306 assert ent.info
307 # Generated from entity definition; info location is approximate.
308 self.add_line(f":ifcond: {doc}", ent.info)
309
310 # Hoist special features such as :deprecated: and :unstable:
311 # into the options block for the entity. If, in the future, new
312 # special features are added, qapi-domain will chirp about
313 # unrecognized options and fail until they are handled in
314 # qapi-domain.
315 for feat in ent.features:
316 if feat.is_special():
317 # FIXME: handle ifcond if present. How to display that
318 # information is TBD.
319 # Generated from entity def; info location is approximate.
320 assert feat.info
321 self.add_line(f":{feat.name}:", feat.info)
322
323 self.ensure_blank_line()
324
325 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None:
326
327 def _get_target(
328 ent: QAPISchemaDefinition,
329 ) -> Optional[QAPISchemaDefinition]:
330 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)):
331 return ent.arg_type
332 if isinstance(ent, QAPISchemaObjectType):
333 return ent.base
334 return None
335
336 target = _get_target(ent)
337 if target is not None and not target.is_implicit():
338 assert ent.info
339 self.add_field(
340 self.member_field_type,
341 "q_dummy",
342 f"The members of :qapi:type:`{target.name}`.",
343 ent.info,
344 "q_dummy",
345 )
346
347 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None:
348 for variant in ent.branches.variants:
349 if variant.type.name == "q_empty":
350 continue
351 assert ent.info
352 self.add_field(
353 self.member_field_type,
354 "q_dummy",
355 f" When ``{ent.branches.tag_member.name}`` is "
356 f"``{variant.name}``: "
357 f"The members of :qapi:type:`{variant.type.name}`.",
358 ent.info,
359 "q_dummy",
360 )
361
362 def visit_sections(self, ent: QAPISchemaDefinition) -> None:
363 # Generate a placeholder right after the member section(s) which
364 # may be used to generate documentation for "The members of..."
365 # pointers in the rendered document.
366 #
367 # This is a temporary hack until the inliner is merged. Note
368 # that although we modify the caller's section list, the
369 # Sphinx document generator has its own copy of the parsed
370 # schema in memory, so this action does not interfere with
371 # other users of the QAPISchema or QAPIDoc objects outside of
372 # the document generator. Fishy, but not harmful.
373 if ent.doc:
374 ent.doc.append_member_stub(
375 QAPIDoc.ArgSection(
376 ent.doc.info, QAPIDoc.Kind.MEMBER, "q_dummy"
377 )
378 )
379
380 sections = ent.doc.all_sections if ent.doc else []
381
382 # Add sections in source order:
383 for section in sections:
384 section.text = self.reformat_arobase(section.text)
385
386 if section.kind.name in ("PLAIN", "INTRO"):
387 self.visit_plaintext(section)
388 elif section.kind == QAPIDoc.Kind.MEMBER:
389 assert isinstance(section, QAPIDoc.ArgSection)
390 if section.name == "q_dummy":
391 # Generate "The members of ..." entries if necessary
392 self._insert_member_pointer(ent)
393 else:
394 self.visit_member(section)
395 elif section.kind == QAPIDoc.Kind.FEATURE:
396 assert isinstance(section, QAPIDoc.ArgSection)
397 self.visit_feature(section)
398 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO):
399 # Since is handled in preamble, TODO is skipped intentionally.
400 pass
401 elif section.kind == QAPIDoc.Kind.RETURNS:
402 self.visit_returns(section)
403 elif section.kind == QAPIDoc.Kind.ERRORS:
404 self.visit_errors(section)
405 else:
406 assert False
407
408 self.ensure_blank_line()
409
410 # Transmogrification core methods
411
412 def visit_module(self, path: str) -> None:
413 name = Path(path).stem
414 # module directives are credited to the first line of a module file.
415 self.add_line_raw(f".. qapi:module:: {name}", path, 1)
416 self.ensure_blank_line()
417
418 def visit_freeform(self, doc: QAPIDoc) -> None:
419 assert len(doc.all_sections) == 1, doc.all_sections
420 body = doc.all_sections[0]
421 self.add_lines(self.reformat_arobase(body.text), doc.info)
422 self.ensure_blank_line()
423
424 def visit_entity(self, ent: QAPISchemaDefinition) -> None:
425 assert ent.info
426
427 try:
428 self._curr_ent = ent
429
430 # Squish structs and unions together into an "object" directive.
431 meta = ent.meta
432 if meta in ("struct", "union"):
433 meta = "object"
434
435 # This line gets credited to the start of the /definition/.
436 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info)
437 with self.indented():
438 self.preamble(ent)
439 self.visit_sections(ent)
440 finally:
441 self._curr_ent = None
442
443 def set_namespace(self, namespace: str, source: str, lineno: int) -> None:
444 self.add_line_raw(
445 f".. qapi:namespace:: {namespace}", source, lineno + 1
446 )
447 self.ensure_blank_line()
448
449
450 class QAPISchemaGenDepVisitor(QAPISchemaVisitor):
451 """A QAPI schema visitor which adds Sphinx dependencies each module
452
453 This class calls the Sphinx note_dependency() function to tell Sphinx
454 that the generated documentation output depends on the input
455 schema file associated with each module in the QAPI input.
456 """
457
458 def __init__(self, env: Any, qapidir: str) -> None:
459 self._env = env
460 self._qapidir = qapidir
461
462 def visit_module(self, name: str) -> None:
463 if name != "./builtin":
464 qapifile = self._qapidir + "/" + name
465 self._env.note_dependency(os.path.abspath(qapifile))
466 super().visit_module(name)
467
468
469 class NestedDirective(SphinxDirective):
470 def run(self) -> Sequence[nodes.Node]:
471 raise NotImplementedError
472
473 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None:
474 """
475 Parse rST source lines and add them to the specified node
476
477 Take the list of rST source lines rstlist, parse them as
478 rST, and add the resulting docutils nodes as children of node.
479 The nodes are parsed in a way that allows them to include
480 subheadings (titles) without confusing the rendering of
481 anything else.
482 """
483 with switch_source_input(self.state, rstlist):
484 nested_parse_with_titles(self.state, rstlist, node)
485
486
487 class QAPIDocDirective(NestedDirective):
488 """Extract documentation from the specified QAPI .json file"""
489
490 required_argument = 1
491 optional_arguments = 1
492 option_spec = {
493 "qapifile": directives.unchanged_required,
494 "namespace": directives.unchanged,
495 }
496 has_content = False
497
498 def transmogrify(self, schema: QAPISchema) -> nodes.Element:
499 logger.info("Transmogrifying QAPI to rST ...")
500 vis = Transmogrifier()
501 modules = set()
502
503 if "namespace" in self.options:
504 vis.set_namespace(
505 self.options["namespace"], *self.get_source_info()
506 )
507
508 for doc in schema.docs:
509 module_source = doc.info.fname
510 if module_source not in modules:
511 vis.visit_module(module_source)
512 modules.add(module_source)
513
514 if doc.symbol:
515 ent = schema.lookup_entity(doc.symbol)
516 assert isinstance(ent, QAPISchemaDefinition)
517 vis.visit_entity(ent)
518 else:
519 vis.visit_freeform(doc)
520
521 logger.info("Transmogrification complete.")
522
523 contentnode = nodes.section()
524 content = vis.result
525 titles_allowed = True
526
527 logger.info("Transmogrifier running nested parse ...")
528 with switch_source_input(self.state, content):
529 if titles_allowed:
530 node: nodes.Element = nodes.section()
531 node.document = self.state.document
532 nested_parse_with_titles(self.state, content, contentnode)
533 else:
534 node = nodes.paragraph()
535 node.document = self.state.document
536 self.state.nested_parse(content, 0, contentnode)
537 logger.info("Transmogrifier's nested parse completed.")
538
539 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"):
540 argname = "_".join(Path(self.arguments[0]).parts)
541 name = Path(argname).stem + ".ir"
542 self.write_intermediate(content, name)
543
544 sys.stdout.flush()
545 return contentnode
546
547 def write_intermediate(self, content: StringList, filename: str) -> None:
548 logger.info(
549 "writing intermediate rST for '%s' to '%s'",
550 self.arguments[0],
551 filename,
552 )
553
554 srctree = Path(self.env.app.config.qapidoc_srctree).resolve()
555 outlines = []
556 lcol_width = 0
557
558 for i, line in enumerate(content):
559 src, lineno = content.info(i)
560 srcpath = Path(src).resolve()
561 srcpath = srcpath.relative_to(srctree)
562
563 lcol = f"{srcpath}:{lineno:04d}"
564 lcol_width = max(lcol_width, len(lcol))
565 outlines.append((lcol, line))
566
567 with open(filename, "w", encoding="UTF-8") as outfile:
568 for lcol, rcol in outlines:
569 outfile.write(lcol.rjust(lcol_width))
570 outfile.write(" |")
571 if rcol:
572 outfile.write(f" {rcol}")
573 outfile.write("\n")
574
575 def run(self) -> Sequence[nodes.Node]:
576 env = self.state.document.settings.env
577 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0]
578 qapidir = os.path.dirname(qapifile)
579
580 try:
581 schema = QAPISchema(qapifile)
582
583 # First tell Sphinx about all the schema files that the
584 # output documentation depends on (including 'qapifile' itself)
585 schema.visit(QAPISchemaGenDepVisitor(env, qapidir))
586 except QAPIError as err:
587 # Launder QAPI parse errors into Sphinx extension errors
588 # so they are displayed nicely to the user
589 raise ExtensionError(str(err)) from err
590
591 contentnode = self.transmogrify(schema)
592 return contentnode.children
593
594
595 class QMPExample(CodeBlock, NestedDirective):
596 """
597 Custom admonition for QMP code examples.
598
599 When the :annotated: option is present, the body of this directive
600 is parsed as normal rST, but with any '::' code blocks set to use
601 the QMP lexer. Code blocks must be explicitly written by the user,
602 but this allows for intermingling explanatory paragraphs with
603 arbitrary rST syntax and code blocks for more involved examples.
604
605 When :annotated: is absent, the directive body is treated as a
606 simple standalone QMP code block literal.
607 """
608
609 required_argument = 0
610 optional_arguments = 0
611 has_content = True
612 option_spec = {
613 "annotated": directives.flag,
614 "title": directives.unchanged,
615 }
616
617 def _highlightlang(self) -> addnodes.highlightlang:
618 """Return the current highlightlang setting for the document"""
619 node = None
620 doc = self.state.document
621
622 if hasattr(doc, "findall"):
623 # docutils >= 0.18.1
624 for node in doc.findall(addnodes.highlightlang):
625 pass
626 else:
627 for elem in doc.traverse():
628 if isinstance(elem, addnodes.highlightlang):
629 node = elem
630
631 if node:
632 return node
633
634 # No explicit directive found, use defaults
635 node = addnodes.highlightlang(
636 lang=self.env.config.highlight_language,
637 force=False,
638 # Yes, Sphinx uses this value to effectively disable line
639 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯
640 linenothreshold=sys.maxsize,
641 )
642 return node
643
644 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]:
645 title = "Example:"
646 if "title" in self.options:
647 title = f"{title} {self.options['title']}"
648
649 admon = nodes.admonition(
650 "",
651 nodes.title("", title),
652 *content,
653 classes=["admonition", "admonition-example"],
654 )
655 return [admon]
656
657 def run_annotated(self) -> List[nodes.Node]:
658 lang_node = self._highlightlang()
659
660 content_node: nodes.Element = nodes.section()
661
662 # Configure QMP highlighting for "::" blocks, if needed
663 if lang_node["lang"] != "QMP":
664 content_node += addnodes.highlightlang(
665 lang="QMP",
666 force=False, # "True" ignores lexing errors
667 linenothreshold=lang_node["linenothreshold"],
668 )
669
670 self.do_parse(self.content, content_node)
671
672 # Restore prior language highlighting, if needed
673 if lang_node["lang"] != "QMP":
674 content_node += addnodes.highlightlang(**lang_node.attributes)
675
676 return content_node.children
677
678 def run(self) -> List[nodes.Node]:
679 annotated = "annotated" in self.options
680
681 if annotated:
682 content_nodes = self.run_annotated()
683 else:
684 self.arguments = ["QMP"]
685 content_nodes = super().run()
686
687 return self.admonition_wrap(*content_nodes)
688
689
690 def setup(app: Sphinx) -> ExtensionMetadata:
691 """Register qapi-doc directive with Sphinx"""
692 app.setup_extension("qapi_domain")
693 app.add_config_value("qapidoc_srctree", None, "env")
694 app.add_directive("qapi-doc", QAPIDocDirective)
695 app.add_directive("qmp-example", QMPExample)
696
697 return {
698 "version": __version__,
699 "parallel_read_safe": True,
700 "parallel_write_safe": True,
701 }