A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Doc] Use intersphinx and update entrypoints docs (#5125)
Cyrus Leung committed
May 31, 2024 at 00:59 UTC
a9bcc7afb23d208efaa1b47549fa93eaa1d9d6cf
4 files changed
+32
-13
docs/source/conf.py
+12
-1
@@ -80,7 +80,7 @@ def setup(app):
80
generate_examples()
81
82
83
-# Mock out external dependencies here.
83
+# Mock out external dependencies here, otherwise the autodoc pages may be blank.
84
autodoc_mock_imports = [
85
"cpuinfo",
86
"torch",
@@ -115,4 +115,15 @@ class MockedClassDocumenter(autodoc.ClassDocumenter):
115
116
autodoc.ClassDocumenter = MockedClassDocumenter
117
118
+intersphinx_mapping = {
119
+ 'python': ('https://docs.python.org/3', None),
120
+ 'typing_extensions':
121
+ ('https://typing-extensions.readthedocs.io/en/latest', None),
122
+ 'numpy': ('https://numpy.org/doc/stable', None),
123
+ 'torch': ('https://pytorch.org/docs/stable', None),
124
+ 'psutil': ('https://psutil.readthedocs.io/en/stable', None),
125
+}
126
+
127
+autodoc_preserve_defaults = True
128
+
129
navigation_with_keys = False
vllm/engine/async_llm_engine.py
-2
@@ -307,8 +307,6 @@ class AsyncLLMEngine:
307
generate method when there are requests in the waiting queue. The generate
308
method yields the outputs from the :class:`LLMEngine` to the caller.
309
310
- NOTE: For the comprehensive list of arguments, see :class:`LLMEngine`.
311
-
310
Args:
311
worker_use_ray: Whether to use Ray for model workers. Required for
312
distributed execution. Should be the same as
vllm/engine/llm_engine.py
+2
-2
@@ -70,8 +70,8 @@ class LLMEngine:
70
The :class:`~vllm.LLM` class wraps this class for offline batched inference
71
and the :class:`AsyncLLMEngine` class wraps this class for online serving.
72
73
- NOTE: The config arguments are derived from the :class:`~vllm.EngineArgs`
74
- class. For the comprehensive list of arguments, see :ref:`engine_args`.
73
+ The config arguments are derived from :class:`~vllm.EngineArgs`. (See
74
+ :ref:`engine_args`)
75
76
Args:
77
model_config: The configuration related to the LLM model.
vllm/entrypoints/llm.py
+18
-8
@@ -30,12 +30,6 @@ class LLM:
30
this class generates texts from the model, using an intelligent batching
31
mechanism and efficient memory management.
32
33
- NOTE: This class is intended to be used for offline inference. For online
34
- serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
35
-
36
- NOTE: For the comprehensive list of arguments, see
37
- :class:`~vllm.EngineArgs`.
38
-
33
Args:
34
model: The name or path of a HuggingFace Transformers model.
35
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
@@ -84,6 +78,12 @@ class LLM:
78
When a sequence has context length larger than this, we fall back
79
to eager mode.
80
disable_custom_all_reduce: See ParallelConfig
81
+ **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
82
+ :ref:`engine_args`)
83
+
84
+ Note:
85
+ This class is intended to be used for offline inference. For online
86
+ serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
87
"""
88
89
DEPRECATE_LEGACY: ClassVar[bool] = False
@@ -253,7 +253,7 @@ class LLM:
253
) -> List[RequestOutput]:
254
"""Generates the completions for the input prompts.
255
256
- NOTE: This class automatically batches the given prompts, considering
256
+ This class automatically batches the given prompts, considering
257
the memory constraint. For the best performance, put all of your prompts
258
into a single list and pass it to this method.
259
@@ -270,6 +270,11 @@ class LLM:
270
Returns:
271
A list of `RequestOutput` objects containing the
272
generated completions in the same order as the input prompts.
273
+
274
+ Note:
275
+ Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
276
+ considered legacy and may be deprecated in the future. You should
277
+ instead pass them via the ``inputs`` parameter.
278
"""
279
if prompt_token_ids is not None or multi_modal_data is not None:
280
inputs = self._convert_v1_inputs(
@@ -393,7 +398,7 @@ class LLM:
398
) -> List[EmbeddingRequestOutput]:
399
"""Generates the completions for the input prompts.
400
396
- NOTE: This class automatically batches the given prompts, considering
401
+ This class automatically batches the given prompts, considering
402
the memory constraint. For the best performance, put all of your prompts
403
into a single list and pass it to this method.
404
@@ -409,6 +414,11 @@ class LLM:
414
Returns:
415
A list of `EmbeddingRequestOutput` objects containing the
416
generated embeddings in the same order as the input prompts.
417
+
418
+ Note:
419
+ Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
420
+ considered legacy and may be deprecated in the future. You should
421
+ instead pass them via the ``inputs`` parameter.
422
"""
423
if prompt_token_ids is not None or multi_modal_data is not None:
424
inputs = self._convert_v1_inputs(