A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[DOC] Add additional comments for LLMEngine and AsyncLLMEngine (#1011)
Jiaxiang committed
Jan 12, 2024 at 11:26 UTC
6549aef24501d5cd1a0d22a19e9e0f8e04f8bd77
9 files changed
+242
-15
docs/source/conf.py
+36
-7
@@ -9,11 +9,15 @@
9
# If extensions (or modules to document with autodoc) are in another directory,
10
# add these directories to sys.path here. If the directory is relative to the
11
# documentation root, use os.path.abspath to make it absolute, like shown here.
12
-#
13
-# import os
14
-# import sys
15
-# sys.path.insert(0, os.path.abspath('.'))
12
13
+import os
14
+import sys
15
+from sphinx.ext import autodoc
16
+import logging
17
+
18
+sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
19
+
20
+logger = logging.getLogger(__name__)
21
22
# -- Project information -----------------------------------------------------
23
@@ -21,7 +25,6 @@ project = 'vLLM'
25
copyright = '2023, vLLM Team'
26
author = 'the vLLM Team'
27
24
-
28
# -- General configuration ---------------------------------------------------
29
30
# Add any Sphinx extension module names here, as strings. They can be
@@ -32,6 +35,8 @@ extensions = [
35
"sphinx.ext.viewcode",
36
"sphinx.ext.intersphinx",
37
"sphinx_copybutton",
38
+ "sphinx.ext.autodoc",
39
+ "sphinx.ext.autosummary",
40
]
41
42
# Add any paths that contain templates here, relative to this directory.
@@ -55,7 +60,6 @@ html_title = project
60
html_theme = 'sphinx_book_theme'
61
html_logo = 'assets/logos/vllm-logo-text-light.png'
62
html_theme_options = {
58
- 'logo_only': True,
63
'path_to_docs': 'docs/source',
64
'repository_url': 'https://github.com/vllm-project/vllm',
65
'use_repository_button': True,
@@ -64,4 +68,29 @@ html_theme_options = {
68
# Add any paths that contain custom static files (such as style sheets) here,
69
# relative to this directory. They are copied after the builtin static files,
70
# so a file named "default.css" will overwrite the builtin "default.css".
67
-html_static_path = ['_static']
71
+# html_static_path = ['_static']
72
+
73
+# Mock out external dependencies here.
74
+autodoc_mock_imports = [
75
+ "torch", "transformers", "psutil", "aioprometheus", "sentencepiece",
76
+ "vllm.cuda_utils", "vllm._C"
77
+]
78
+
79
+for mock_target in autodoc_mock_imports:
80
+ if mock_target in sys.modules:
81
+ logger.info(
82
+ f"Potentially problematic mock target ({mock_target}) found; "
83
+ "autodoc_mock_imports cannot mock modules that have already "
84
+ "been loaded into sys.modules when the sphinx build starts.")
85
+
86
+
87
+class MockedClassDocumenter(autodoc.ClassDocumenter):
88
+ """Remove note about base class when a class is derived from object."""
89
+
90
+ def add_line(self, line: str, source: str, *lineno: int) -> None:
91
+ if line == " Bases: :py:class:`object`":
92
+ return
93
+ super().add_line(line, source, *lineno)
94
+
95
+
96
+autodoc.ClassDocumenter = MockedClassDocumenter
docs/source/dev/engine/async_llm_engine.rst
new
+7
@@ -0,0 +1,7 @@
1
+
2
+AsyncLLMEngine
3
+=================================
4
+
5
+.. autoclass:: vllm.engine.async_llm_engine.AsyncLLMEngine
6
+ :members: generate, abort
7
+ :show-inheritance:
docs/source/dev/engine/engine_index.rst
new
+13
@@ -0,0 +1,13 @@
1
+vLLM Engine
2
+=================================
3
+
4
+.. automodule:: vllm.engine
5
+.. currentmodule:: vllm.engine
6
+
7
+.. toctree::
8
+ :maxdepth: 2
9
+ :caption: Engines
10
+
11
+ llm_engine
12
+ async_llm_engine
13
+
docs/source/dev/engine/llm_engine.rst
new
+6
@@ -0,0 +1,6 @@
1
+LLMEngine
2
+=================================
3
+
4
+.. autoclass:: vllm.engine.llm_engine.LLMEngine
5
+ :members: add_request, abort_request, step, _init_cache
6
+ :show-inheritance:
\ No newline at end of file
docs/source/index.rst
+13
-1
@@ -85,4 +85,16 @@ Documentation
85
:maxdepth: 1
86
:caption: Quantization
87
88
- quantization/auto_awq
\ No newline at end of file
88
+ quantization/auto_awq
89
+
90
+.. toctree::
91
+ :maxdepth: 2
92
+ :caption: Developer Documentation
93
+
94
+ dev/engine/engine_index
95
+
96
+Indices and tables
97
+==================
98
+
99
+* :ref:`genindex`
100
+* :ref:`modindex`
vllm/core/scheduler.py
+12
@@ -88,6 +88,18 @@ class Scheduler:
88
self.waiting.append(seq_group)
89
90
def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
91
+ """Aborts a sequence group with the given ID.
92
+
93
+ Check if the sequence group with the given ID
94
+ is present in any of the state queue.
95
+ If present, remove the sequence group from the state queue.
96
+ Also, if any of the sequences in the sequence group is not finished,
97
+ free the sequence with status `FINISHED_ABORTED`.
98
+ Otherwise, do nothing.
99
+
100
+ Args:
101
+ request_id: The ID(s) of the sequence group to abort.
102
+ """
103
if isinstance(request_id, str):
104
request_id = (request_id, )
105
request_ids = set(request_id)
vllm/engine/async_llm_engine.py
+45
-1
@@ -253,7 +253,8 @@ class AsyncLLMEngine:
253
log_requests: Whether to log the requests.
254
start_engine_loop: If True, the background task to run the engine
255
will be automatically started in the generate call.
256
- *args, *kwargs: Arguments for LLMEngine.
256
+ *args: Arguments for LLMEngine.
257
+ *kwargs: Arguments for LLMEngine.
258
"""
259
260
_engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
@@ -428,6 +429,49 @@ class AsyncLLMEngine:
429
Yields:
430
The output `RequestOutput` objects from the LLMEngine for the
431
request.
432
+
433
+ Details:
434
+ - If the engine is not running, start the background loop,
435
+ which iteratively invokes
436
+ :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
437
+ to process the waiting requests.
438
+ - Add the request to the engine's `RequestTracker`.
439
+ On the next background loop, this request will be sent to
440
+ the underlying engine.
441
+ Also, a corresponding `AsyncStream` will be created.
442
+ - Wait for the request outputs from `AsyncStream` and yield them.
443
+
444
+ Example:
445
+ >>> # Please refer to entrypoints/api_server.py for
446
+ >>> # the complete example.
447
+ >>>
448
+ >>> # initialize the engine and the example input
449
+ >>> engine = AsyncLLMEngine.from_engine_args(engine_args)
450
+ >>> example_input = {
451
+ >>> "prompt": "What is LLM?",
452
+ >>> "stream": False, # assume the non-streaming case
453
+ >>> "temperature": 0.0,
454
+ >>> "request_id": 0,
455
+ >>> }
456
+ >>>
457
+ >>> # start the generation
458
+ >>> results_generator = engine.generate(
459
+ >>> example_input["prompt"],
460
+ >>> SamplingParams(temperature=example_input["temperature"]),
461
+ >>> example_input["request_id"])
462
+ >>>
463
+ >>> # get the results
464
+ >>> final_output = None
465
+ >>> async for request_output in results_generator:
466
+ >>> if await request.is_disconnected():
467
+ >>> # Abort the request if the client disconnects.
468
+ >>> await engine.abort(request_id)
469
+ >>> # Return or raise an error
470
+ >>> ...
471
+ >>> final_output = request_output
472
+ >>>
473
+ >>> # Process and return the final output
474
+ >>> ...
475
"""
476
# Preprocess the request.
477
# This should not be used for logging, as it is monotonic time.
vllm/engine/llm_engine.py
+102
-6
@@ -257,7 +257,26 @@ class LLMEngine:
257
self.cache_config.verify_with_parallel_config(self.parallel_config)
258
259
def _init_cache(self) -> None:
260
- """Profiles the memory usage and initializes the KV cache."""
260
+ """Profiles the memory usage and initializes the KV cache.
261
+
262
+ The engine will first conduct a profiling of the existing memory usage.
263
+ Then, it calculate the maximum possible number of GPU and CPU blocks
264
+ that can be allocated with the remaining free memory.
265
+ More details can be found in the
266
+ :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
267
+ from class :class:`~vllm.worker.Worker`.
268
+
269
+ Afterwards, as there may be multiple workers,
270
+ we take the minimum number of blocks across all workers
271
+ to ensure this can be applied to all of them.
272
+
273
+ Finally, the engine will initialize the KV cache
274
+ with the calculated number of blocks.
275
+
276
+ .. tip::
277
+ You may limit the usage of GPU memory
278
+ by adjusting the `gpu_memory_utilization` parameters.
279
+ """
280
# Get the maximum number of blocks that can be allocated on GPU and CPU.
281
num_blocks = self._run_workers(
282
"profile_num_available_blocks",
@@ -334,6 +353,30 @@ class LLMEngine:
353
use the tokenizer to convert the prompts to token IDs.
354
arrival_time: The arrival time of the request. If None, we use
355
the current monotonic time.
356
+
357
+ Details:
358
+ - Set arrival_time to the current time if it is None.
359
+ - Set prompt_token_ids to the encoded prompt if it is None.
360
+ - Create `best_of` number of :class:`~vllm.Sequence` objects.
361
+ - Create a :class:`~vllm.SequenceGroup` object
362
+ from the list of :class:`~vllm.Sequence`.
363
+ - Add the :class:`~vllm.SequenceGroup` object to the scheduler.
364
+
365
+ Example:
366
+ >>> # initialize engine
367
+ >>> engine = LLMEngine.from_engine_args(engine_args)
368
+ >>> # set request arguments
369
+ >>> example_prompt = "Who is the president of the United States?"
370
+ >>> sampling_params = SamplingParams(temperature=0.0)
371
+ >>> request_id = 0
372
+ >>>
373
+ >>> # add the request to the engine
374
+ >>> engine.add_request(
375
+ >>> str(request_id),
376
+ >>> example_prompt,
377
+ >>> SamplingParams(temperature=0.0))
378
+ >>> # continue the request processing
379
+ >>> ...
380
"""
381
if arrival_time is None:
382
arrival_time = time.monotonic()
@@ -358,6 +401,17 @@ class LLMEngine:
401
402
Args:
403
request_id: The ID(s) of the request to abort.
404
+
405
+ Details:
406
+ - Refer to the
407
+ :meth:`~vllm.core.scheduler.Scheduler.abort_seq_group`
408
+ from class :class:`~vllm.core.scheduler.Scheduler`.
409
+
410
+ Example:
411
+ >>> # initialize engine and add a request with request_id
412
+ >>> request_id = str(0)
413
+ >>> # abort the request
414
+ >>> engine.abort_request(request_id)
415
"""
416
self.scheduler.abort_seq_group(request_id)
417
@@ -617,11 +671,53 @@ class LLMEngine:
671
def step(self) -> List[RequestOutput]:
672
"""Performs one decoding iteration and returns newly generated results.
673
620
- This function performs one decoding iteration of the engine. It first
621
- schedules the sequences to be executed in the next iteration and the
622
- token blocks to be swapped in/out/copy. Then, it executes the model
623
- and updates the scheduler with the model outputs. Finally, it decodes
624
- the sequences and returns the newly generated results.
674
+ .. figure:: https://i.imgur.com/sv2HssD.png
675
+ :alt: Overview of the step function
676
+ :align: center
677
+
678
+ Overview of the step function.
679
+
680
+ Details:
681
+ - Step 1: Schedules the sequences to be executed in the next
682
+ iteration and the token blocks to be swapped in/out/copy.
683
+
684
+ - Depending on the scheduling policy,
685
+ sequences may be `preempted/reordered`.
686
+ - A Sequence Group (SG) refer to a group of sequences
687
+ that are generated from the same prompt.
688
+
689
+ - Step 2: Calls the workers to execute the model.
690
+ - Step 3: Processes the model output. This mainly includes:
691
+
692
+ - Decodes the relevant outputs.
693
+ - Updates the scheduled sequence groups with model outputs
694
+ based on its `sampling parameters` (`use_beam_search` or not).
695
+ - Frees the finished sequence groups.
696
+
697
+ - Finally, it creates and returns the newly generated results.
698
+
699
+ Example:
700
+ >>> # Please see the example/ folder for more detailed examples.
701
+ >>>
702
+ >>> # initialize engine and request arguments
703
+ >>> engine = LLMEngine.from_engine_args(engine_args)
704
+ >>> example_inputs = [(0, "What is LLM?",
705
+ >>> SamplingParams(temperature=0.0))]
706
+ >>>
707
+ >>> # Start the engine with an event loop
708
+ >>> while True:
709
+ >>> if example_inputs:
710
+ >>> req_id, prompt, sampling_params = example_inputs.pop(0)
711
+ >>> engine.add_request(str(req_id), prompt, sampling_params)
712
+ >>>
713
+ >>> # continue the request processing
714
+ >>> request_outputs = engine.step()
715
+ >>> for request_output in request_outputs:
716
+ >>> if request_output.finished:
717
+ >>> # return or show the request output
718
+ >>>
719
+ >>> if not (engine.has_unfinished_requests() or example_inputs):
720
+ >>> break
721
"""
722
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
723
vllm/worker/worker.py
+8
@@ -87,6 +87,14 @@ class Worker:
87
gpu_memory_utilization: float,
88
cpu_swap_space: int,
89
) -> Tuple[int, int]:
90
+ """Profiles the peak memory usage of the model and returns the maximum
91
+ number of GPU and CPU cache blocks that can be allocated.
92
+
93
+ Args:
94
+ block_size: The size of the cache block.
95
+ gpu_memory_utilization: The fraction of the total GPU memory to use.
96
+ cpu_swap_space: The size of the CPU swap space in bytes.
97
+ """
98
# Profile the memory usage of the model and get the maximum number of
99
# cache blocks that can be allocated with the remaining free memory.
100
torch.cuda.empty_cache()