A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[9/N] torch.compile LLM usage (#10552)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 21, 2024 at 19:13 UTC
33e0a2540a6bff23cbc6a4b8f7a6784a2bc87d47
2 files changed
+16
-4
tests/tpu/test_compilation.py
+2
-3
@@ -4,7 +4,7 @@ import tempfile
4
5
import depyf
6
7
-from vllm.config import CompilationConfig, CompilationLevel
7
+from vllm.config import CompilationLevel
8
9
temp_dir = tempfile.mkdtemp()
10
with depyf.prepare_debug(temp_dir):
@@ -34,8 +34,7 @@ with depyf.prepare_debug(temp_dir):
34
# all the control
35
llm = LLM(model="google/gemma-2b",
36
enforce_eager=True,
37
- compilation_config=CompilationConfig(
38
- level=CompilationLevel.DYNAMO_AS_IS))
37
+ compilation_config={"level": CompilationLevel.DYNAMO_AS_IS})
38
outputs = llm.generate(prompts, sampling_params)
39
for output, answer in zip(outputs, answers):
40
prompt = output.prompt
vllm/entrypoints/llm.py
+14
-1
@@ -1,4 +1,5 @@
1
import itertools
2
+import json
3
import warnings
4
from contextlib import contextmanager
5
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
@@ -9,6 +10,7 @@ from tqdm import tqdm
10
from vllm import envs
11
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
12
BeamSearchSequence, get_beam_search_score)
13
+from vllm.config import CompilationConfig
14
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
15
TaskOption)
16
from vllm.engine.llm_engine import LLMEngine
@@ -107,13 +109,16 @@ class LLM:
109
hf_overrides: If a dictionary, contains arguments to be forwarded to the
110
HuggingFace config. If a callable, it is called to update the
111
HuggingFace config.
112
+ compilation_config: Either an integer or a dictionary. If it is an integer,
113
+ it is used as the level of compilation optimization. If it is a dictionary,
114
+ it can specify the full compilation configuration.
115
**kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
116
:ref:`engine_args`)
117
118
Note:
119
This class is intended to be used for offline inference. For online
120
serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
116
- """
121
+ """ # noqa
122
123
DEPRECATE_LEGACY: ClassVar[bool] = False
124
"""A flag to toggle whether to deprecate the legacy generate/encode API."""
@@ -166,6 +171,7 @@ class LLM:
171
# After positional args are removed, move this right below `model`
172
task: TaskOption = "auto",
173
override_pooler_config: Optional[PoolerConfig] = None,
174
+ compilation_config: Optional[Union[int, Dict[str, Any]]] = None,
175
**kwargs,
176
) -> None:
177
'''
@@ -178,6 +184,12 @@ class LLM:
184
if "disable_log_stats" not in kwargs:
185
kwargs["disable_log_stats"] = True
186
187
+ if compilation_config is not None:
188
+ compilation_config_instance = CompilationConfig.from_cli(
189
+ json.dumps(compilation_config))
190
+ else:
191
+ compilation_config_instance = None
192
+
193
engine_args = EngineArgs(
194
model=model,
195
task=task,
@@ -202,6 +214,7 @@ class LLM:
214
hf_overrides=hf_overrides,
215
mm_processor_kwargs=mm_processor_kwargs,
216
override_pooler_config=override_pooler_config,
217
+ compilation_config=compilation_config_instance,
218
**kwargs,
219
)
220
# Logic to switch between engines is done at runtime instead of import