@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 33e0a2540a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[9/N] torch.compile LLM usage (#10552)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 21, 2024 at 19:13 UTC 33e0a2540a6bff23cbc6a4b8f7a6784a2bc87d47
2 files changed +16 -4
tests/tpu/test_compilation.py
+2 -3
@@ -4,7 +4,7 @@ import tempfile
4
5 import depyf
6
7 -from vllm.config import CompilationConfig, CompilationLevel
7 +from vllm.config import CompilationLevel
8
9 temp_dir = tempfile.mkdtemp()
10 with depyf.prepare_debug(temp_dir):
@@ -34,8 +34,7 @@ with depyf.prepare_debug(temp_dir):
34 # all the control
35 llm = LLM(model="google/gemma-2b",
36 enforce_eager=True,
37 - compilation_config=CompilationConfig(
38 - level=CompilationLevel.DYNAMO_AS_IS))
37 + compilation_config={"level": CompilationLevel.DYNAMO_AS_IS})
38 outputs = llm.generate(prompts, sampling_params)
39 for output, answer in zip(outputs, answers):
40 prompt = output.prompt
vllm/entrypoints/llm.py
+14 -1
@@ -1,4 +1,5 @@
1 import itertools
2 +import json
3 import warnings
4 from contextlib import contextmanager
5 from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
@@ -9,6 +10,7 @@ from tqdm import tqdm
10 from vllm import envs
11 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
12 BeamSearchSequence, get_beam_search_score)
13 +from vllm.config import CompilationConfig
14 from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
15 TaskOption)
16 from vllm.engine.llm_engine import LLMEngine
@@ -107,13 +109,16 @@ class LLM:
109 hf_overrides: If a dictionary, contains arguments to be forwarded to the
110 HuggingFace config. If a callable, it is called to update the
111 HuggingFace config.
112 + compilation_config: Either an integer or a dictionary. If it is an integer,
113 + it is used as the level of compilation optimization. If it is a dictionary,
114 + it can specify the full compilation configuration.
115 **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
116 :ref:`engine_args`)
117
118 Note:
119 This class is intended to be used for offline inference. For online
120 serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
116 - """
121 + """ # noqa
122
123 DEPRECATE_LEGACY: ClassVar[bool] = False
124 """A flag to toggle whether to deprecate the legacy generate/encode API."""
@@ -166,6 +171,7 @@ class LLM:
171 # After positional args are removed, move this right below `model`
172 task: TaskOption = "auto",
173 override_pooler_config: Optional[PoolerConfig] = None,
174 + compilation_config: Optional[Union[int, Dict[str, Any]]] = None,
175 **kwargs,
176 ) -> None:
177 '''
@@ -178,6 +184,12 @@ class LLM:
184 if "disable_log_stats" not in kwargs:
185 kwargs["disable_log_stats"] = True
186
187 + if compilation_config is not None:
188 + compilation_config_instance = CompilationConfig.from_cli(
189 + json.dumps(compilation_config))
190 + else:
191 + compilation_config_instance = None
192 +
193 engine_args = EngineArgs(
194 model=model,
195 task=task,
@@ -202,6 +214,7 @@ class LLM:
214 hf_overrides=hf_overrides,
215 mm_processor_kwargs=mm_processor_kwargs,
216 override_pooler_config=override_pooler_config,
217 + compilation_config=compilation_config_instance,
218 **kwargs,
219 )
220 # Logic to switch between engines is done at runtime instead of import