A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] allow candidate compile sizes (#10984)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 8, 2024 at 03:05 UTC
fd57d2b5347e8fe6da9287553d4b5a3aaf2e6693
4 files changed
+28
-35
tests/engine/test_arg_utils.py
+4
-4
@@ -50,12 +50,12 @@ def test_compilation_config():
50
args = parser.parse_args(["-O=3"])
51
assert args.compilation_config.level == 3
52
53
- # set to json
54
- args = parser.parse_args(["--compilation-config", '{"level": 3}'])
53
+ # set to string form of a dict
54
+ args = parser.parse_args(["--compilation-config", "{'level': 3}"])
55
assert args.compilation_config.level == 3
56
57
- # set to json
58
- args = parser.parse_args(['--compilation-config={"level": 3}'])
57
+ # set to string form of a dict
58
+ args = parser.parse_args(["--compilation-config={'level': 3}"])
59
assert args.compilation_config.level == 3
60
61
vllm/config.py
+22
-22
@@ -1,3 +1,4 @@
1
+import ast
2
import copy
3
import enum
4
import hashlib
@@ -2191,14 +2192,10 @@ class CompilationConfig(BaseModel):
2192
- use_inductor: whether to use inductor compilation.
2193
- False: inductor compilation is not used. graph runs in eager.
2194
- True: inductor compilation is used. one graph for symbolic shape
2194
- is compiled. In addition, compile for different sizes specified
2195
- in inductor_compile_sizes, using configurations
2195
+ is compiled. In addition, compile for cudagraph sizes that are
2196
+ in candidate_compile_sizes, using configurations
2197
in inductor_compile_config.
2197
- - inductor_compile_sizes: sizes to compile for inductor.
2198
- - inductor_specialize_for_cudagraph_no_more_than: an optional integer
2199
- to specialize inductor for cudagraph sizes no more than the
2200
- specified size. It is useful when we want to specialize inductor
2201
- with a subset of cudagraph sizes.
2198
+ - candidate_compile_sizes: sizes to compile for inductor.
2199
- inductor_compile_config: additional configurations for inductor.
2200
- None: use default configurations.
2201
- inductor_passes: additional passes for inductor. It is a dictionary
@@ -2227,8 +2224,7 @@ class CompilationConfig(BaseModel):
2224
])
2225
2226
use_inductor: bool = True
2230
- inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
2231
- inductor_compile_sizes: Optional[List[int]] = Field(default=None)
2227
+ candidate_compile_sizes: Optional[List[int]] = Field(default=None)
2228
inductor_compile_config: Dict = Field(default_factory=dict)
2229
inductor_passes: Dict[str, str] = Field(default_factory=dict)
2230
@@ -2294,7 +2290,9 @@ class CompilationConfig(BaseModel):
2290
"""Parse the CLI value for the compilation config."""
2291
if cli_value in ["0", "1", "2", "3"]:
2292
return cls(level=int(cli_value))
2297
- return CompilationConfig.model_validate_json(cli_value)
2293
+ # do not use `eval`, it is dangerous and can execute arbitrary code
2294
+ dict_value = ast.literal_eval(cli_value)
2295
+ return CompilationConfig.model_validate(dict_value)
2296
2297
def model_post_init(self, __context: Any) -> None:
2298
@@ -2355,18 +2353,20 @@ class CompilationConfig(BaseModel):
2353
logger.info(("cudagraph sizes specified by model runner"
2354
" %s is overridden by config %s"),
2355
sizes_to_specialize, self.cudagraph_capture_sizes)
2358
- if self.inductor_specialize_for_cudagraph_no_more_than is not None:
2359
- assert self.inductor_compile_sizes is None, (
2360
- "inductor_compile_sizes should be None when "
2361
- "inductor_specialize_for_cudagraph_no_more_than is not None")
2362
- self.compile_sizes = [
2363
- x for x in self.capture_sizes
2364
- if x <= self.inductor_specialize_for_cudagraph_no_more_than
2365
- ]
2366
- else:
2367
- if self.inductor_compile_sizes is None:
2368
- self.inductor_compile_sizes = []
2369
- self.compile_sizes = self.inductor_compile_sizes
2356
+
2357
+ if self.candidate_compile_sizes is None:
2358
+ self.candidate_compile_sizes = []
2359
+ self.compile_sizes = [
2360
+ x for x in self.candidate_compile_sizes if x in self.capture_sizes
2361
+ ]
2362
+ ignored_sizes = [
2363
+ x for x in self.candidate_compile_sizes
2364
+ if x not in self.capture_sizes
2365
+ ]
2366
+ if ignored_sizes:
2367
+ logger.warning(("candidate_compile_sizes %s are ignored "
2368
+ "because they are not cudagraph capture sizes."),
2369
+ ignored_sizes)
2370
2371
# sort to make sure cudagraph capture sizes are in descending order
2372
self.capture_sizes.sort(reverse=True)
vllm/engine/arg_utils.py
+1
-4
@@ -209,12 +209,9 @@ class EngineArgs:
209
# support `EngineArgs(compilation_config={...})`
210
# without having to manually construct a
211
# CompilationConfig object
212
- if isinstance(self.compilation_config, (int)):
212
+ if isinstance(self.compilation_config, (int, dict)):
213
self.compilation_config = CompilationConfig.from_cli(
214
str(self.compilation_config))
215
- elif isinstance(self.compilation_config, (dict)):
216
- self.compilation_config = CompilationConfig.from_cli(
217
- json.dumps(self.compilation_config))
215
216
# Setup plugins
217
from vllm.plugins import load_general_plugins
vllm/entrypoints/llm.py
+1
-5
@@ -1,5 +1,4 @@
1
import itertools
2
-import json
2
import warnings
3
from contextlib import contextmanager
4
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
@@ -186,12 +185,9 @@ class LLM:
185
kwargs["disable_log_stats"] = True
186
187
if compilation_config is not None:
189
- if isinstance(compilation_config, (int)):
188
+ if isinstance(compilation_config, (int, dict)):
189
compilation_config_instance = CompilationConfig.from_cli(
190
str(compilation_config))
192
- elif isinstance(compilation_config, (dict)):
193
- compilation_config_instance = CompilationConfig.from_cli(
194
- json.dumps(compilation_config))
191
else:
192
compilation_config_instance = compilation_config
193
else: