@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fd57d2b534
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] allow candidate compile sizes (#10984)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 8, 2024 at 03:05 UTC fd57d2b5347e8fe6da9287553d4b5a3aaf2e6693
4 files changed +28 -35
tests/engine/test_arg_utils.py
+4 -4
@@ -50,12 +50,12 @@ def test_compilation_config():
50 args = parser.parse_args(["-O=3"])
51 assert args.compilation_config.level == 3
52
53 - # set to json
54 - args = parser.parse_args(["--compilation-config", '{"level": 3}'])
53 + # set to string form of a dict
54 + args = parser.parse_args(["--compilation-config", "{'level': 3}"])
55 assert args.compilation_config.level == 3
56
57 - # set to json
58 - args = parser.parse_args(['--compilation-config={"level": 3}'])
57 + # set to string form of a dict
58 + args = parser.parse_args(["--compilation-config={'level': 3}"])
59 assert args.compilation_config.level == 3
60
61
vllm/config.py
+22 -22
@@ -1,3 +1,4 @@
1 +import ast
2 import copy
3 import enum
4 import hashlib
@@ -2191,14 +2192,10 @@ class CompilationConfig(BaseModel):
2192 - use_inductor: whether to use inductor compilation.
2193 - False: inductor compilation is not used. graph runs in eager.
2194 - True: inductor compilation is used. one graph for symbolic shape
2194 - is compiled. In addition, compile for different sizes specified
2195 - in inductor_compile_sizes, using configurations
2195 + is compiled. In addition, compile for cudagraph sizes that are
2196 + in candidate_compile_sizes, using configurations
2197 in inductor_compile_config.
2197 - - inductor_compile_sizes: sizes to compile for inductor.
2198 - - inductor_specialize_for_cudagraph_no_more_than: an optional integer
2199 - to specialize inductor for cudagraph sizes no more than the
2200 - specified size. It is useful when we want to specialize inductor
2201 - with a subset of cudagraph sizes.
2198 + - candidate_compile_sizes: sizes to compile for inductor.
2199 - inductor_compile_config: additional configurations for inductor.
2200 - None: use default configurations.
2201 - inductor_passes: additional passes for inductor. It is a dictionary
@@ -2227,8 +2224,7 @@ class CompilationConfig(BaseModel):
2224 ])
2225
2226 use_inductor: bool = True
2230 - inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
2231 - inductor_compile_sizes: Optional[List[int]] = Field(default=None)
2227 + candidate_compile_sizes: Optional[List[int]] = Field(default=None)
2228 inductor_compile_config: Dict = Field(default_factory=dict)
2229 inductor_passes: Dict[str, str] = Field(default_factory=dict)
2230
@@ -2294,7 +2290,9 @@ class CompilationConfig(BaseModel):
2290 """Parse the CLI value for the compilation config."""
2291 if cli_value in ["0", "1", "2", "3"]:
2292 return cls(level=int(cli_value))
2297 - return CompilationConfig.model_validate_json(cli_value)
2293 + # do not use `eval`, it is dangerous and can execute arbitrary code
2294 + dict_value = ast.literal_eval(cli_value)
2295 + return CompilationConfig.model_validate(dict_value)
2296
2297 def model_post_init(self, __context: Any) -> None:
2298
@@ -2355,18 +2353,20 @@ class CompilationConfig(BaseModel):
2353 logger.info(("cudagraph sizes specified by model runner"
2354 " %s is overridden by config %s"),
2355 sizes_to_specialize, self.cudagraph_capture_sizes)
2358 - if self.inductor_specialize_for_cudagraph_no_more_than is not None:
2359 - assert self.inductor_compile_sizes is None, (
2360 - "inductor_compile_sizes should be None when "
2361 - "inductor_specialize_for_cudagraph_no_more_than is not None")
2362 - self.compile_sizes = [
2363 - x for x in self.capture_sizes
2364 - if x <= self.inductor_specialize_for_cudagraph_no_more_than
2365 - ]
2366 - else:
2367 - if self.inductor_compile_sizes is None:
2368 - self.inductor_compile_sizes = []
2369 - self.compile_sizes = self.inductor_compile_sizes
2356 +
2357 + if self.candidate_compile_sizes is None:
2358 + self.candidate_compile_sizes = []
2359 + self.compile_sizes = [
2360 + x for x in self.candidate_compile_sizes if x in self.capture_sizes
2361 + ]
2362 + ignored_sizes = [
2363 + x for x in self.candidate_compile_sizes
2364 + if x not in self.capture_sizes
2365 + ]
2366 + if ignored_sizes:
2367 + logger.warning(("candidate_compile_sizes %s are ignored "
2368 + "because they are not cudagraph capture sizes."),
2369 + ignored_sizes)
2370
2371 # sort to make sure cudagraph capture sizes are in descending order
2372 self.capture_sizes.sort(reverse=True)
vllm/engine/arg_utils.py
+1 -4
@@ -209,12 +209,9 @@ class EngineArgs:
209 # support `EngineArgs(compilation_config={...})`
210 # without having to manually construct a
211 # CompilationConfig object
212 - if isinstance(self.compilation_config, (int)):
212 + if isinstance(self.compilation_config, (int, dict)):
213 self.compilation_config = CompilationConfig.from_cli(
214 str(self.compilation_config))
215 - elif isinstance(self.compilation_config, (dict)):
216 - self.compilation_config = CompilationConfig.from_cli(
217 - json.dumps(self.compilation_config))
215
216 # Setup plugins
217 from vllm.plugins import load_general_plugins
vllm/entrypoints/llm.py
+1 -5
@@ -1,5 +1,4 @@
1 import itertools
2 -import json
2 import warnings
3 from contextlib import contextmanager
4 from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
@@ -186,12 +185,9 @@ class LLM:
185 kwargs["disable_log_stats"] = True
186
187 if compilation_config is not None:
189 - if isinstance(compilation_config, (int)):
188 + if isinstance(compilation_config, (int, dict)):
189 compilation_config_instance = CompilationConfig.from_cli(
190 str(compilation_config))
192 - elif isinstance(compilation_config, (dict)):
193 - compilation_config_instance = CompilationConfig.from_cli(
194 - json.dumps(compilation_config))
191 else:
192 compilation_config_instance = compilation_config
193 else: