@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 803f37eaaa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[6/N] torch.compile rollout to users (#10437)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 19, 2024 at 10:09 UTC 803f37eaaa11568f65acbf0bcd1044fb9b1610bf
15 files changed +129 -141
tests/compile/piecewise/piecewise_compilation_config.json deleted
-5
@@ -1,5 +0,0 @@
1 -{
2 - "use_cudagraph": true,
3 - "non_cudagraph_ops": ["silly.attention"],
4 - "cudagraph_copy_inputs": true
5 -}
\ No newline at end of file
tests/compile/piecewise/test_simple.py
+7 -11
@@ -2,7 +2,6 @@
2 Test the piecewise compilation with a simple model so that we
3 can exactly calculate the expected output and side effects.
4 """
5 -import os
5
6 import torch
7 from torch import nn
@@ -11,7 +10,7 @@ from torch.library import Library
10 from vllm.compilation.compile_context import set_compile_context
11 from vllm.compilation.counter import compilation_counter
12 from vllm.compilation.decorators import support_torch_compile
14 -from vllm.config import CompilationLevel, VllmConfig
13 +from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
14 from vllm.plugins import set_current_vllm_config
15 from vllm.utils import direct_register_custom_op
16
@@ -77,12 +76,12 @@ class SillyModel(nn.Module):
76
77 def test_simple_piecewise_compile():
78
80 - directory = os.path.dirname(__file__)
81 - config = os.path.join(directory, "piecewise_compilation_config.json")
82 - os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
83 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
84 -
85 - vllm_config = VllmConfig()
79 + vllm_config = VllmConfig(compilation_config=CompilationConfig(
80 + level=CompilationLevel.PIECEWISE,
81 + use_cudagraph=True,
82 + non_cudagraph_ops=["silly.attention"],
83 + cudagraph_copy_inputs=True,
84 + ))
85 with set_current_vllm_config(vllm_config):
86 model = SillyModel(vllm_config=vllm_config, prefix='')
87
@@ -109,6 +108,3 @@ def test_simple_piecewise_compile():
108 output = model(input)
109 assert global_counter == 2
110 assert torch.allclose(output.cpu(), torch.tensor([3., 1.]))
112 -
113 - # clean up to avoid side effects for other tests
114 - del os.environ["VLLM_TORCH_COMPILE_CONFIG"]
tests/compile/piecewise/test_toy_llama.py
+17 -28
@@ -6,7 +6,6 @@ This is a tractable model, the weights and computation are specially designed
6 if the config `tractable_init` is set to True. Otherwise, the weights are
7 initialized randomly with a fixed seed.
8 """
9 -import os
9 from dataclasses import dataclass
10 from typing import Optional, Tuple
11
@@ -18,7 +17,7 @@ from vllm.compilation.compile_context import set_compile_context
17 from vllm.compilation.counter import compilation_counter
18 from vllm.compilation.decorators import support_torch_compile
19 from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
21 -from vllm.plugins import set_compilation_config, set_current_vllm_config
20 +from vllm.plugins import set_current_vllm_config
21 from vllm.utils import direct_register_custom_op
22
23 # create a library to hold the custom op
@@ -254,23 +253,17 @@ def run_model(llama_config,
253 split_attn: bool = False) -> torch.Tensor:
254
255 if use_compile:
257 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
258 - CompilationLevel.PIECEWISE)
259 -
256 + compilation_config = CompilationConfig(
257 + level=CompilationLevel.PIECEWISE,
258 + use_cudagraph=True,
259 + )
260 if split_attn:
261 - set_compilation_config(
262 - CompilationConfig(
263 - use_cudagraph=True,
264 - non_cudagraph_ops=["silly.attention"],
265 - ))
266 - else:
267 - set_compilation_config(CompilationConfig(use_cudagraph=True, ))
261 + compilation_config.non_cudagraph_ops = ["silly.attention"]
262 else:
269 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
270 - CompilationLevel.NO_COMPILATION)
271 - set_compilation_config(None)
263 + compilation_config = CompilationConfig(
264 + level=CompilationLevel.NO_COMPILATION, )
265
273 - vllm_config = VllmConfig()
266 + vllm_config = VllmConfig(compilation_config=compilation_config)
267 with set_current_vllm_config(vllm_config):
268 model = LlamaModel(config=llama_config,
269 vllm_config=vllm_config,
@@ -288,10 +281,6 @@ def run_model(llama_config,
281 input_ids[:2].zero_()
282 output = model(input_ids[:2], positions[:2])
283
291 - # manual cleanup
292 - del os.environ["VLLM_TORCH_COMPILE_LEVEL"]
293 - set_compilation_config(None)
294 -
284 output = output.cpu()
285
286 if llama_config.tractable_init:
@@ -361,7 +350,6 @@ def test_toy_llama():
350
351 @torch.inference_mode
352 def benchmark():
364 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
353 from triton.testing import do_bench
354
355 # similar to llama 3.1-8B
@@ -387,15 +375,16 @@ def benchmark():
375
376 for piecewise in [False, True]:
377 if piecewise:
390 - set_compilation_config(
391 - CompilationConfig(
392 - use_cudagraph=True,
393 - non_cudagraph_ops=["silly.attention"],
394 - ))
378 + compilation_config = CompilationConfig(
379 + level=CompilationLevel.PIECEWISE,
380 + use_cudagraph=True,
381 + non_cudagraph_ops=["silly.attention"],
382 + )
383 else:
396 - set_compilation_config(None)
384 + compilation_config = CompilationConfig(
385 + level=CompilationLevel.PIECEWISE, )
386
398 - vllm_config = VllmConfig()
387 + vllm_config = VllmConfig(compilation_config=compilation_config)
388 with set_current_vllm_config(vllm_config):
389 model = LlamaModel(config=llama_config,
390 vllm_config=vllm_config,
tests/compile/test_basic_correctness.py
+9 -4
@@ -96,31 +96,36 @@ def test_compile_correctness(test_setting: TestSetting):
96 final_args = ["--enforce-eager"] + model_args + ["-pp", str(pp_size)] + \
97 ["-tp", str(tp_size)]
98
99 + all_args: List[List[str]] = []
100 all_envs: List[Optional[Dict[str, str]]] = []
101
102 for level in [
103 CompilationLevel.NO_COMPILATION,
104 CompilationLevel.PIECEWISE,
105 ]:
105 - all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
106 + all_args.append(final_args + ["-O", str(level)])
107 + all_envs.append({})
108
109 # inductor will change the output, so we only compare if the output
110 # is close, not exactly the same.
111 compare_all_settings(
110 - model, [final_args] * 2,
112 + model,
113 + all_args,
114 all_envs,
115 method=method if method != "generate" else "generate_close")
116 all_envs.clear()
117 + all_args.clear()
118
119 for level in [
120 CompilationLevel.NO_COMPILATION,
121 CompilationLevel.DYNAMO_AS_IS,
122 CompilationLevel.DYNAMO_ONCE,
123 ]:
120 - all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
124 + all_args.append(final_args + ["-O", str(level)])
125 + all_envs.append({})
126 if level != CompilationLevel.DYNAMO_ONCE and not fullgraph:
127 # "DYNAMO_ONCE" will always use fullgraph
128 all_envs[-1][
129 "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0" # type: ignore
130
126 - compare_all_settings(model, [final_args] * 3, all_envs, method=method)
131 + compare_all_settings(model, all_args * 3, all_envs, method=method)
tests/compile/utils.py
+2 -2
@@ -4,7 +4,7 @@ import torch
4
5 from tests.quantization.utils import is_quant_method_supported
6 from vllm import LLM, SamplingParams
7 -from vllm.config import CompilationLevel
7 +from vllm.config import CompilationConfig, CompilationLevel
8 from vllm.platforms import current_platform
9
10 TEST_MODELS = [
@@ -65,7 +65,6 @@ def check_full_graph_support(model,
65 optimization_level,
66 tp_size=1):
67 # make sure these models can be captured in full graph mode
68 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
68 os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
69
70 # The base meta llama uses too much memory.
@@ -86,6 +85,7 @@ def check_full_graph_support(model,
85 enforce_eager=True,
86 tensor_parallel_size=tp_size,
87 disable_custom_all_reduce=True,
88 + compilation_config=CompilationConfig(level=optimization_level),
89 **model_kwargs)
90
91 outputs = llm.generate(prompts, sampling_params)
tests/model_executor/test_enabled_custom_ops.py
+1 -3
@@ -1,4 +1,3 @@
1 -import os
1 from typing import List
2
3 import pytest
@@ -53,9 +52,8 @@ class Relu3(ReLUSquaredActivation):
52 ])
53 def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],
54 default_on: bool):
56 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(torch_level)
55 vllm_config = VllmConfig(compilation_config=CompilationConfig(
58 - custom_ops=env.split(",")))
56 + level=torch_level, custom_ops=env.split(",")))
57 with set_current_vllm_config(vllm_config):
58 assert CustomOp.default_on() == default_on
59
tests/tpu/test_compilation.py
+35 -12
@@ -1,24 +1,47 @@
1 import glob
2 import os
3 -import runpy
3 import tempfile
4
5 import depyf
6
8 -from vllm.config import CompilationLevel
9 -
10 -# disable custom dispatcher, let Dynamo takes over
11 -# all the control
12 -os.environ['VLLM_TORCH_COMPILE_LEVEL'] = str(CompilationLevel.DYNAMO_AS_IS)
7 +from vllm.config import CompilationConfig, CompilationLevel
8
9 temp_dir = tempfile.mkdtemp()
10 with depyf.prepare_debug(temp_dir):
16 - cur_dir = os.path.dirname(__file__)
17 - parent_dir = os.path.dirname(cur_dir)
18 - root_dir = os.path.dirname(parent_dir)
19 - example_file = os.path.join(root_dir, "examples",
20 - "offline_inference_tpu.py")
21 - runpy.run_path(example_file)
11 + from vllm import LLM, SamplingParams
12 +
13 + prompts = [
14 + "A robot may not injure a human being",
15 + "It is only with the heart that one can see rightly;",
16 + "The greatest glory in living lies not in never falling,",
17 + ]
18 + answers = [
19 + " or, through inaction, allow a human being to come to harm.",
20 + " what is essential is invisible to the eye.",
21 + " but in rising every time we fall.",
22 + ]
23 + N = 1
24 + # Currently, top-p sampling is disabled. `top_p` should be 1.0.
25 + sampling_params = SamplingParams(temperature=0.7,
26 + top_p=1.0,
27 + n=N,
28 + max_tokens=16)
29 +
30 + # Set `enforce_eager=True` to avoid ahead-of-time compilation.
31 + # In real workloads, `enforace_eager` should be `False`.
32 +
33 + # disable custom dispatcher, let Dynamo takes over
34 + # all the control
35 + llm = LLM(model="google/gemma-2b",
36 + enforce_eager=True,
37 + compilation_config=CompilationConfig(
38 + level=CompilationLevel.DYNAMO_AS_IS))
39 + outputs = llm.generate(prompts, sampling_params)
40 + for output, answer in zip(outputs, answers):
41 + prompt = output.prompt
42 + generated_text = output.outputs[0].text
43 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
44 + assert generated_text.startswith(answer)
45
46 compiled_code = sorted(
47 glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
tests/tpu/test_custom_dispatcher.py
+6 -4
@@ -13,7 +13,9 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
13 def test_custom_dispatcher():
14 compare_two_settings(
15 "google/gemma-2b",
16 - arg1=["--enforce-eager"],
17 - arg2=["--enforce-eager"],
18 - env1={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_ONCE)},
19 - env2={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_AS_IS)})
16 + arg1=["--enforce-eager", "-O",
17 + str(CompilationLevel.DYNAMO_ONCE)],
18 + arg2=["--enforce-eager", "-O",
19 + str(CompilationLevel.DYNAMO_AS_IS)],
20 + env1={},
21 + env2={})
vllm/config.py
+20 -23
@@ -2174,8 +2174,14 @@ class CompilationConfig(BaseModel):
2174 enabled_custom_ops: Counter[str] = PrivateAttr
2175 disabled_custom_ops: Counter[str] = PrivateAttr
2176
2177 + @classmethod
2178 + def from_cli(cls, cli_value: str) -> "CompilationConfig":
2179 + """Parse the CLI value for the compilation config."""
2180 + if cli_value in ["0", "1", "2", "3"]:
2181 + return cls(level=int(cli_value))
2182 + return CompilationConfig.model_validate_json(cli_value)
2183 +
2184 def model_post_init(self, __context: Any) -> None:
2178 - self.level = envs.VLLM_TORCH_COMPILE_LEVEL
2185
2186 count_none = self.custom_ops.count("none")
2187 count_all = self.custom_ops.count("all")
@@ -2249,26 +2255,6 @@ class CompilationConfig(BaseModel):
2255 "inductor_specialize_for_cudagraph_no_more_than is None")
2256 self.compile_sizes = self.inductor_compile_sizes
2257
2252 - @staticmethod
2253 - def select_and_init_config() -> "CompilationConfig":
2254 - """The order of selecting config is:
2255 - 1. Use the config specified in environment variable.
2256 - 2. Use the config specified in plugins.
2257 - 3. Use the default config.
2258 - """
2259 - config_path = envs.VLLM_TORCH_COMPILE_CONFIG
2260 - if config_path is not None:
2261 - with open(config_path) as json_file:
2262 - config = CompilationConfig.model_validate_json(
2263 - json_file.read())
2264 - else:
2265 - from vllm.plugins import get_compilation_config
2266 - predefined_config = get_compilation_config()
2267 - config = predefined_config if predefined_config is not None else (
2268 - CompilationConfig())
2269 -
2270 - return config
2271 -
2258
2259 @dataclass
2260 class VllmConfig:
@@ -2354,8 +2340,19 @@ class VllmConfig:
2340 self.model_config, self.load_config)
2341
2342 if self.compilation_config is None:
2357 - self.compilation_config = CompilationConfig.select_and_init_config(
2358 - )
2343 + self.compilation_config = CompilationConfig()
2344 + if envs.VLLM_USE_V1:
2345 + # NOTE(woosuk): Currently, we use inductor because the piecewise
2346 + # CUDA graphs do not work properly with the custom CUDA kernels.
2347 + # FIXME(woosuk): Disable inductor to reduce the compilation time
2348 + # and avoid any potential issues with the inductor.
2349 + self.compilation_config.custom_ops = ["none"]
2350 + self.compilation_config.use_cudagraph = True
2351 + self.compilation_config.non_cudagraph_ops = [
2352 + "vllm.unified_v1_flash_attention"
2353 + ]
2354 + self.compilation_config.use_inductor = True
2355 + self.compilation_config.enable_fusion = False
2356
2357 current_platform.check_and_update_config(self)
2358
vllm/engine/arg_utils.py
+23 -6
@@ -8,12 +8,13 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
8 import torch
9
10 import vllm.envs as envs
11 -from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12 - DeviceConfig, HfOverrides, LoadConfig, LoadFormat,
13 - LoRAConfig, ModelConfig, ObservabilityConfig,
14 - ParallelConfig, PoolerConfig, PromptAdapterConfig,
15 - SchedulerConfig, SpeculativeConfig, TaskOption,
16 - TokenizerPoolConfig, VllmConfig)
11 +from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
12 + DecodingConfig, DeviceConfig, HfOverrides, LoadConfig,
13 + LoadFormat, LoRAConfig, ModelConfig,
14 + ObservabilityConfig, ParallelConfig, PoolerConfig,
15 + PromptAdapterConfig, SchedulerConfig,
16 + SpeculativeConfig, TaskOption, TokenizerPoolConfig,
17 + VllmConfig)
18 from vllm.executor.executor_base import ExecutorBase
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -189,6 +190,7 @@ class EngineArgs:
190
191 override_neuron_config: Optional[Dict[str, Any]] = None
192 override_pooler_config: Optional[PoolerConfig] = None
193 + compilation_config: Optional[CompilationConfig] = None
194
195 def __post_init__(self):
196 if not self.tokenizer:
@@ -868,6 +870,20 @@ class EngineArgs:
870 help="Override or set the pooling method in the embedding model. "
871 "e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
872
873 + parser.add_argument('--compilation-config',
874 + '-O',
875 + type=CompilationConfig.from_cli,
876 + default=None,
877 + help='torch.compile configuration for the model.'
878 + 'When it is a number (0, 1, 2, 3), it will be '
879 + 'interpreted as the optimization level.\n'
880 + 'NOTE: level 0 is the default level without '
881 + 'any optimization. level 1 and 2 are for internal '
882 + 'testing only. level 3 is the recommended level '
883 + 'for production.\n'
884 + 'To specify the full compilation config, '
885 + 'use a JSON string.')
886 +
887 return parser
888
889 @classmethod
@@ -1142,6 +1158,7 @@ class EngineArgs:
1158 decoding_config=decoding_config,
1159 observability_config=observability_config,
1160 prompt_adapter_config=prompt_adapter_config,
1161 + compilation_config=self.compilation_config,
1162 )
1163
1164
vllm/engine/llm_engine.py
+3 -1
@@ -262,7 +262,8 @@ class LLMEngine:
262 "num_scheduler_steps=%d, chunked_prefill_enabled=%s "
263 "multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
264 "use_async_output_proc=%s, use_cached_outputs=%s, "
265 - "mm_processor_kwargs=%s, pooler_config=%r)",
265 + "mm_processor_kwargs=%s, pooler_config=%r,"
266 + "compilation_config=%r",
267 VLLM_VERSION,
268 model_config.model,
269 speculative_config,
@@ -297,6 +298,7 @@ class LLMEngine:
298 use_cached_outputs,
299 model_config.mm_processor_kwargs,
300 model_config.pooler_config,
301 + vllm_config.compilation_config,
302 )
303 # TODO(woosuk): Print more configs in debug mode.
304 self.model_config = model_config
vllm/envs.py
-8
@@ -67,8 +67,6 @@ if TYPE_CHECKING:
67 VLLM_USE_TRITON_AWQ: bool = False
68 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
69 VLLM_SKIP_P2P_CHECK: bool = False
70 - VLLM_TORCH_COMPILE_LEVEL: int = 0
71 - VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
70 VLLM_DISABLED_KERNELS: List[str] = []
71 VLLM_USE_V1: bool = False
72 VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
@@ -209,12 +207,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
207 "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
208 lambda: bool(
209 os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
212 - "VLLM_TORCH_COMPILE_LEVEL":
213 - lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
214 -
215 - # Path to the config file for torch compile
216 - "VLLM_TORCH_COMPILE_CONFIG":
217 - lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
210
211 # local rank of the process in the distributed setting, used to determine
212 # the GPU device id
vllm/platforms/tpu.py
+2 -2
@@ -1,4 +1,3 @@
1 -import os
1 from typing import TYPE_CHECKING
2
3 import torch
@@ -40,7 +39,8 @@ class TpuPlatform(Platform):
39 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
40 from vllm.config import CompilationLevel
41 compilation_config = vllm_config.compilation_config
43 - if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
42 + if compilation_config.level == CompilationLevel.NO_COMPILATION:
43 + # TPU does not support NO_COMPILATION
44 compilation_config.level = CompilationLevel.DYNAMO_ONCE
45 assert compilation_config.level < CompilationLevel.PIECEWISE,\
46 "TPU does not support Inductor."
vllm/plugins/__init__.py
+1 -13
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional
5 import vllm.envs as envs
6
7 if TYPE_CHECKING:
8 - from vllm.config import CompilationConfig, VllmConfig
8 + from vllm.config import VllmConfig
9
10 logger = logging.getLogger(__name__)
11
@@ -54,18 +54,6 @@ def load_general_plugins():
54 logger.exception("Failed to load plugin %s", plugin.name)
55
56
57 -_compilation_config: Optional["CompilationConfig"] = None
58 -
59 -
60 -def set_compilation_config(config: Optional["CompilationConfig"]):
61 - global _compilation_config
62 - _compilation_config = config
63 -
64 -
65 -def get_compilation_config() -> Optional["CompilationConfig"]:
66 - return _compilation_config
67 -
68 -
57 _current_vllm_config: Optional["VllmConfig"] = None
58
59
vllm/v1/worker/gpu_model_runner.py
+3 -19
@@ -8,13 +8,12 @@ import torch.distributed
8 import torch.nn as nn
9
10 from vllm.compilation.compile_context import set_compile_context
11 -from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
11 +from vllm.config import CompilationLevel, VllmConfig
12 from vllm.forward_context import set_forward_context
13 from vllm.inputs import INPUT_REGISTRY, InputRegistry
14 from vllm.logger import init_logger
15 from vllm.model_executor.model_loader import get_model
16 from vllm.multimodal import MultiModalKwargs
17 -from vllm.plugins import set_compilation_config
17 from vllm.sampling_params import SamplingParams, SamplingType
18 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19 is_pin_memory_available)
@@ -508,20 +507,6 @@ class GPUModelRunner:
507 return model_runner_output
508
509 def load_model(self) -> None:
511 - if self.use_cuda_graph:
512 - # NOTE(woosuk): Currently, we use inductor because the piecewise
513 - # CUDA graphs do not work properly with the custom CUDA kernels.
514 - # FIXME(woosuk): Disable inductor to reduce the compilation time
515 - # and avoid any potential issues with the inductor.
516 - set_compilation_config(
517 - CompilationConfig(
518 - custom_ops=["none"],
519 - use_cudagraph=True,
520 - non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
521 - use_inductor=True,
522 - enable_fusion=False,
523 - ))
524 -
510 logger.info("Starting to load model %s...", self.model_config.model)
511 with DeviceMemoryProfiler() as m: # noqa: SIM117
512 self.model = get_model(vllm_config=self.vllm_config)
@@ -562,9 +547,8 @@ class GPUModelRunner:
547 def capture_model(self) -> None:
548 if not self.use_cuda_graph:
549 logger.warning(
565 - "Skipping CUDA graph capture. Please set "
566 - "VLLM_TORCH_COMPILE_LEVEL=%d to use CUDA graphs.",
567 - CompilationLevel.PIECEWISE)
550 + "Skipping CUDA graph capture. Please add "
551 + "-O 3 to use CUDA graphs.", CompilationLevel.PIECEWISE)
552 return
553
554 start_time = time.perf_counter()