A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[6/N] torch.compile rollout to users (#10437)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 19, 2024 at 10:09 UTC
803f37eaaa11568f65acbf0bcd1044fb9b1610bf
15 files changed
+129
-141
tests/compile/piecewise/piecewise_compilation_config.json
deleted
-5
@@ -1,5 +0,0 @@
1
-{
2
- "use_cudagraph": true,
3
- "non_cudagraph_ops": ["silly.attention"],
4
- "cudagraph_copy_inputs": true
5
-}
\ No newline at end of file
tests/compile/piecewise/test_simple.py
+7
-11
@@ -2,7 +2,6 @@
2
Test the piecewise compilation with a simple model so that we
3
can exactly calculate the expected output and side effects.
4
"""
5
-import os
5
6
import torch
7
from torch import nn
@@ -11,7 +10,7 @@ from torch.library import Library
10
from vllm.compilation.compile_context import set_compile_context
11
from vllm.compilation.counter import compilation_counter
12
from vllm.compilation.decorators import support_torch_compile
14
-from vllm.config import CompilationLevel, VllmConfig
13
+from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
14
from vllm.plugins import set_current_vllm_config
15
from vllm.utils import direct_register_custom_op
16
@@ -77,12 +76,12 @@ class SillyModel(nn.Module):
76
77
def test_simple_piecewise_compile():
78
80
- directory = os.path.dirname(__file__)
81
- config = os.path.join(directory, "piecewise_compilation_config.json")
82
- os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
83
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
84
-
85
- vllm_config = VllmConfig()
79
+ vllm_config = VllmConfig(compilation_config=CompilationConfig(
80
+ level=CompilationLevel.PIECEWISE,
81
+ use_cudagraph=True,
82
+ non_cudagraph_ops=["silly.attention"],
83
+ cudagraph_copy_inputs=True,
84
+ ))
85
with set_current_vllm_config(vllm_config):
86
model = SillyModel(vllm_config=vllm_config, prefix='')
87
@@ -109,6 +108,3 @@ def test_simple_piecewise_compile():
108
output = model(input)
109
assert global_counter == 2
110
assert torch.allclose(output.cpu(), torch.tensor([3., 1.]))
112
-
113
- # clean up to avoid side effects for other tests
114
- del os.environ["VLLM_TORCH_COMPILE_CONFIG"]
tests/compile/piecewise/test_toy_llama.py
+17
-28
@@ -6,7 +6,6 @@ This is a tractable model, the weights and computation are specially designed
6
if the config `tractable_init` is set to True. Otherwise, the weights are
7
initialized randomly with a fixed seed.
8
"""
9
-import os
9
from dataclasses import dataclass
10
from typing import Optional, Tuple
11
@@ -18,7 +17,7 @@ from vllm.compilation.compile_context import set_compile_context
17
from vllm.compilation.counter import compilation_counter
18
from vllm.compilation.decorators import support_torch_compile
19
from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
21
-from vllm.plugins import set_compilation_config, set_current_vllm_config
20
+from vllm.plugins import set_current_vllm_config
21
from vllm.utils import direct_register_custom_op
22
23
# create a library to hold the custom op
@@ -254,23 +253,17 @@ def run_model(llama_config,
253
split_attn: bool = False) -> torch.Tensor:
254
255
if use_compile:
257
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
258
- CompilationLevel.PIECEWISE)
259
-
256
+ compilation_config = CompilationConfig(
257
+ level=CompilationLevel.PIECEWISE,
258
+ use_cudagraph=True,
259
+ )
260
if split_attn:
261
- set_compilation_config(
262
- CompilationConfig(
263
- use_cudagraph=True,
264
- non_cudagraph_ops=["silly.attention"],
265
- ))
266
- else:
267
- set_compilation_config(CompilationConfig(use_cudagraph=True, ))
261
+ compilation_config.non_cudagraph_ops = ["silly.attention"]
262
else:
269
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
270
- CompilationLevel.NO_COMPILATION)
271
- set_compilation_config(None)
263
+ compilation_config = CompilationConfig(
264
+ level=CompilationLevel.NO_COMPILATION, )
265
273
- vllm_config = VllmConfig()
266
+ vllm_config = VllmConfig(compilation_config=compilation_config)
267
with set_current_vllm_config(vllm_config):
268
model = LlamaModel(config=llama_config,
269
vllm_config=vllm_config,
@@ -288,10 +281,6 @@ def run_model(llama_config,
281
input_ids[:2].zero_()
282
output = model(input_ids[:2], positions[:2])
283
291
- # manual cleanup
292
- del os.environ["VLLM_TORCH_COMPILE_LEVEL"]
293
- set_compilation_config(None)
294
-
284
output = output.cpu()
285
286
if llama_config.tractable_init:
@@ -361,7 +350,6 @@ def test_toy_llama():
350
351
@torch.inference_mode
352
def benchmark():
364
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
353
from triton.testing import do_bench
354
355
# similar to llama 3.1-8B
@@ -387,15 +375,16 @@ def benchmark():
375
376
for piecewise in [False, True]:
377
if piecewise:
390
- set_compilation_config(
391
- CompilationConfig(
392
- use_cudagraph=True,
393
- non_cudagraph_ops=["silly.attention"],
394
- ))
378
+ compilation_config = CompilationConfig(
379
+ level=CompilationLevel.PIECEWISE,
380
+ use_cudagraph=True,
381
+ non_cudagraph_ops=["silly.attention"],
382
+ )
383
else:
396
- set_compilation_config(None)
384
+ compilation_config = CompilationConfig(
385
+ level=CompilationLevel.PIECEWISE, )
386
398
- vllm_config = VllmConfig()
387
+ vllm_config = VllmConfig(compilation_config=compilation_config)
388
with set_current_vllm_config(vllm_config):
389
model = LlamaModel(config=llama_config,
390
vllm_config=vllm_config,
tests/compile/test_basic_correctness.py
+9
-4
@@ -96,31 +96,36 @@ def test_compile_correctness(test_setting: TestSetting):
96
final_args = ["--enforce-eager"] + model_args + ["-pp", str(pp_size)] + \
97
["-tp", str(tp_size)]
98
99
+ all_args: List[List[str]] = []
100
all_envs: List[Optional[Dict[str, str]]] = []
101
102
for level in [
103
CompilationLevel.NO_COMPILATION,
104
CompilationLevel.PIECEWISE,
105
]:
105
- all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
106
+ all_args.append(final_args + ["-O", str(level)])
107
+ all_envs.append({})
108
109
# inductor will change the output, so we only compare if the output
110
# is close, not exactly the same.
111
compare_all_settings(
110
- model, [final_args] * 2,
112
+ model,
113
+ all_args,
114
all_envs,
115
method=method if method != "generate" else "generate_close")
116
all_envs.clear()
117
+ all_args.clear()
118
119
for level in [
120
CompilationLevel.NO_COMPILATION,
121
CompilationLevel.DYNAMO_AS_IS,
122
CompilationLevel.DYNAMO_ONCE,
123
]:
120
- all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
124
+ all_args.append(final_args + ["-O", str(level)])
125
+ all_envs.append({})
126
if level != CompilationLevel.DYNAMO_ONCE and not fullgraph:
127
# "DYNAMO_ONCE" will always use fullgraph
128
all_envs[-1][
129
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0" # type: ignore
130
126
- compare_all_settings(model, [final_args] * 3, all_envs, method=method)
131
+ compare_all_settings(model, all_args * 3, all_envs, method=method)
tests/compile/utils.py
+2
-2
@@ -4,7 +4,7 @@ import torch
4
5
from tests.quantization.utils import is_quant_method_supported
6
from vllm import LLM, SamplingParams
7
-from vllm.config import CompilationLevel
7
+from vllm.config import CompilationConfig, CompilationLevel
8
from vllm.platforms import current_platform
9
10
TEST_MODELS = [
@@ -65,7 +65,6 @@ def check_full_graph_support(model,
65
optimization_level,
66
tp_size=1):
67
# make sure these models can be captured in full graph mode
68
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
68
os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
69
70
# The base meta llama uses too much memory.
@@ -86,6 +85,7 @@ def check_full_graph_support(model,
85
enforce_eager=True,
86
tensor_parallel_size=tp_size,
87
disable_custom_all_reduce=True,
88
+ compilation_config=CompilationConfig(level=optimization_level),
89
**model_kwargs)
90
91
outputs = llm.generate(prompts, sampling_params)
tests/model_executor/test_enabled_custom_ops.py
+1
-3
@@ -1,4 +1,3 @@
1
-import os
1
from typing import List
2
3
import pytest
@@ -53,9 +52,8 @@ class Relu3(ReLUSquaredActivation):
52
])
53
def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],
54
default_on: bool):
56
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(torch_level)
55
vllm_config = VllmConfig(compilation_config=CompilationConfig(
58
- custom_ops=env.split(",")))
56
+ level=torch_level, custom_ops=env.split(",")))
57
with set_current_vllm_config(vllm_config):
58
assert CustomOp.default_on() == default_on
59
tests/tpu/test_compilation.py
+35
-12
@@ -1,24 +1,47 @@
1
import glob
2
import os
3
-import runpy
3
import tempfile
4
5
import depyf
6
8
-from vllm.config import CompilationLevel
9
-
10
-# disable custom dispatcher, let Dynamo takes over
11
-# all the control
12
-os.environ['VLLM_TORCH_COMPILE_LEVEL'] = str(CompilationLevel.DYNAMO_AS_IS)
7
+from vllm.config import CompilationConfig, CompilationLevel
8
9
temp_dir = tempfile.mkdtemp()
10
with depyf.prepare_debug(temp_dir):
16
- cur_dir = os.path.dirname(__file__)
17
- parent_dir = os.path.dirname(cur_dir)
18
- root_dir = os.path.dirname(parent_dir)
19
- example_file = os.path.join(root_dir, "examples",
20
- "offline_inference_tpu.py")
21
- runpy.run_path(example_file)
11
+ from vllm import LLM, SamplingParams
12
+
13
+ prompts = [
14
+ "A robot may not injure a human being",
15
+ "It is only with the heart that one can see rightly;",
16
+ "The greatest glory in living lies not in never falling,",
17
+ ]
18
+ answers = [
19
+ " or, through inaction, allow a human being to come to harm.",
20
+ " what is essential is invisible to the eye.",
21
+ " but in rising every time we fall.",
22
+ ]
23
+ N = 1
24
+ # Currently, top-p sampling is disabled. `top_p` should be 1.0.
25
+ sampling_params = SamplingParams(temperature=0.7,
26
+ top_p=1.0,
27
+ n=N,
28
+ max_tokens=16)
29
+
30
+ # Set `enforce_eager=True` to avoid ahead-of-time compilation.
31
+ # In real workloads, `enforace_eager` should be `False`.
32
+
33
+ # disable custom dispatcher, let Dynamo takes over
34
+ # all the control
35
+ llm = LLM(model="google/gemma-2b",
36
+ enforce_eager=True,
37
+ compilation_config=CompilationConfig(
38
+ level=CompilationLevel.DYNAMO_AS_IS))
39
+ outputs = llm.generate(prompts, sampling_params)
40
+ for output, answer in zip(outputs, answers):
41
+ prompt = output.prompt
42
+ generated_text = output.outputs[0].text
43
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
44
+ assert generated_text.startswith(answer)
45
46
compiled_code = sorted(
47
glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
tests/tpu/test_custom_dispatcher.py
+6
-4
@@ -13,7 +13,9 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
13
def test_custom_dispatcher():
14
compare_two_settings(
15
"google/gemma-2b",
16
- arg1=["--enforce-eager"],
17
- arg2=["--enforce-eager"],
18
- env1={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_ONCE)},
19
- env2={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_AS_IS)})
16
+ arg1=["--enforce-eager", "-O",
17
+ str(CompilationLevel.DYNAMO_ONCE)],
18
+ arg2=["--enforce-eager", "-O",
19
+ str(CompilationLevel.DYNAMO_AS_IS)],
20
+ env1={},
21
+ env2={})
vllm/config.py
+20
-23
@@ -2174,8 +2174,14 @@ class CompilationConfig(BaseModel):
2174
enabled_custom_ops: Counter[str] = PrivateAttr
2175
disabled_custom_ops: Counter[str] = PrivateAttr
2176
2177
+ @classmethod
2178
+ def from_cli(cls, cli_value: str) -> "CompilationConfig":
2179
+ """Parse the CLI value for the compilation config."""
2180
+ if cli_value in ["0", "1", "2", "3"]:
2181
+ return cls(level=int(cli_value))
2182
+ return CompilationConfig.model_validate_json(cli_value)
2183
+
2184
def model_post_init(self, __context: Any) -> None:
2178
- self.level = envs.VLLM_TORCH_COMPILE_LEVEL
2185
2186
count_none = self.custom_ops.count("none")
2187
count_all = self.custom_ops.count("all")
@@ -2249,26 +2255,6 @@ class CompilationConfig(BaseModel):
2255
"inductor_specialize_for_cudagraph_no_more_than is None")
2256
self.compile_sizes = self.inductor_compile_sizes
2257
2252
- @staticmethod
2253
- def select_and_init_config() -> "CompilationConfig":
2254
- """The order of selecting config is:
2255
- 1. Use the config specified in environment variable.
2256
- 2. Use the config specified in plugins.
2257
- 3. Use the default config.
2258
- """
2259
- config_path = envs.VLLM_TORCH_COMPILE_CONFIG
2260
- if config_path is not None:
2261
- with open(config_path) as json_file:
2262
- config = CompilationConfig.model_validate_json(
2263
- json_file.read())
2264
- else:
2265
- from vllm.plugins import get_compilation_config
2266
- predefined_config = get_compilation_config()
2267
- config = predefined_config if predefined_config is not None else (
2268
- CompilationConfig())
2269
-
2270
- return config
2271
-
2258
2259
@dataclass
2260
class VllmConfig:
@@ -2354,8 +2340,19 @@ class VllmConfig:
2340
self.model_config, self.load_config)
2341
2342
if self.compilation_config is None:
2357
- self.compilation_config = CompilationConfig.select_and_init_config(
2358
- )
2343
+ self.compilation_config = CompilationConfig()
2344
+ if envs.VLLM_USE_V1:
2345
+ # NOTE(woosuk): Currently, we use inductor because the piecewise
2346
+ # CUDA graphs do not work properly with the custom CUDA kernels.
2347
+ # FIXME(woosuk): Disable inductor to reduce the compilation time
2348
+ # and avoid any potential issues with the inductor.
2349
+ self.compilation_config.custom_ops = ["none"]
2350
+ self.compilation_config.use_cudagraph = True
2351
+ self.compilation_config.non_cudagraph_ops = [
2352
+ "vllm.unified_v1_flash_attention"
2353
+ ]
2354
+ self.compilation_config.use_inductor = True
2355
+ self.compilation_config.enable_fusion = False
2356
2357
current_platform.check_and_update_config(self)
2358
vllm/engine/arg_utils.py
+23
-6
@@ -8,12 +8,13 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
8
import torch
9
10
import vllm.envs as envs
11
-from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12
- DeviceConfig, HfOverrides, LoadConfig, LoadFormat,
13
- LoRAConfig, ModelConfig, ObservabilityConfig,
14
- ParallelConfig, PoolerConfig, PromptAdapterConfig,
15
- SchedulerConfig, SpeculativeConfig, TaskOption,
16
- TokenizerPoolConfig, VllmConfig)
11
+from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
12
+ DecodingConfig, DeviceConfig, HfOverrides, LoadConfig,
13
+ LoadFormat, LoRAConfig, ModelConfig,
14
+ ObservabilityConfig, ParallelConfig, PoolerConfig,
15
+ PromptAdapterConfig, SchedulerConfig,
16
+ SpeculativeConfig, TaskOption, TokenizerPoolConfig,
17
+ VllmConfig)
18
from vllm.executor.executor_base import ExecutorBase
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -189,6 +190,7 @@ class EngineArgs:
190
191
override_neuron_config: Optional[Dict[str, Any]] = None
192
override_pooler_config: Optional[PoolerConfig] = None
193
+ compilation_config: Optional[CompilationConfig] = None
194
195
def __post_init__(self):
196
if not self.tokenizer:
@@ -868,6 +870,20 @@ class EngineArgs:
870
help="Override or set the pooling method in the embedding model. "
871
"e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
872
873
+ parser.add_argument('--compilation-config',
874
+ '-O',
875
+ type=CompilationConfig.from_cli,
876
+ default=None,
877
+ help='torch.compile configuration for the model.'
878
+ 'When it is a number (0, 1, 2, 3), it will be '
879
+ 'interpreted as the optimization level.\n'
880
+ 'NOTE: level 0 is the default level without '
881
+ 'any optimization. level 1 and 2 are for internal '
882
+ 'testing only. level 3 is the recommended level '
883
+ 'for production.\n'
884
+ 'To specify the full compilation config, '
885
+ 'use a JSON string.')
886
+
887
return parser
888
889
@classmethod
@@ -1142,6 +1158,7 @@ class EngineArgs:
1158
decoding_config=decoding_config,
1159
observability_config=observability_config,
1160
prompt_adapter_config=prompt_adapter_config,
1161
+ compilation_config=self.compilation_config,
1162
)
1163
1164
vllm/engine/llm_engine.py
+3
-1
@@ -262,7 +262,8 @@ class LLMEngine:
262
"num_scheduler_steps=%d, chunked_prefill_enabled=%s "
263
"multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
264
"use_async_output_proc=%s, use_cached_outputs=%s, "
265
- "mm_processor_kwargs=%s, pooler_config=%r)",
265
+ "mm_processor_kwargs=%s, pooler_config=%r,"
266
+ "compilation_config=%r",
267
VLLM_VERSION,
268
model_config.model,
269
speculative_config,
@@ -297,6 +298,7 @@ class LLMEngine:
298
use_cached_outputs,
299
model_config.mm_processor_kwargs,
300
model_config.pooler_config,
301
+ vllm_config.compilation_config,
302
)
303
# TODO(woosuk): Print more configs in debug mode.
304
self.model_config = model_config
vllm/envs.py
-8
@@ -67,8 +67,6 @@ if TYPE_CHECKING:
67
VLLM_USE_TRITON_AWQ: bool = False
68
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
69
VLLM_SKIP_P2P_CHECK: bool = False
70
- VLLM_TORCH_COMPILE_LEVEL: int = 0
71
- VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
70
VLLM_DISABLED_KERNELS: List[str] = []
71
VLLM_USE_V1: bool = False
72
VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
@@ -209,12 +207,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
207
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
208
lambda: bool(
209
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
212
- "VLLM_TORCH_COMPILE_LEVEL":
213
- lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
214
-
215
- # Path to the config file for torch compile
216
- "VLLM_TORCH_COMPILE_CONFIG":
217
- lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
210
211
# local rank of the process in the distributed setting, used to determine
212
# the GPU device id
vllm/platforms/tpu.py
+2
-2
@@ -1,4 +1,3 @@
1
-import os
1
from typing import TYPE_CHECKING
2
3
import torch
@@ -40,7 +39,8 @@ class TpuPlatform(Platform):
39
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
40
from vllm.config import CompilationLevel
41
compilation_config = vllm_config.compilation_config
43
- if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
42
+ if compilation_config.level == CompilationLevel.NO_COMPILATION:
43
+ # TPU does not support NO_COMPILATION
44
compilation_config.level = CompilationLevel.DYNAMO_ONCE
45
assert compilation_config.level < CompilationLevel.PIECEWISE,\
46
"TPU does not support Inductor."
vllm/plugins/__init__.py
+1
-13
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional
5
import vllm.envs as envs
6
7
if TYPE_CHECKING:
8
- from vllm.config import CompilationConfig, VllmConfig
8
+ from vllm.config import VllmConfig
9
10
logger = logging.getLogger(__name__)
11
@@ -54,18 +54,6 @@ def load_general_plugins():
54
logger.exception("Failed to load plugin %s", plugin.name)
55
56
57
-_compilation_config: Optional["CompilationConfig"] = None
58
-
59
-
60
-def set_compilation_config(config: Optional["CompilationConfig"]):
61
- global _compilation_config
62
- _compilation_config = config
63
-
64
-
65
-def get_compilation_config() -> Optional["CompilationConfig"]:
66
- return _compilation_config
67
-
68
-
57
_current_vllm_config: Optional["VllmConfig"] = None
58
59
vllm/v1/worker/gpu_model_runner.py
+3
-19
@@ -8,13 +8,12 @@ import torch.distributed
8
import torch.nn as nn
9
10
from vllm.compilation.compile_context import set_compile_context
11
-from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
11
+from vllm.config import CompilationLevel, VllmConfig
12
from vllm.forward_context import set_forward_context
13
from vllm.inputs import INPUT_REGISTRY, InputRegistry
14
from vllm.logger import init_logger
15
from vllm.model_executor.model_loader import get_model
16
from vllm.multimodal import MultiModalKwargs
17
-from vllm.plugins import set_compilation_config
17
from vllm.sampling_params import SamplingParams, SamplingType
18
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19
is_pin_memory_available)
@@ -508,20 +507,6 @@ class GPUModelRunner:
507
return model_runner_output
508
509
def load_model(self) -> None:
511
- if self.use_cuda_graph:
512
- # NOTE(woosuk): Currently, we use inductor because the piecewise
513
- # CUDA graphs do not work properly with the custom CUDA kernels.
514
- # FIXME(woosuk): Disable inductor to reduce the compilation time
515
- # and avoid any potential issues with the inductor.
516
- set_compilation_config(
517
- CompilationConfig(
518
- custom_ops=["none"],
519
- use_cudagraph=True,
520
- non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
521
- use_inductor=True,
522
- enable_fusion=False,
523
- ))
524
-
510
logger.info("Starting to load model %s...", self.model_config.model)
511
with DeviceMemoryProfiler() as m: # noqa: SIM117
512
self.model = get_model(vllm_config=self.vllm_config)
@@ -562,9 +547,8 @@ class GPUModelRunner:
547
def capture_model(self) -> None:
548
if not self.use_cuda_graph:
549
logger.warning(
565
- "Skipping CUDA graph capture. Please set "
566
- "VLLM_TORCH_COMPILE_LEVEL=%d to use CUDA graphs.",
567
- CompilationLevel.PIECEWISE)
550
+ "Skipping CUDA graph capture. Please add "
551
+ "-O 3 to use CUDA graphs.", CompilationLevel.PIECEWISE)
552
return
553
554
start_time = time.perf_counter()