A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[platform] support custom torch.compile backend key (#11318)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
wangxiyuan committed
Jan 10, 2025 at 23:46 UTC
20410b2fdac1818ead453018fb07c2ff90ee6770
5 files changed
+14
-5
vllm/model_executor/layers/rejection_sampler.py
+2
-1
@@ -9,6 +9,7 @@ import vllm.envs as envs
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.spec_decode_base_sampler import (
11
SpecDecodeStochasticBaseSampler)
12
+from vllm.platforms import current_platform
13
14
logger = init_logger(__name__)
15
@@ -368,7 +369,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
369
# Note that we always sample with replacement.
370
# probs will be modified in place, but this is fine, as we pass
371
# in a copy already.
371
-@torch.compile(dynamic=True)
372
+@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
373
def _multinomial(
374
probs: torch.Tensor,
375
num_samples: int,
vllm/model_executor/layers/vocab_parallel_embedding.py
+1
-1
@@ -133,7 +133,7 @@ class VocabParallelEmbeddingShardIndices:
133
assert self.num_added_elements <= self.num_added_elements_padded
134
135
136
-@torch.compile(dynamic=True)
136
+@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
137
def get_masked_input_and_mask(
138
input_: torch.Tensor, org_vocab_start_index: int,
139
org_vocab_end_index: int, num_org_vocab_padding: int,
vllm/model_executor/models/commandr.py
+2
-1
@@ -45,6 +45,7 @@ from vllm.model_executor.model_loader.weight_utils import (
45
row_parallel_weight_loader)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.utils import set_weight_attrs
48
+from vllm.platforms import current_platform
49
from vllm.sequence import IntermediateTensors
50
51
from .interfaces import SupportsLoRA, SupportsPP
@@ -53,7 +54,7 @@ from .utils import (extract_layer_index, is_pp_missing_parameter,
54
maybe_prefix)
55
56
56
-@torch.compile
57
+@torch.compile(backend=current_platform.simple_compile_backend)
58
def layer_norm_func(hidden_states, weight, variance_epsilon):
59
input_dtype = hidden_states.dtype
60
hidden_states = hidden_states.to(torch.float32)
vllm/model_executor/models/phi3_small.py
+3
-2
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
20
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
21
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.platforms import current_platform
24
from vllm.sequence import IntermediateTensors
25
26
from .interfaces import SupportsPP
@@ -54,12 +55,12 @@ class HeadMajorColumnParallelLinear(MergedColumnParallelLinear):
55
return load_column_parallel_weight(param, loaded_weight)
56
57
57
-@torch.compile(dynamic=True)
58
+@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
59
def quick_gelu(x):
60
return x * torch.sigmoid(1.702 * x)
61
62
62
-@torch.compile(dynamic=True)
63
+@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
64
def gegelu(input, limit: Optional[float] = None):
65
a_gelu, a_linear = input[..., ::2], input[..., 1::2]
66
if limit is not None:
vllm/platforms/interface.py
+6
@@ -82,6 +82,12 @@ class Platform:
82
# check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
83
# use "CPU" as a fallback for platforms not registered in PyTorch
84
dispatch_key: str = "CPU"
85
+ # The torch.compile backend for compiling simple and
86
+ # standalone functions. The default value is "inductor" to keep
87
+ # the same behavior as PyTorch.
88
+ # NOTE: for the forward part of the model, vLLM has another separate
89
+ # compilation strategy.
90
+ simple_compile_backend: str = "inductor"
91
supported_quantization: list[str] = []
92
93
def is_cuda(self) -> bool: