@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 20410b2fda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[platform] support custom torch.compile backend key (#11318)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

wangxiyuan committed Jan 10, 2025 at 23:46 UTC 20410b2fdac1818ead453018fb07c2ff90ee6770
5 files changed +14 -5
vllm/model_executor/layers/rejection_sampler.py
+2 -1
@@ -9,6 +9,7 @@ import vllm.envs as envs
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.spec_decode_base_sampler import (
11 SpecDecodeStochasticBaseSampler)
12 +from vllm.platforms import current_platform
13
14 logger = init_logger(__name__)
15
@@ -368,7 +369,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
369 # Note that we always sample with replacement.
370 # probs will be modified in place, but this is fine, as we pass
371 # in a copy already.
371 -@torch.compile(dynamic=True)
372 +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
373 def _multinomial(
374 probs: torch.Tensor,
375 num_samples: int,
vllm/model_executor/layers/vocab_parallel_embedding.py
+1 -1
@@ -133,7 +133,7 @@ class VocabParallelEmbeddingShardIndices:
133 assert self.num_added_elements <= self.num_added_elements_padded
134
135
136 -@torch.compile(dynamic=True)
136 +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
137 def get_masked_input_and_mask(
138 input_: torch.Tensor, org_vocab_start_index: int,
139 org_vocab_end_index: int, num_org_vocab_padding: int,
vllm/model_executor/models/commandr.py
+2 -1
@@ -45,6 +45,7 @@ from vllm.model_executor.model_loader.weight_utils import (
45 row_parallel_weight_loader)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.utils import set_weight_attrs
48 +from vllm.platforms import current_platform
49 from vllm.sequence import IntermediateTensors
50
51 from .interfaces import SupportsLoRA, SupportsPP
@@ -53,7 +54,7 @@ from .utils import (extract_layer_index, is_pp_missing_parameter,
54 maybe_prefix)
55
56
56 -@torch.compile
57 +@torch.compile(backend=current_platform.simple_compile_backend)
58 def layer_norm_func(hidden_states, weight, variance_epsilon):
59 input_dtype = hidden_states.dtype
60 hidden_states = hidden_states.to(torch.float32)
vllm/model_executor/models/phi3_small.py
+3 -2
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
20 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
21 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.platforms import current_platform
24 from vllm.sequence import IntermediateTensors
25
26 from .interfaces import SupportsPP
@@ -54,12 +55,12 @@ class HeadMajorColumnParallelLinear(MergedColumnParallelLinear):
55 return load_column_parallel_weight(param, loaded_weight)
56
57
57 -@torch.compile(dynamic=True)
58 +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
59 def quick_gelu(x):
60 return x * torch.sigmoid(1.702 * x)
61
62
62 -@torch.compile(dynamic=True)
63 +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
64 def gegelu(input, limit: Optional[float] = None):
65 a_gelu, a_linear = input[..., ::2], input[..., 1::2]
66 if limit is not None:
vllm/platforms/interface.py
+6
@@ -82,6 +82,12 @@ class Platform:
82 # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
83 # use "CPU" as a fallback for platforms not registered in PyTorch
84 dispatch_key: str = "CPU"
85 + # The torch.compile backend for compiling simple and
86 + # standalone functions. The default value is "inductor" to keep
87 + # the same behavior as PyTorch.
88 + # NOTE: for the forward part of the model, vLLM has another separate
89 + # compilation strategy.
90 + simple_compile_backend: str = "inductor"
91 supported_quantization: list[str] = []
92
93 def is_cuda(self) -> bool: