A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[5/N][torch.compile] torch.jit.script --> torch.compile (#10406)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 18, 2024 at 07:20 UTC
7851b45196aff994277ec832c0cf5bec0073f08e
4 files changed
+6
-6
vllm/model_executor/layers/rejection_sampler.py
+1
-1
@@ -368,7 +368,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
368
# Note that we always sample with replacement.
369
# probs will be modified in place, but this is fine, as we pass
370
# in a copy already.
371
-@torch.jit.script
371
+@torch.compile(dynamic=True)
372
def _multinomial(
373
probs: torch.Tensor,
374
num_samples: int,
vllm/model_executor/layers/vocab_parallel_embedding.py
+2
-2
@@ -133,13 +133,13 @@ class VocabParallelEmbeddingShardIndices:
133
assert self.num_added_elements <= self.num_added_elements_padded
134
135
136
-@torch.jit.script
136
+@torch.compile(dynamic=True)
137
def get_masked_input_and_mask(
138
input_: torch.Tensor, org_vocab_start_index: int,
139
org_vocab_end_index: int, num_org_vocab_padding: int,
140
added_vocab_start_index: int,
141
added_vocab_end_index: int) -> Tuple[torch.Tensor, torch.Tensor]:
142
- # torch.jit.script will fuse all of the pointwise ops below
142
+ # torch.compile will fuse all of the pointwise ops below
143
# into a single kernel, making it very fast
144
org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ <
145
org_vocab_end_index)
vllm/model_executor/models/phi3_small.py
+2
-2
@@ -54,12 +54,12 @@ class HeadMajorColumnParallelLinear(MergedColumnParallelLinear):
54
return load_column_parallel_weight(param, loaded_weight)
55
56
57
-@torch.jit.script
57
+@torch.compile(dynamic=True)
58
def quick_gelu(x):
59
return x * torch.sigmoid(1.702 * x)
60
61
62
-@torch.jit.script
62
+@torch.compile(dynamic=True)
63
def gegelu(input, limit: Optional[float] = None):
64
a_gelu, a_linear = input[..., ::2], input[..., 1::2]
65
if limit is not None:
vllm/worker/model_runner.py
+1
-1
@@ -1769,7 +1769,7 @@ class CUDAGraphRunner(nn.Module):
1769
# Run the model a few times without capturing the graph.
1770
# This is to make sure that the captured graph does not include the
1771
# kernel launches for initial benchmarking (e.g., Triton autotune).
1772
- # Note one iteration is not enough for torch.jit.script
1772
+ # Note one iteration is not enough for torch.compile
1773
for _ in range(_NUM_WARMUP_ITERS):
1774
self.model(
1775
input_ids=input_ids,