@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7851b45196
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[5/N][torch.compile] torch.jit.script --> torch.compile (#10406)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 18, 2024 at 07:20 UTC 7851b45196aff994277ec832c0cf5bec0073f08e
4 files changed +6 -6
vllm/model_executor/layers/rejection_sampler.py
+1 -1
@@ -368,7 +368,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
368 # Note that we always sample with replacement.
369 # probs will be modified in place, but this is fine, as we pass
370 # in a copy already.
371 -@torch.jit.script
371 +@torch.compile(dynamic=True)
372 def _multinomial(
373 probs: torch.Tensor,
374 num_samples: int,
vllm/model_executor/layers/vocab_parallel_embedding.py
+2 -2
@@ -133,13 +133,13 @@ class VocabParallelEmbeddingShardIndices:
133 assert self.num_added_elements <= self.num_added_elements_padded
134
135
136 -@torch.jit.script
136 +@torch.compile(dynamic=True)
137 def get_masked_input_and_mask(
138 input_: torch.Tensor, org_vocab_start_index: int,
139 org_vocab_end_index: int, num_org_vocab_padding: int,
140 added_vocab_start_index: int,
141 added_vocab_end_index: int) -> Tuple[torch.Tensor, torch.Tensor]:
142 - # torch.jit.script will fuse all of the pointwise ops below
142 + # torch.compile will fuse all of the pointwise ops below
143 # into a single kernel, making it very fast
144 org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ <
145 org_vocab_end_index)
vllm/model_executor/models/phi3_small.py
+2 -2
@@ -54,12 +54,12 @@ class HeadMajorColumnParallelLinear(MergedColumnParallelLinear):
54 return load_column_parallel_weight(param, loaded_weight)
55
56
57 -@torch.jit.script
57 +@torch.compile(dynamic=True)
58 def quick_gelu(x):
59 return x * torch.sigmoid(1.702 * x)
60
61
62 -@torch.jit.script
62 +@torch.compile(dynamic=True)
63 def gegelu(input, limit: Optional[float] = None):
64 a_gelu, a_linear = input[..., ::2], input[..., 1::2]
65 if limit is not None:
vllm/worker/model_runner.py
+1 -1
@@ -1769,7 +1769,7 @@ class CUDAGraphRunner(nn.Module):
1769 # Run the model a few times without capturing the graph.
1770 # This is to make sure that the captured graph does not include the
1771 # kernel launches for initial benchmarking (e.g., Triton autotune).
1772 - # Note one iteration is not enough for torch.jit.script
1772 + # Note one iteration is not enough for torch.compile
1773 for _ in range(_NUM_WARMUP_ITERS):
1774 self.model(
1775 input_ids=input_ids,