A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Performance][Spec Decode] Optimize ngram lookup performance (#9333)
Lily Liu committed
Oct 16, 2024 at 12:37 UTC
83450458339b07765b0e72a822e5fe93eeaf5258
1 file changed
+11
-6
vllm/spec_decode/ngram_worker.py
+11
-6
@@ -67,9 +67,16 @@ class NGramWorker(NonLLMProposerWorkerBase):
67
execute_model_req.seq_group_metadata_list):
68
seq_data = next(iter(seq_group_metadata.seq_data.values()))
69
70
+ seq_len = seq_data.get_len()
71
+ # When seq_len is less than 3072 (3K), we use CPU to perform
72
+ # the ngram match. Otherwise, we use the device specified in
73
+ # the model config (normally GPU). 3072 is a rough threshold
74
+ # based on profiling on H100, and it can be adjusted based
75
+ # on the actual performance on different hardware.
76
+ cur_device = "cpu" if seq_len < 3072 else self.device
77
input_ids = torch.as_tensor(seq_data.get_token_ids(),
78
dtype=torch.long,
72
- device=self.device)
79
+ device=cur_device)
80
input_length = seq_data.get_len()
81
82
for ngram_size in range(
@@ -91,17 +98,15 @@ class NGramWorker(NonLLMProposerWorkerBase):
98
# first_match includes "values" (bool), indicating whether
99
# the match is found, and "indices", indicating the index
100
# of the first match.
94
- # Note that "first_match.values.item()" triggers GPU-CPU
95
- # sync so it is a bit inefficient, but we have not found
96
- # a better way to do this.
101
first_match = matches.max(dim=-1)
102
if first_match.values.item():
103
proposal_start_idx = first_match.indices.add_(ngram_size)
104
spec_indices = (
105
proposal_start_idx).repeat(sample_len) + torch.arange(
102
- sample_len, device=self.device)
106
+ sample_len, device=cur_device)
107
spec_indices.clamp_(max=input_ids.shape[-1] - 1)
104
- res = input_ids.gather(dim=-1, index=spec_indices)
108
+ res = input_ids.gather(dim=-1,
109
+ index=spec_indices).to(self.device)
110
token_id_list.append(res)
111
token_prob_list.append(
112
torch.nn.functional.one_hot(