@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8345045833
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Performance][Spec Decode] Optimize ngram lookup performance (#9333)

Lily Liu committed Oct 16, 2024 at 12:37 UTC 83450458339b07765b0e72a822e5fe93eeaf5258
1 file changed +11 -6
vllm/spec_decode/ngram_worker.py
+11 -6
@@ -67,9 +67,16 @@ class NGramWorker(NonLLMProposerWorkerBase):
67 execute_model_req.seq_group_metadata_list):
68 seq_data = next(iter(seq_group_metadata.seq_data.values()))
69
70 + seq_len = seq_data.get_len()
71 + # When seq_len is less than 3072 (3K), we use CPU to perform
72 + # the ngram match. Otherwise, we use the device specified in
73 + # the model config (normally GPU). 3072 is a rough threshold
74 + # based on profiling on H100, and it can be adjusted based
75 + # on the actual performance on different hardware.
76 + cur_device = "cpu" if seq_len < 3072 else self.device
77 input_ids = torch.as_tensor(seq_data.get_token_ids(),
78 dtype=torch.long,
72 - device=self.device)
79 + device=cur_device)
80 input_length = seq_data.get_len()
81
82 for ngram_size in range(
@@ -91,17 +98,15 @@ class NGramWorker(NonLLMProposerWorkerBase):
98 # first_match includes "values" (bool), indicating whether
99 # the match is found, and "indices", indicating the index
100 # of the first match.
94 - # Note that "first_match.values.item()" triggers GPU-CPU
95 - # sync so it is a bit inefficient, but we have not found
96 - # a better way to do this.
101 first_match = matches.max(dim=-1)
102 if first_match.values.item():
103 proposal_start_idx = first_match.indices.add_(ngram_size)
104 spec_indices = (
105 proposal_start_idx).repeat(sample_len) + torch.arange(
102 - sample_len, device=self.device)
106 + sample_len, device=cur_device)
107 spec_indices.clamp_(max=input_ids.shape[-1] - 1)
104 - res = input_ids.gather(dim=-1, index=spec_indices)
108 + res = input_ids.gather(dim=-1,
109 + index=spec_indices).to(self.device)
110 token_id_list.append(res)
111 token_prob_list.append(
112 torch.nn.functional.one_hot(