A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc] Various simplifications and typing fixes (#5368)
Nick Hill committed
Jun 10, 2024 at 19:29 UTC
a008629807b45c2b015d4e0bc7c4186cecbb642f
8 files changed
+63
-90
vllm/engine/output_processor/multi_step.py
+1
-1
@@ -78,7 +78,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
78
79
# Since there's only one sequence per sequence group, we can take the
80
# first sample.
81
- samples = [outputs[step].samples[0] for step in range(len(outputs))]
81
+ samples = [output.samples[0] for output in outputs]
82
83
# -1 means the output token is not valid (eg. due to spec decode
84
# rejecting tokens).
vllm/model_executor/layers/rejection_sampler.py
+4
-2
@@ -306,8 +306,10 @@ class RejectionSampler(nn.Module):
306
307
# Fill in the first k columns of the output tensor using masks and data
308
# tensors.
309
- output[:, :k] = torch.where(accepted_mask, draft_token_ids,
310
- -torch.ones_like(draft_token_ids))
309
+ torch.where(accepted_mask,
310
+ draft_token_ids,
311
+ -torch.ones_like(draft_token_ids),
312
+ out=output)
313
314
# Fill the last column.
315
# We check output directly as accepted may have True values inconsistent
vllm/spec_decode/batch_expansion.py
+12
-27
@@ -80,7 +80,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
80
81
target_sampler_output = self._scorer_worker.execute_model(
82
execute_model_req=execute_model_req.clone(
83
- seq_group_metadata_list=target_seq_group_metadata_list, ))
83
+ seq_group_metadata_list=target_seq_group_metadata_list))
84
assert len(target_sampler_output) == 1, "expected single-step output"
85
target_sampler_output = target_sampler_output[0]
86
@@ -140,8 +140,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
140
num_scoring_tokens)
141
142
def _contract_batch(
143
- self, contracted_bs: int,
144
- target_sampler_output: List[SamplerOutput],
143
+ self, contracted_bs: int, target_sampler_output: SamplerOutput,
144
proposals: SpeculativeProposals, num_scoring_tokens: int,
145
non_spec_indices: List[int], spec_indices: List[int],
146
k: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
@@ -167,30 +166,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
166
non_spec_expanded_bs, _ = non_spec_target_token_ids.shape
167
spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
168
170
- target_token_ids = target_token_ids.squeeze().reshape(
171
- spec_expanded_bs, k + 1)
172
- target_probs = target_probs.squeeze().reshape(spec_expanded_bs, k + 1,
173
- self._vocab_size)
174
- target_logprobs = target_logprobs.squeeze().reshape(
175
- spec_expanded_bs, k + 1, self._vocab_size)
176
-
177
- all_tokens = torch.full(size=(contracted_bs, k + 1),
178
- fill_value=-1,
179
- device=self._device,
180
- dtype=torch.long)
181
- all_probs = torch.zeros(contracted_bs,
182
- k + 1,
183
- self._vocab_size,
184
- device=self._device,
185
- dtype=torch.float32)
186
- all_logprobs = torch.full(size=(
187
- contracted_bs,
188
- k + 1,
189
- self._vocab_size,
190
- ),
191
- fill_value=-float("inf"),
192
- device=self._device,
193
- dtype=torch.float32)
169
+ target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
170
+ target_probs = target_probs.reshape(*target_token_ids.shape,
171
+ self._vocab_size)
172
+ target_logprobs = target_logprobs.reshape(target_probs.shape)
173
+
174
+ all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1),
175
+ fill_value=-1)
176
+ all_probs = target_probs.new_zeros(*all_tokens.shape, self._vocab_size)
177
+ all_logprobs = target_logprobs.new_full(size=all_probs.shape,
178
+ fill_value=-float("inf"))
179
180
if non_spec_indices:
181
all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
vllm/spec_decode/spec_decode_worker.py
+7
-7
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
3
4
import torch
5
6
+from vllm.config import SpeculativeConfig
7
from vllm.distributed.communication_op import broadcast_tensor_dict
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
@@ -30,7 +31,7 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
31
WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config.
32
"""
33
assert "speculative_config" in kwargs
33
- speculative_config = kwargs.get("speculative_config")
34
+ speculative_config: SpeculativeConfig = kwargs.get("speculative_config")
35
assert speculative_config is not None
36
37
target_worker = Worker(*args, **kwargs)
@@ -109,12 +110,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
110
logger.info("Configuring SpecDecodeWorker with proposer=%s",
111
type(proposer_worker))
112
112
- return SpecDecodeWorker(
113
- proposer_worker,
114
- scorer_worker,
115
- disable_by_batch_size=disable_by_batch_size,
116
- rejection_sampler=RejectionSampler(
117
- disable_bonus_tokens=disable_bonus_tokens, ))
113
+ return SpecDecodeWorker(proposer_worker,
114
+ scorer_worker,
115
+ disable_by_batch_size=disable_by_batch_size,
116
+ rejection_sampler=RejectionSampler(
117
+ disable_bonus_tokens=disable_bonus_tokens))
118
119
def __init__(
120
self,
vllm/spec_decode/top1_proposer.py
+18
-27
@@ -148,7 +148,8 @@ class Top1Proposer(SpeculativeProposer):
148
nonzero_proposal_len_indices,
149
)
150
151
- def _remove_no_proposal_seqs(self, proposal_lens, maybe_sampler_output,
151
+ @staticmethod
152
+ def _remove_no_proposal_seqs(proposal_lens, maybe_sampler_output,
153
nonzero_proposal_len_indices, transposed):
154
"""Remove sequences from nonzero_proposal_len_indices and reset
155
their proposal_len to 0 the draft worker does not provide a proposal
@@ -207,7 +208,7 @@ class Top1Proposer(SpeculativeProposer):
208
self,
209
batch_size: int,
210
proposal_len: int,
210
- maybe_sampler_output: Optional[SamplerOutput],
211
+ maybe_sampler_output: Optional[List[SamplerOutput]],
212
proposal_lens: List[int],
213
nonzero_proposal_len_indices: List[int],
214
sampler_transposed: bool,
@@ -218,25 +219,19 @@ class Top1Proposer(SpeculativeProposer):
219
if maybe_sampler_output is None:
220
# If no speculative tokens, the sampler output will be None.
221
# In this case we return empty proposals.
221
- proposal_tokens = torch.full(
222
- size=(
223
- batch_size,
224
- proposal_len,
225
- ),
226
- fill_value=-1,
227
- dtype=torch.long,
228
- device=self._device,
229
- )
230
- proposal_probs = torch.zeros(
231
- batch_size,
232
- proposal_len,
233
- self._vocab_size,
234
- dtype=torch.float32,
235
- device=self._device,
236
- )
237
- proposal_lens_tensor = torch.zeros(len(proposal_lens),
238
- dtype=torch.long,
239
- device=self._device)
222
+ proposal_tokens = torch.tensor(-1,
223
+ dtype=torch.long,
224
+ device=self._device).expand(
225
+ batch_size, proposal_len)
226
+ proposal_probs = torch.tensor(0,
227
+ dtype=torch.float32,
228
+ device=self._device).expand(
229
+ batch_size, proposal_len,
230
+ self._vocab_size)
231
+ proposal_lens_tensor = torch.tensor(0,
232
+ dtype=torch.long,
233
+ device=self._device).expand(
234
+ len(proposal_lens))
235
return proposal_tokens, proposal_probs, proposal_lens_tensor
236
237
sampler_output = maybe_sampler_output
@@ -246,18 +241,14 @@ class Top1Proposer(SpeculativeProposer):
241
# Now, reformat the output GPU tensors such that each sequence has
242
# a proposal. the proposal can be empty, e.g. [-1, -1, -1]
243
249
- entire_proposal_tokens = torch.full(
244
+ entire_proposal_tokens = proposal_tokens.new_full(
245
size=(batch_size, *proposal_tokens.shape[1:]),
246
fill_value=-1,
252
- dtype=torch.long,
253
- device=self._device,
247
)
248
entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
256
- entire_proposal_probs = torch.zeros(
249
+ entire_proposal_probs = proposal_probs.new_zeros(
250
batch_size,
251
*proposal_probs.shape[1:],
259
- dtype=torch.float32,
260
- device=self._device,
252
)
253
entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
254
vllm/spec_decode/util.py
+3
-8
@@ -1,12 +1,11 @@
1
from contextlib import contextmanager
2
-from itertools import chain
2
from typing import Dict, List, Tuple
3
4
import torch
5
6
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
7
SamplerOutput, SequenceGroupMetadata,
9
- SequenceGroupOutput, SequenceOutput)
8
+ SequenceOutput)
9
10
SeqId = int
11
@@ -16,11 +15,7 @@ def get_all_seq_ids(
15
"""Given a list of SequenceGroupMetadata, create a list of all
16
sequence ids.
17
"""
19
- return list(
20
- chain.from_iterable([
21
- seq_group_metadata.seq_data.keys()
22
- for seq_group_metadata in seq_group_metadata_list
23
- ]))
18
+ return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
19
20
21
def get_all_num_logprobs(
@@ -68,7 +63,7 @@ def create_sequence_group_output(
63
seq_id: SeqId,
64
topk_token_ids: List[int],
65
topk_logprobs: List[float],
71
-) -> SequenceGroupOutput:
66
+) -> CompletionSequenceGroupOutput:
67
"""Create a SequenceGroupOutput given the sampling results.
68
69
Args:
vllm/transformers_utils/config.py
+3
-3
@@ -1,4 +1,4 @@
1
-from typing import Dict, Optional
1
+from typing import Dict, Optional, Type
2
3
from transformers import PretrainedConfig
4
@@ -9,7 +9,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
10
logger = init_logger(__name__)
11
12
-_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
12
+_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
13
"chatglm": ChatGLMConfig,
14
"dbrx": DbrxConfig,
15
"mpt": MPTConfig,
@@ -68,4 +68,4 @@ def get_hf_text_config(config: PretrainedConfig):
68
assert hasattr(config.text_config, "num_attention_heads")
69
return config.text_config
70
else:
71
- return config
\ No newline at end of file
71
+ return config
vllm/worker/model_runner.py
+15
-15
@@ -527,16 +527,6 @@ class ModelRunner:
527
)
528
assert max_query_len > 0, ("query_lens: {}".format(query_lens))
529
530
- context_lens_tensor = torch.tensor(context_lens,
531
- dtype=torch.int,
532
- device=self.device)
533
- query_lens_tensor = torch.tensor(query_lens,
534
- dtype=torch.long,
535
- device=self.device)
536
- query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
537
- dtype=torch.int32,
538
- device=self.device)
539
-
530
seq_lens_tensor = torch.tensor(seq_lens,
531
dtype=torch.int,
532
device=self.device)
@@ -544,11 +534,6 @@ class ModelRunner:
534
dtype=torch.int32,
535
device=self.device)
536
547
- torch.cumsum(query_lens_tensor,
548
- dim=0,
549
- dtype=query_start_loc.dtype,
550
- out=query_start_loc[1:])
551
-
537
torch.cumsum(seq_lens_tensor,
538
dim=0,
539
dtype=seq_start_loc.dtype,
@@ -601,6 +586,21 @@ class ModelRunner:
586
seq_start_loc=seq_start_loc,
587
data_type=kv_cache_dtype)
588
else:
589
+ context_lens_tensor = torch.tensor(context_lens,
590
+ dtype=torch.int,
591
+ device=self.device)
592
+ query_lens_tensor = torch.tensor(query_lens,
593
+ dtype=torch.long,
594
+ device=self.device)
595
+ query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
596
+ dtype=torch.int32,
597
+ device=self.device)
598
+
599
+ torch.cumsum(query_lens_tensor,
600
+ dim=0,
601
+ dtype=query_start_loc.dtype,
602
+ out=query_start_loc[1:])
603
+
604
attn_metadata = self.attn_backend.make_metadata(
605
num_prefills=num_prefills,
606
slot_mapping=slot_mapping_tensor,