@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a008629807
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Various simplifications and typing fixes (#5368)

Nick Hill committed Jun 10, 2024 at 19:29 UTC a008629807b45c2b015d4e0bc7c4186cecbb642f
8 files changed +63 -90
vllm/engine/output_processor/multi_step.py
+1 -1
@@ -78,7 +78,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
78
79 # Since there's only one sequence per sequence group, we can take the
80 # first sample.
81 - samples = [outputs[step].samples[0] for step in range(len(outputs))]
81 + samples = [output.samples[0] for output in outputs]
82
83 # -1 means the output token is not valid (eg. due to spec decode
84 # rejecting tokens).
vllm/model_executor/layers/rejection_sampler.py
+4 -2
@@ -306,8 +306,10 @@ class RejectionSampler(nn.Module):
306
307 # Fill in the first k columns of the output tensor using masks and data
308 # tensors.
309 - output[:, :k] = torch.where(accepted_mask, draft_token_ids,
310 - -torch.ones_like(draft_token_ids))
309 + torch.where(accepted_mask,
310 + draft_token_ids,
311 + -torch.ones_like(draft_token_ids),
312 + out=output)
313
314 # Fill the last column.
315 # We check output directly as accepted may have True values inconsistent
vllm/spec_decode/batch_expansion.py
+12 -27
@@ -80,7 +80,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
80
81 target_sampler_output = self._scorer_worker.execute_model(
82 execute_model_req=execute_model_req.clone(
83 - seq_group_metadata_list=target_seq_group_metadata_list, ))
83 + seq_group_metadata_list=target_seq_group_metadata_list))
84 assert len(target_sampler_output) == 1, "expected single-step output"
85 target_sampler_output = target_sampler_output[0]
86
@@ -140,8 +140,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
140 num_scoring_tokens)
141
142 def _contract_batch(
143 - self, contracted_bs: int,
144 - target_sampler_output: List[SamplerOutput],
143 + self, contracted_bs: int, target_sampler_output: SamplerOutput,
144 proposals: SpeculativeProposals, num_scoring_tokens: int,
145 non_spec_indices: List[int], spec_indices: List[int],
146 k: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
@@ -167,30 +166,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
166 non_spec_expanded_bs, _ = non_spec_target_token_ids.shape
167 spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
168
170 - target_token_ids = target_token_ids.squeeze().reshape(
171 - spec_expanded_bs, k + 1)
172 - target_probs = target_probs.squeeze().reshape(spec_expanded_bs, k + 1,
173 - self._vocab_size)
174 - target_logprobs = target_logprobs.squeeze().reshape(
175 - spec_expanded_bs, k + 1, self._vocab_size)
176 -
177 - all_tokens = torch.full(size=(contracted_bs, k + 1),
178 - fill_value=-1,
179 - device=self._device,
180 - dtype=torch.long)
181 - all_probs = torch.zeros(contracted_bs,
182 - k + 1,
183 - self._vocab_size,
184 - device=self._device,
185 - dtype=torch.float32)
186 - all_logprobs = torch.full(size=(
187 - contracted_bs,
188 - k + 1,
189 - self._vocab_size,
190 - ),
191 - fill_value=-float("inf"),
192 - device=self._device,
193 - dtype=torch.float32)
169 + target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
170 + target_probs = target_probs.reshape(*target_token_ids.shape,
171 + self._vocab_size)
172 + target_logprobs = target_logprobs.reshape(target_probs.shape)
173 +
174 + all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1),
175 + fill_value=-1)
176 + all_probs = target_probs.new_zeros(*all_tokens.shape, self._vocab_size)
177 + all_logprobs = target_logprobs.new_full(size=all_probs.shape,
178 + fill_value=-float("inf"))
179
180 if non_spec_indices:
181 all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
vllm/spec_decode/spec_decode_worker.py
+7 -7
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5
6 +from vllm.config import SpeculativeConfig
7 from vllm.distributed.communication_op import broadcast_tensor_dict
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
@@ -30,7 +31,7 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
31 WorkerWrapper. It constructs a SpecDecodeWorker from the speculative config.
32 """
33 assert "speculative_config" in kwargs
33 - speculative_config = kwargs.get("speculative_config")
34 + speculative_config: SpeculativeConfig = kwargs.get("speculative_config")
35 assert speculative_config is not None
36
37 target_worker = Worker(*args, **kwargs)
@@ -109,12 +110,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
110 logger.info("Configuring SpecDecodeWorker with proposer=%s",
111 type(proposer_worker))
112
112 - return SpecDecodeWorker(
113 - proposer_worker,
114 - scorer_worker,
115 - disable_by_batch_size=disable_by_batch_size,
116 - rejection_sampler=RejectionSampler(
117 - disable_bonus_tokens=disable_bonus_tokens, ))
113 + return SpecDecodeWorker(proposer_worker,
114 + scorer_worker,
115 + disable_by_batch_size=disable_by_batch_size,
116 + rejection_sampler=RejectionSampler(
117 + disable_bonus_tokens=disable_bonus_tokens))
118
119 def __init__(
120 self,
vllm/spec_decode/top1_proposer.py
+18 -27
@@ -148,7 +148,8 @@ class Top1Proposer(SpeculativeProposer):
148 nonzero_proposal_len_indices,
149 )
150
151 - def _remove_no_proposal_seqs(self, proposal_lens, maybe_sampler_output,
151 + @staticmethod
152 + def _remove_no_proposal_seqs(proposal_lens, maybe_sampler_output,
153 nonzero_proposal_len_indices, transposed):
154 """Remove sequences from nonzero_proposal_len_indices and reset
155 their proposal_len to 0 the draft worker does not provide a proposal
@@ -207,7 +208,7 @@ class Top1Proposer(SpeculativeProposer):
208 self,
209 batch_size: int,
210 proposal_len: int,
210 - maybe_sampler_output: Optional[SamplerOutput],
211 + maybe_sampler_output: Optional[List[SamplerOutput]],
212 proposal_lens: List[int],
213 nonzero_proposal_len_indices: List[int],
214 sampler_transposed: bool,
@@ -218,25 +219,19 @@ class Top1Proposer(SpeculativeProposer):
219 if maybe_sampler_output is None:
220 # If no speculative tokens, the sampler output will be None.
221 # In this case we return empty proposals.
221 - proposal_tokens = torch.full(
222 - size=(
223 - batch_size,
224 - proposal_len,
225 - ),
226 - fill_value=-1,
227 - dtype=torch.long,
228 - device=self._device,
229 - )
230 - proposal_probs = torch.zeros(
231 - batch_size,
232 - proposal_len,
233 - self._vocab_size,
234 - dtype=torch.float32,
235 - device=self._device,
236 - )
237 - proposal_lens_tensor = torch.zeros(len(proposal_lens),
238 - dtype=torch.long,
239 - device=self._device)
222 + proposal_tokens = torch.tensor(-1,
223 + dtype=torch.long,
224 + device=self._device).expand(
225 + batch_size, proposal_len)
226 + proposal_probs = torch.tensor(0,
227 + dtype=torch.float32,
228 + device=self._device).expand(
229 + batch_size, proposal_len,
230 + self._vocab_size)
231 + proposal_lens_tensor = torch.tensor(0,
232 + dtype=torch.long,
233 + device=self._device).expand(
234 + len(proposal_lens))
235 return proposal_tokens, proposal_probs, proposal_lens_tensor
236
237 sampler_output = maybe_sampler_output
@@ -246,18 +241,14 @@ class Top1Proposer(SpeculativeProposer):
241 # Now, reformat the output GPU tensors such that each sequence has
242 # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
243
249 - entire_proposal_tokens = torch.full(
244 + entire_proposal_tokens = proposal_tokens.new_full(
245 size=(batch_size, *proposal_tokens.shape[1:]),
246 fill_value=-1,
252 - dtype=torch.long,
253 - device=self._device,
247 )
248 entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
256 - entire_proposal_probs = torch.zeros(
249 + entire_proposal_probs = proposal_probs.new_zeros(
250 batch_size,
251 *proposal_probs.shape[1:],
259 - dtype=torch.float32,
260 - device=self._device,
252 )
253 entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
254
vllm/spec_decode/util.py
+3 -8
@@ -1,12 +1,11 @@
1 from contextlib import contextmanager
2 -from itertools import chain
2 from typing import Dict, List, Tuple
3
4 import torch
5
6 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
7 SamplerOutput, SequenceGroupMetadata,
9 - SequenceGroupOutput, SequenceOutput)
8 + SequenceOutput)
9
10 SeqId = int
11
@@ -16,11 +15,7 @@ def get_all_seq_ids(
15 """Given a list of SequenceGroupMetadata, create a list of all
16 sequence ids.
17 """
19 - return list(
20 - chain.from_iterable([
21 - seq_group_metadata.seq_data.keys()
22 - for seq_group_metadata in seq_group_metadata_list
23 - ]))
18 + return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
19
20
21 def get_all_num_logprobs(
@@ -68,7 +63,7 @@ def create_sequence_group_output(
63 seq_id: SeqId,
64 topk_token_ids: List[int],
65 topk_logprobs: List[float],
71 -) -> SequenceGroupOutput:
66 +) -> CompletionSequenceGroupOutput:
67 """Create a SequenceGroupOutput given the sampling results.
68
69 Args:
vllm/transformers_utils/config.py
+3 -3
@@ -1,4 +1,4 @@
1 -from typing import Dict, Optional
1 +from typing import Dict, Optional, Type
2
3 from transformers import PretrainedConfig
4
@@ -9,7 +9,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
10 logger = init_logger(__name__)
11
12 -_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
12 +_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
13 "chatglm": ChatGLMConfig,
14 "dbrx": DbrxConfig,
15 "mpt": MPTConfig,
@@ -68,4 +68,4 @@ def get_hf_text_config(config: PretrainedConfig):
68 assert hasattr(config.text_config, "num_attention_heads")
69 return config.text_config
70 else:
71 - return config
\ No newline at end of file
71 + return config
vllm/worker/model_runner.py
+15 -15
@@ -527,16 +527,6 @@ class ModelRunner:
527 )
528 assert max_query_len > 0, ("query_lens: {}".format(query_lens))
529
530 - context_lens_tensor = torch.tensor(context_lens,
531 - dtype=torch.int,
532 - device=self.device)
533 - query_lens_tensor = torch.tensor(query_lens,
534 - dtype=torch.long,
535 - device=self.device)
536 - query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
537 - dtype=torch.int32,
538 - device=self.device)
539 -
530 seq_lens_tensor = torch.tensor(seq_lens,
531 dtype=torch.int,
532 device=self.device)
@@ -544,11 +534,6 @@ class ModelRunner:
534 dtype=torch.int32,
535 device=self.device)
536
547 - torch.cumsum(query_lens_tensor,
548 - dim=0,
549 - dtype=query_start_loc.dtype,
550 - out=query_start_loc[1:])
551 -
537 torch.cumsum(seq_lens_tensor,
538 dim=0,
539 dtype=seq_start_loc.dtype,
@@ -601,6 +586,21 @@ class ModelRunner:
586 seq_start_loc=seq_start_loc,
587 data_type=kv_cache_dtype)
588 else:
589 + context_lens_tensor = torch.tensor(context_lens,
590 + dtype=torch.int,
591 + device=self.device)
592 + query_lens_tensor = torch.tensor(query_lens,
593 + dtype=torch.long,
594 + device=self.device)
595 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
596 + dtype=torch.int32,
597 + device=self.device)
598 +
599 + torch.cumsum(query_lens_tensor,
600 + dim=0,
601 + dtype=query_start_loc.dtype,
602 + out=query_start_loc[1:])
603 +
604 attn_metadata = self.attn_backend.make_metadata(
605 num_prefills=num_prefills,
606 slot_mapping=slot_mapping_tensor,