A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware][TPU] Raise errors for unsupported sampling params (#5850)
Woosuk Kwon committed
Jun 25, 2024 at 16:58 UTC
f178e56c68d97e3a29a8a885a09dd61f8d534732
1 file changed
+44
-19
vllm/worker/tpu_model_runner.py
+44
-19
@@ -20,6 +20,8 @@ from vllm.utils import make_tensor_with_pad
20
logger = init_logger(__name__)
21
22
_PAD_SLOT_ID = 0 # FIXME(woosuk)
23
+# FIXME(woosuk): Temporarily disabled top-p sampling since it's too slow.
24
+_ENABLE_TOP_P = False
25
26
27
class TPUModelRunner:
@@ -339,9 +341,34 @@ class TPUModelRunner:
341
assert seq_group_metadata.sampling_params is not None
342
sampling_params = seq_group_metadata.sampling_params
343
344
+ # NOTE(woosuk): Here we mimic argmax sampling by applying a very
345
+ # low temperature. This is not accurate.
346
t.append(sampling_params.temperature
347
if sampling_params.temperature >= 1e-5 else 1e-5)
348
+ if sampling_params.top_p != 1 and not _ENABLE_TOP_P:
349
+ raise NotImplementedError(
350
+ "Top-p sampling is currently disabled for the TPU backend "
351
+ "due to performance issues.")
352
p.append(sampling_params.top_p)
353
+ if sampling_params.top_k != -1:
354
+ raise NotImplementedError(
355
+ "Top-k sampling is currently disabled for the TPU backend "
356
+ "due to performance issues.")
357
+ if sampling_params.best_of > 1:
358
+ raise NotImplementedError(
359
+ "best_of > 1 is not currently supported by the TPU "
360
+ "backend.")
361
+ if sampling_params.use_beam_search:
362
+ raise NotImplementedError(
363
+ "Beam search is not supported by the TPU backend.")
364
+ if sampling_params.logprobs is not None:
365
+ raise NotImplementedError(
366
+ "logprobs is not currently supported by the TPU backend.")
367
+ if sampling_params.prompt_logprobs is not None:
368
+ raise NotImplementedError(
369
+ "prompt_logprobs is not currently supported by the TPU "
370
+ "backend.")
371
+
372
num_paddings = padded_batch_size - len(seq_group_metadata_list)
373
t += [1.0] * num_paddings
374
p += [1.0] * num_paddings
@@ -350,35 +377,32 @@ class TPUModelRunner:
377
p = torch.tensor(p, dtype=torch.float32, device=self.device)
378
return t, p
379
353
- def prepare_inputs(
380
+ def _execute_model(
381
self,
355
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
356
- ):
357
- assert seq_group_metadata_list is not None
382
+ seq_group_metadata_list: List[SequenceGroupMetadata],
383
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
384
+ ) -> List[CompletionSequenceGroupOutput]:
385
+ # Prepare inputs.
386
assert len(seq_group_metadata_list) > 0
387
# NOTE: We assume that all sequences in the group are all prompts or
388
# all decodes.
361
- if seq_group_metadata_list[0].is_prompt:
389
+ is_prompt = seq_group_metadata_list[0].is_prompt
390
+ if is_prompt:
391
inputs = self._prepare_prompt(seq_group_metadata_list)
392
else:
393
inputs = self._prepare_decode(seq_group_metadata_list)
394
padded_batch_size = inputs[0].shape[0]
366
- sample_inputs = self._prepare_sample(seq_group_metadata_list,
367
- padded_batch_size)
368
- return inputs + sample_inputs
395
+ t, p = self._prepare_sample(seq_group_metadata_list, padded_batch_size)
396
370
- def _execute_model(
371
- self,
372
- seq_group_metadata_list: List[SequenceGroupMetadata],
373
- kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
374
- ) -> List[CompletionSequenceGroupOutput]:
375
- inputs = self.prepare_inputs(seq_group_metadata_list)
397
+ # Execute the model.
398
next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
377
- *inputs[2:])
378
- if not self.is_driver_worker:
379
- return []
399
+ *inputs[2:], t, p)
400
+ # Retrieve the outputs to CPU.
401
next_token_ids = next_token_ids.cpu().tolist()
402
403
+ # NOTE(woosuk): Minimal code to construct the sampler outputs.
404
+ # The TPU backend does not reuse the sampler, since the TPU backend
405
+ # does not support the advanced sampling parameters such as logprobs.
406
i = 0
407
sampler_outputs = []
408
for seq_group_metadata in seq_group_metadata_list:
@@ -400,6 +424,7 @@ class TPUModelRunner:
424
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
425
) -> SamplerOutput:
426
assert seq_group_metadata_list is not None
427
+ assert len(seq_group_metadata_list) > 0
428
if seq_group_metadata_list[0].is_prompt:
429
# NOTE(woosuk): To reduce the compilation time, we only compile the
430
# prefill inputs with batch size 1. Because the scheduler is not
@@ -492,8 +517,8 @@ class ModelWrapper(nn.Module):
517
logits = self.model.compute_logits(hidden_states, sampling_metadata)
518
519
logits = logits / t.unsqueeze(dim=1)
495
- # FIXME(woosuk): Disabled top-p sampling since it's too slow.
496
- # logits = _apply_top_p(logits, p.unsqueeze(dim=1))
520
+ if _ENABLE_TOP_P:
521
+ logits = _apply_top_p(logits, p.unsqueeze(dim=1))
522
probs = torch.softmax(logits, dim=-1, dtype=torch.float32)
523
# FIXME(woosuk): best_of > 1 is not supported.
524
next_token_ids = torch.multinomial(probs, num_samples=1).squeeze(dim=1)