@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80c7b089b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[TPU] Async output processing for TPU (#8011)

Woosuk Kwon committed Aug 29, 2024 at 19:35 UTC 80c7b089b1189c5e2f40b3be250a118e9349a024
2 files changed +10 -4
vllm/config.py
+3 -3
@@ -347,10 +347,10 @@ class ModelConfig:
347 self.use_async_output_proc = False
348 return
349
350 - if device_config.device_type != "cuda":
350 + if device_config.device_type not in ("cuda", "tpu"):
351 logger.warning(
352 - "Async output processing is only supported for CUDA."
353 - " Disabling it for other platforms.")
352 + "Async output processing is only supported for CUDA or TPU. "
353 + "Disabling it for other platforms.")
354 self.use_async_output_proc = False
355 return
356
vllm/worker/tpu_model_runner.py
+7 -1
@@ -1,6 +1,7 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
4 + Type, Union)
5 from unittest.mock import patch
6
7 import numpy as np
@@ -51,6 +52,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
52 best_of: List[int]
53 seq_groups: List[List[int]]
54 virtual_engine: int = 0
55 + async_callback: Optional[Callable] = None
56
57 def as_broadcastable_tensor_dict(
58 self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -562,6 +564,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
564 model_input.attn_metadata, model_input.input_lens[i:i + 1],
565 model_input.t[i:i + 1], model_input.p[i:i + 1],
566 model_input.num_samples, kv_caches)
567 + if i == 0 and model_input.async_callback is not None:
568 + model_input.async_callback()
569 # Retrieve the outputs to CPU.
570 next_token_ids += output_token_ids.cpu().tolist()
571 start_idx = end_idx
@@ -572,6 +576,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
576 model_input.attn_metadata, model_input.input_lens,
577 model_input.t, model_input.p, model_input.num_samples,
578 kv_caches)
579 + if model_input.async_callback is not None:
580 + model_input.async_callback()
581 # Retrieve the outputs to CPU.
582 next_token_ids = output_token_ids.cpu().tolist()
583