A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[TPU] Async output processing for TPU (#8011)
Woosuk Kwon committed
Aug 29, 2024 at 19:35 UTC
80c7b089b1189c5e2f40b3be250a118e9349a024
2 files changed
+10
-4
vllm/config.py
+3
-3
@@ -347,10 +347,10 @@ class ModelConfig:
347
self.use_async_output_proc = False
348
return
349
350
- if device_config.device_type != "cuda":
350
+ if device_config.device_type not in ("cuda", "tpu"):
351
logger.warning(
352
- "Async output processing is only supported for CUDA."
353
- " Disabling it for other platforms.")
352
+ "Async output processing is only supported for CUDA or TPU. "
353
+ "Disabling it for other platforms.")
354
self.use_async_output_proc = False
355
return
356
vllm/worker/tpu_model_runner.py
+7
-1
@@ -1,6 +1,7 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
3
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
4
+ Type, Union)
5
from unittest.mock import patch
6
7
import numpy as np
@@ -51,6 +52,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
52
best_of: List[int]
53
seq_groups: List[List[int]]
54
virtual_engine: int = 0
55
+ async_callback: Optional[Callable] = None
56
57
def as_broadcastable_tensor_dict(
58
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -562,6 +564,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
564
model_input.attn_metadata, model_input.input_lens[i:i + 1],
565
model_input.t[i:i + 1], model_input.p[i:i + 1],
566
model_input.num_samples, kv_caches)
567
+ if i == 0 and model_input.async_callback is not None:
568
+ model_input.async_callback()
569
# Retrieve the outputs to CPU.
570
next_token_ids += output_token_ids.cpu().tolist()
571
start_idx = end_idx
@@ -572,6 +576,8 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
576
model_input.attn_metadata, model_input.input_lens,
577
model_input.t, model_input.p, model_input.num_samples,
578
kv_caches)
579
+ if model_input.async_callback is not None:
580
+ model_input.async_callback()
581
# Retrieve the outputs to CPU.
582
next_token_ids = output_token_ids.cpu().tolist()
583