A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Hardware][intel GPU] add async output process for xpu (#8897)
Kunshang Ji committed
Oct 15, 2024 at 02:23 UTC
4141608c6a636952242b86e50d8f90ca674b7425
2 files changed
+8
-4
vllm/config.py
+2
-2
@@ -361,9 +361,9 @@ class ModelConfig:
361
362
# Reminder: Please update docs/source/serving/compatibility_matrix.rst
363
# If the feature combo become valid
364
- if device_config.device_type not in ("cuda", "tpu"):
364
+ if device_config.device_type not in ("cuda", "tpu", "xpu"):
365
logger.warning(
366
- "Async output processing is only supported for CUDA or TPU. "
366
+ "Async output processing is only supported for CUDA, TPU, XPU. "
367
"Disabling it for other platforms.")
368
self.use_async_output_proc = False
369
return
vllm/worker/xpu_model_runner.py
+6
-2
@@ -2,8 +2,8 @@ import dataclasses
2
import time
3
import weakref
4
from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6
- TypeVar)
5
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
6
+ Type, TypeVar)
7
8
import torch
9
import torch.nn as nn
@@ -57,6 +57,7 @@ class ModelInputForXPU(ModelRunnerInputBase):
57
virtual_engine: Optional[int] = None
58
seq_lens: Optional[List[int]] = None
59
query_lens: Optional[List[int]] = None
60
+ async_callback: Optional[Callable] = None
61
62
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
63
tensor_dict = {
@@ -582,6 +583,9 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
583
if not self.is_driver_worker:
584
return []
585
586
+ if model_input.async_callback is not None:
587
+ model_input.async_callback()
588
+
589
# Sample the next token.
590
output: SamplerOutput = self.model.sample(
591
logits=logits,