@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4141608c6a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Hardware][intel GPU] add async output process for xpu (#8897)

Kunshang Ji committed Oct 15, 2024 at 02:23 UTC 4141608c6a636952242b86e50d8f90ca674b7425
2 files changed +8 -4
vllm/config.py
+2 -2
@@ -361,9 +361,9 @@ class ModelConfig:
361
362 # Reminder: Please update docs/source/serving/compatibility_matrix.rst
363 # If the feature combo become valid
364 - if device_config.device_type not in ("cuda", "tpu"):
364 + if device_config.device_type not in ("cuda", "tpu", "xpu"):
365 logger.warning(
366 - "Async output processing is only supported for CUDA or TPU. "
366 + "Async output processing is only supported for CUDA, TPU, XPU. "
367 "Disabling it for other platforms.")
368 self.use_async_output_proc = False
369 return
vllm/worker/xpu_model_runner.py
+6 -2
@@ -2,8 +2,8 @@ import dataclasses
2 import time
3 import weakref
4 from dataclasses import dataclass
5 -from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6 - TypeVar)
5 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
6 + Type, TypeVar)
7
8 import torch
9 import torch.nn as nn
@@ -57,6 +57,7 @@ class ModelInputForXPU(ModelRunnerInputBase):
57 virtual_engine: Optional[int] = None
58 seq_lens: Optional[List[int]] = None
59 query_lens: Optional[List[int]] = None
60 + async_callback: Optional[Callable] = None
61
62 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
63 tensor_dict = {
@@ -582,6 +583,9 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
583 if not self.is_driver_worker:
584 return []
585
586 + if model_input.async_callback is not None:
587 + model_input.async_callback()
588 +
589 # Sample the next token.
590 output: SamplerOutput = self.model.sample(
591 logits=logits,