@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c42590f97a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Hardware] [Intel GPU] refactor xpu worker/executor (#7686)

Kunshang Ji committed Aug 21, 2024 at 00:54 UTC c42590f97a8fd7bcc22137777f031eeee6df8187
3 files changed +26 -28
vllm/executor/xpu_executor.py
+15 -23
@@ -1,16 +1,16 @@
1 -from typing import List, Optional
1 +from typing import List, Optional, Tuple, Union
2
3 import torch
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ParallelConfig, PromptAdapterConfig,
7 - SchedulerConfig, SpeculativeConfig)
6 + ModelConfig, ObservabilityConfig, ParallelConfig,
7 + PromptAdapterConfig, SchedulerConfig,
8 + SpeculativeConfig)
9 from vllm.executor.executor_base import ExecutorAsyncBase
10 from vllm.executor.gpu_executor import GPUExecutor
11 from vllm.logger import init_logger
11 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 +from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
13 from vllm.utils import make_async
13 -from vllm.worker.worker_base import WorkerWrapperBase
14
15 logger = init_logger(__name__)
16
@@ -30,6 +30,7 @@ class XPUExecutor(GPUExecutor):
30 lora_config: Optional[LoRAConfig],
31 prompt_adapter_config: Optional[PromptAdapterConfig],
32 speculative_config: Optional[SpeculativeConfig],
33 + observability_config: Optional[ObservabilityConfig],
34 ) -> None:
35 assert device_config.device_type == "xpu"
36 assert (not speculative_config
@@ -46,32 +47,23 @@ class XPUExecutor(GPUExecutor):
47 self.device_config = device_config
48 self.prompt_adapter_config = prompt_adapter_config
49 self.speculative_config = None
50 + self.observability_config = observability_config
51
52 # Instantiate the worker and load the model to GPU.
53 self._init_executor()
54
53 - def _create_worker(self,
54 - local_rank: int = 0,
55 - rank: int = 0,
56 - distributed_init_method: Optional[str] = None):
57 - if self.speculative_config is None:
58 - worker_module_name = "vllm.worker.xpu_worker"
59 - worker_class_name = "XPUWorker"
60 - else:
55 + def _get_worker_module_and_class(self) -> Tuple[str, str]:
56 + if self.speculative_config is not None:
57 raise NotImplementedError(
58 "XPU does not support speculative decoding")
63 -
64 - wrapper = WorkerWrapperBase(
65 - worker_module_name=worker_module_name,
66 - worker_class_name=worker_class_name,
67 - )
68 - wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
69 - distributed_init_method))
70 - return wrapper.worker
59 + else:
60 + worker_module_name = "vllm.worker.xpu_worker"
61 + worker_class_name = "XPUWorker"
62 + return (worker_module_name, worker_class_name)
63
64 def execute_model(
73 - self,
74 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
65 + self, execute_model_req: ExecuteModelRequest
66 + ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
67 output = self.driver_worker.execute_model(execute_model_req)
68 return output
69
vllm/worker/xpu_model_runner.py
-1
@@ -137,7 +137,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
137 device_config=self.device_config,
138 load_config=self.load_config,
139 lora_config=self.lora_config,
140 - multimodal_config=self.multimodal_config,
140 parallel_config=self.parallel_config,
141 scheduler_config=self.scheduler_config,
142 cache_config=self.cache_config,
vllm/worker/xpu_worker.py
+11 -4
@@ -9,8 +9,8 @@ import torch
9 import torch.distributed
10
11 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 - ModelConfig, MultiModalConfig, ParallelConfig,
13 - PromptAdapterConfig, SchedulerConfig,
12 + ModelConfig, MultiModalConfig, ObservabilityConfig,
13 + ParallelConfig, PromptAdapterConfig, SchedulerConfig,
14 SpeculativeConfig)
15 from vllm.distributed import (ensure_model_parallel_initialized,
16 init_distributed_environment)
@@ -50,6 +50,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
50 speculative_config: Optional[SpeculativeConfig] = None,
51 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52 is_driver_worker: bool = False,
53 + observability_config: Optional[ObservabilityConfig] = None,
54 ) -> None:
55 assert device_config.device_type == "xpu"
56 assert is_xpu()
@@ -67,8 +68,10 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
68 self.lora_config = lora_config
69 self.prompt_adapter_config = prompt_adapter_config
70 self.is_driver_worker = is_driver_worker
70 - if self.is_driver_worker:
71 - assert self.rank == 0, "The driver worker must have rank 0."
71 + self.observability_config = observability_config
72 + if parallel_config and is_driver_worker:
73 + assert rank % parallel_config.tensor_parallel_size == 0, \
74 + "Driver worker should be rank 0 of tensor parallel group."
75
76 self.multimodal_config = multimodal_config
77
@@ -183,7 +186,11 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
186 # dependency (libdrm and drm headers) on your system.
187 ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE",
188 "sockets")
189 + ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE",
190 + str(parallel_config.world_size))
191 os.environ['CCL_ZE_IPC_EXCHANGE'] = ENV_CCL_ZE_IPC_EXCHANGE
192 + os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
193 + os.environ["LOCAL_RANK"] = str(self.local_rank)
194 init_distributed_environment(
195 world_size=parallel_config.world_size,
196 rank=rank,