A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Hardware] [Intel GPU] refactor xpu worker/executor (#7686)
Kunshang Ji committed
Aug 21, 2024 at 00:54 UTC
c42590f97a8fd7bcc22137777f031eeee6df8187
3 files changed
+26
-28
vllm/executor/xpu_executor.py
+15
-23
@@ -1,16 +1,16 @@
1
-from typing import List, Optional
1
+from typing import List, Optional, Tuple, Union
2
3
import torch
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ParallelConfig, PromptAdapterConfig,
7
- SchedulerConfig, SpeculativeConfig)
6
+ ModelConfig, ObservabilityConfig, ParallelConfig,
7
+ PromptAdapterConfig, SchedulerConfig,
8
+ SpeculativeConfig)
9
from vllm.executor.executor_base import ExecutorAsyncBase
10
from vllm.executor.gpu_executor import GPUExecutor
11
from vllm.logger import init_logger
11
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
+from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
13
from vllm.utils import make_async
13
-from vllm.worker.worker_base import WorkerWrapperBase
14
15
logger = init_logger(__name__)
16
@@ -30,6 +30,7 @@ class XPUExecutor(GPUExecutor):
30
lora_config: Optional[LoRAConfig],
31
prompt_adapter_config: Optional[PromptAdapterConfig],
32
speculative_config: Optional[SpeculativeConfig],
33
+ observability_config: Optional[ObservabilityConfig],
34
) -> None:
35
assert device_config.device_type == "xpu"
36
assert (not speculative_config
@@ -46,32 +47,23 @@ class XPUExecutor(GPUExecutor):
47
self.device_config = device_config
48
self.prompt_adapter_config = prompt_adapter_config
49
self.speculative_config = None
50
+ self.observability_config = observability_config
51
52
# Instantiate the worker and load the model to GPU.
53
self._init_executor()
54
53
- def _create_worker(self,
54
- local_rank: int = 0,
55
- rank: int = 0,
56
- distributed_init_method: Optional[str] = None):
57
- if self.speculative_config is None:
58
- worker_module_name = "vllm.worker.xpu_worker"
59
- worker_class_name = "XPUWorker"
60
- else:
55
+ def _get_worker_module_and_class(self) -> Tuple[str, str]:
56
+ if self.speculative_config is not None:
57
raise NotImplementedError(
58
"XPU does not support speculative decoding")
63
-
64
- wrapper = WorkerWrapperBase(
65
- worker_module_name=worker_module_name,
66
- worker_class_name=worker_class_name,
67
- )
68
- wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
69
- distributed_init_method))
70
- return wrapper.worker
59
+ else:
60
+ worker_module_name = "vllm.worker.xpu_worker"
61
+ worker_class_name = "XPUWorker"
62
+ return (worker_module_name, worker_class_name)
63
64
def execute_model(
73
- self,
74
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
65
+ self, execute_model_req: ExecuteModelRequest
66
+ ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
67
output = self.driver_worker.execute_model(execute_model_req)
68
return output
69
vllm/worker/xpu_model_runner.py
-1
@@ -137,7 +137,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
137
device_config=self.device_config,
138
load_config=self.load_config,
139
lora_config=self.lora_config,
140
- multimodal_config=self.multimodal_config,
140
parallel_config=self.parallel_config,
141
scheduler_config=self.scheduler_config,
142
cache_config=self.cache_config,
vllm/worker/xpu_worker.py
+11
-4
@@ -9,8 +9,8 @@ import torch
9
import torch.distributed
10
11
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
- ModelConfig, MultiModalConfig, ParallelConfig,
13
- PromptAdapterConfig, SchedulerConfig,
12
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
13
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig,
14
SpeculativeConfig)
15
from vllm.distributed import (ensure_model_parallel_initialized,
16
init_distributed_environment)
@@ -50,6 +50,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
50
speculative_config: Optional[SpeculativeConfig] = None,
51
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52
is_driver_worker: bool = False,
53
+ observability_config: Optional[ObservabilityConfig] = None,
54
) -> None:
55
assert device_config.device_type == "xpu"
56
assert is_xpu()
@@ -67,8 +68,10 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
68
self.lora_config = lora_config
69
self.prompt_adapter_config = prompt_adapter_config
70
self.is_driver_worker = is_driver_worker
70
- if self.is_driver_worker:
71
- assert self.rank == 0, "The driver worker must have rank 0."
71
+ self.observability_config = observability_config
72
+ if parallel_config and is_driver_worker:
73
+ assert rank % parallel_config.tensor_parallel_size == 0, \
74
+ "Driver worker should be rank 0 of tensor parallel group."
75
76
self.multimodal_config = multimodal_config
77
@@ -183,7 +186,11 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
186
# dependency (libdrm and drm headers) on your system.
187
ENV_CCL_ZE_IPC_EXCHANGE = os.getenv("CCL_ZE_IPC_EXCHANGE",
188
"sockets")
189
+ ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE",
190
+ str(parallel_config.world_size))
191
os.environ['CCL_ZE_IPC_EXCHANGE'] = ENV_CCL_ZE_IPC_EXCHANGE
192
+ os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
193
+ os.environ["LOCAL_RANK"] = str(self.local_rank)
194
init_distributed_environment(
195
world_size=parallel_config.world_size,
196
rank=rank,