A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Hardware][Intel GPU] Add intel GPU pipeline parallel support. (#7810)
Kunshang Ji committed
Aug 28, 2024 at 01:07 UTC
076169f603a44b3a3377e59bad62d1cfc62cf98a
6 files changed
+82
-19
vllm/engine/async_llm_engine.py
+5
@@ -666,6 +666,11 @@ class AsyncLLMEngine:
666
initialize_ray_cluster(engine_config.parallel_config)
667
from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
668
executor_class = RayXPUExecutorAsync
669
+ elif distributed_executor_backend == "mp":
670
+ initialize_ray_cluster(engine_config.parallel_config)
671
+ from vllm.executor.multiproc_xpu_executor import (
672
+ MultiprocessingXPUExecutorAsync)
673
+ executor_class = MultiprocessingXPUExecutorAsync
674
else:
675
raise RuntimeError(
676
"Not supported distributed execution model on XPU device.")
vllm/engine/llm_engine.py
+7
@@ -472,6 +472,13 @@ class LLMEngine:
472
initialize_ray_cluster(engine_config.parallel_config)
473
from vllm.executor.ray_xpu_executor import RayXPUExecutor
474
executor_class = RayXPUExecutor
475
+ elif distributed_executor_backend == "mp":
476
+ # FIXME(kunshang):
477
+ # spawn needs calling `if __name__ == '__main__':``
478
+ # fork is not supported for xpu start new process.
479
+ logger.error(
480
+ "Both start methods (spawn and fork) have issue "
481
+ "on XPU if you use mp backend, Please try ray instead.")
482
else:
483
from vllm.executor.xpu_executor import XPUExecutor
484
executor_class = XPUExecutor
vllm/executor/multiproc_gpu_executor.py
+22
-16
@@ -30,16 +30,12 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
30
uses_ray: bool = False
31
32
def _init_executor(self) -> None:
33
+ self._check_executor_parameters()
34
+
35
# Create the parallel GPU workers.
36
world_size = self.parallel_config.world_size
37
tensor_parallel_size = self.parallel_config.tensor_parallel_size
38
37
- # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
38
- if "CUDA_VISIBLE_DEVICES" not in os.environ:
39
- update_environment_variables({
40
- "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
41
- })
42
-
39
# Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
40
os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
41
@@ -68,16 +64,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
64
if world_size > 1:
65
maybe_set_triton_cache_manager()
66
71
- cuda_device_count = cuda_device_count_stateless()
72
- # Use confusing message for more common TP-only case.
73
- assert tensor_parallel_size <= cuda_device_count, (
74
- f"please set tensor_parallel_size ({tensor_parallel_size}) "
75
- f"to less than max local gpu count ({cuda_device_count})")
76
-
77
- assert world_size <= cuda_device_count, (
78
- f"please ensure that world_size ({world_size}) "
79
- f"is less than than max local gpu count ({cuda_device_count})")
80
-
67
# Multiprocessing-based executor does not support multi-node setting.
68
# Since it only works for single node, we can use the loopback address
69
# 127.0.0.1 for communication.
@@ -139,6 +125,26 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
125
max_concurrent_workers=self.parallel_config.
126
max_parallel_loading_workers)
127
128
+ def _check_executor_parameters(self):
129
+ world_size = self.parallel_config.tensor_parallel_size
130
+ tensor_parallel_size = self.parallel_config.tensor_parallel_size
131
+
132
+ # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
133
+ if "CUDA_VISIBLE_DEVICES" not in os.environ:
134
+ update_environment_variables({
135
+ "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
136
+ })
137
+
138
+ cuda_device_count = cuda_device_count_stateless()
139
+ # Use confusing message for more common TP-only case.
140
+ assert tensor_parallel_size <= cuda_device_count, (
141
+ f"please set tensor_parallel_size ({tensor_parallel_size}) "
142
+ f"to less than max local gpu count ({cuda_device_count})")
143
+
144
+ assert world_size <= cuda_device_count, (
145
+ f"please ensure that world_size ({world_size}) "
146
+ f"is less than than max local gpu count ({cuda_device_count})")
147
+
148
def shutdown(self):
149
if (worker_monitor := getattr(self, "worker_monitor",
150
None)) is not None:
vllm/executor/multiproc_xpu_executor.py
new
+26
@@ -0,0 +1,26 @@
1
+import vllm.envs as envs
2
+from vllm.executor.multiproc_gpu_executor import (
3
+ MultiprocessingGPUExecutor, MultiprocessingGPUExecutorAsync)
4
+from vllm.executor.xpu_executor import XPUExecutor
5
+from vllm.logger import init_logger
6
+from vllm.utils import make_async
7
+
8
+logger = init_logger(__name__)
9
+
10
+
11
+class MultiprocessingXPUExecutor(MultiprocessingGPUExecutor, XPUExecutor):
12
+ """Python multiprocessing-based multi-XPU executor"""
13
+
14
+ def _check_executor_parameters(self):
15
+ mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
16
+ if mp_method != "spawn":
17
+ raise RuntimeError(
18
+ "XPU multiprocess executor only support spawn as mp method")
19
+
20
+
21
+class MultiprocessingXPUExecutorAsync(MultiprocessingXPUExecutor,
22
+ MultiprocessingGPUExecutorAsync):
23
+
24
+ def __init__(self, *args, **kwargs):
25
+ super().__init__(*args, **kwargs)
26
+ self.driver_exec_model = make_async(self.driver_worker.execute_model)
vllm/worker/xpu_model_runner.py
+16
-3
@@ -12,6 +12,7 @@ from vllm.attention import get_attn_backend
12
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
ModelConfig, ObservabilityConfig, ParallelConfig,
14
PromptAdapterConfig, SchedulerConfig)
15
+from vllm.distributed import get_pp_group
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry
17
from vllm.logger import init_logger
18
from vllm.model_executor.model_loader import get_model
@@ -439,9 +440,11 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
440
"Setting it to the minimum value of 1.", expr)
441
max_num_seqs = 1
442
443
+ batch_size = 0
444
for group_id in range(max_num_seqs):
445
seq_len = (max_num_batched_tokens // max_num_seqs +
446
(group_id < max_num_batched_tokens % max_num_seqs))
447
+ batch_size += seq_len
448
449
seq_data, dummy_multi_modal_data = self.input_registry \
450
.dummy_data_for_profiling(self.model_config,
@@ -465,7 +468,13 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
468
finished_requests_ids = [seq.request_id for seq in seqs]
469
model_input = self.prepare_model_input(
470
seqs, finished_requests_ids=finished_requests_ids)
468
- self.execute_model(model_input, kv_caches)
471
+ intermediate_tensors = None
472
+ if not get_pp_group().is_first_rank:
473
+ intermediate_tensors = self.model.make_empty_intermediate_tensors(
474
+ batch_size=batch_size,
475
+ dtype=self.model_config.dtype,
476
+ device=self.device)
477
+ self.execute_model(model_input, kv_caches, intermediate_tensors)
478
torch.xpu.synchronize()
479
return
480
@@ -537,7 +546,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
546
and self.observability_config.collect_model_forward_time):
547
model_forward_start_time = time.time()
548
540
- hidden_states = model_executable(
549
+ hidden_or_intermediate_states = model_executable(
550
input_ids=model_input.input_tokens,
551
positions=model_input.input_positions,
552
kv_caches=kv_caches,
@@ -545,12 +554,16 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
554
intermediate_tensors=intermediate_tensors,
555
**MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
556
device=self.device))
557
+ # Compute the logits in the last pipeline stage.
558
+ if not get_pp_group().is_last_rank:
559
+ return hidden_or_intermediate_states
560
+
561
if (self.observability_config is not None
562
and self.observability_config.collect_model_forward_time):
563
model_forward_end_time = time.time()
564
565
# Compute the logits.
553
- logits = self.model.compute_logits(hidden_states,
566
+ logits = self.model.compute_logits(hidden_or_intermediate_states,
567
model_input.sampling_metadata)
568
569
# Only perform sampling in the driver worker.
vllm/worker/xpu_worker.py
+6
@@ -14,6 +14,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14
SpeculativeConfig)
15
from vllm.distributed import (ensure_model_parallel_initialized,
16
init_distributed_environment)
17
+from vllm.distributed.parallel_state import get_pp_group
18
from vllm.logger import init_logger
19
from vllm.model_executor import set_random_seed
20
from vllm.utils import is_xpu
@@ -198,3 +199,8 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
199
ensure_model_parallel_initialized(
200
parallel_config.tensor_parallel_size,
201
parallel_config.pipeline_parallel_size)
202
+
203
+ if parallel_config.pipeline_parallel_size > 1:
204
+ # torch-ccl xpu need a collective API warm up
205
+ # before calling send/recv API
206
+ get_pp_group().all_reduce(torch.zeros(1).xpu())