@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 076169f603
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Hardware][Intel GPU] Add intel GPU pipeline parallel support. (#7810)

Kunshang Ji committed Aug 28, 2024 at 01:07 UTC 076169f603a44b3a3377e59bad62d1cfc62cf98a
6 files changed +82 -19
vllm/engine/async_llm_engine.py
+5
@@ -666,6 +666,11 @@ class AsyncLLMEngine:
666 initialize_ray_cluster(engine_config.parallel_config)
667 from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
668 executor_class = RayXPUExecutorAsync
669 + elif distributed_executor_backend == "mp":
670 + initialize_ray_cluster(engine_config.parallel_config)
671 + from vllm.executor.multiproc_xpu_executor import (
672 + MultiprocessingXPUExecutorAsync)
673 + executor_class = MultiprocessingXPUExecutorAsync
674 else:
675 raise RuntimeError(
676 "Not supported distributed execution model on XPU device.")
vllm/engine/llm_engine.py
+7
@@ -472,6 +472,13 @@ class LLMEngine:
472 initialize_ray_cluster(engine_config.parallel_config)
473 from vllm.executor.ray_xpu_executor import RayXPUExecutor
474 executor_class = RayXPUExecutor
475 + elif distributed_executor_backend == "mp":
476 + # FIXME(kunshang):
477 + # spawn needs calling `if __name__ == '__main__':``
478 + # fork is not supported for xpu start new process.
479 + logger.error(
480 + "Both start methods (spawn and fork) have issue "
481 + "on XPU if you use mp backend, Please try ray instead.")
482 else:
483 from vllm.executor.xpu_executor import XPUExecutor
484 executor_class = XPUExecutor
vllm/executor/multiproc_gpu_executor.py
+22 -16
@@ -30,16 +30,12 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
30 uses_ray: bool = False
31
32 def _init_executor(self) -> None:
33 + self._check_executor_parameters()
34 +
35 # Create the parallel GPU workers.
36 world_size = self.parallel_config.world_size
37 tensor_parallel_size = self.parallel_config.tensor_parallel_size
38
37 - # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
38 - if "CUDA_VISIBLE_DEVICES" not in os.environ:
39 - update_environment_variables({
40 - "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
41 - })
42 -
39 # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
40 os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
41
@@ -68,16 +64,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
64 if world_size > 1:
65 maybe_set_triton_cache_manager()
66
71 - cuda_device_count = cuda_device_count_stateless()
72 - # Use confusing message for more common TP-only case.
73 - assert tensor_parallel_size <= cuda_device_count, (
74 - f"please set tensor_parallel_size ({tensor_parallel_size}) "
75 - f"to less than max local gpu count ({cuda_device_count})")
76 -
77 - assert world_size <= cuda_device_count, (
78 - f"please ensure that world_size ({world_size}) "
79 - f"is less than than max local gpu count ({cuda_device_count})")
80 -
67 # Multiprocessing-based executor does not support multi-node setting.
68 # Since it only works for single node, we can use the loopback address
69 # 127.0.0.1 for communication.
@@ -139,6 +125,26 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
125 max_concurrent_workers=self.parallel_config.
126 max_parallel_loading_workers)
127
128 + def _check_executor_parameters(self):
129 + world_size = self.parallel_config.tensor_parallel_size
130 + tensor_parallel_size = self.parallel_config.tensor_parallel_size
131 +
132 + # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
133 + if "CUDA_VISIBLE_DEVICES" not in os.environ:
134 + update_environment_variables({
135 + "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
136 + })
137 +
138 + cuda_device_count = cuda_device_count_stateless()
139 + # Use confusing message for more common TP-only case.
140 + assert tensor_parallel_size <= cuda_device_count, (
141 + f"please set tensor_parallel_size ({tensor_parallel_size}) "
142 + f"to less than max local gpu count ({cuda_device_count})")
143 +
144 + assert world_size <= cuda_device_count, (
145 + f"please ensure that world_size ({world_size}) "
146 + f"is less than than max local gpu count ({cuda_device_count})")
147 +
148 def shutdown(self):
149 if (worker_monitor := getattr(self, "worker_monitor",
150 None)) is not None:
vllm/executor/multiproc_xpu_executor.py new
+26
@@ -0,0 +1,26 @@
1 +import vllm.envs as envs
2 +from vllm.executor.multiproc_gpu_executor import (
3 + MultiprocessingGPUExecutor, MultiprocessingGPUExecutorAsync)
4 +from vllm.executor.xpu_executor import XPUExecutor
5 +from vllm.logger import init_logger
6 +from vllm.utils import make_async
7 +
8 +logger = init_logger(__name__)
9 +
10 +
11 +class MultiprocessingXPUExecutor(MultiprocessingGPUExecutor, XPUExecutor):
12 + """Python multiprocessing-based multi-XPU executor"""
13 +
14 + def _check_executor_parameters(self):
15 + mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
16 + if mp_method != "spawn":
17 + raise RuntimeError(
18 + "XPU multiprocess executor only support spawn as mp method")
19 +
20 +
21 +class MultiprocessingXPUExecutorAsync(MultiprocessingXPUExecutor,
22 + MultiprocessingGPUExecutorAsync):
23 +
24 + def __init__(self, *args, **kwargs):
25 + super().__init__(*args, **kwargs)
26 + self.driver_exec_model = make_async(self.driver_worker.execute_model)
vllm/worker/xpu_model_runner.py
+16 -3
@@ -12,6 +12,7 @@ from vllm.attention import get_attn_backend
12 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 ModelConfig, ObservabilityConfig, ParallelConfig,
14 PromptAdapterConfig, SchedulerConfig)
15 +from vllm.distributed import get_pp_group
16 from vllm.inputs import INPUT_REGISTRY, InputRegistry
17 from vllm.logger import init_logger
18 from vllm.model_executor.model_loader import get_model
@@ -439,9 +440,11 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
440 "Setting it to the minimum value of 1.", expr)
441 max_num_seqs = 1
442
443 + batch_size = 0
444 for group_id in range(max_num_seqs):
445 seq_len = (max_num_batched_tokens // max_num_seqs +
446 (group_id < max_num_batched_tokens % max_num_seqs))
447 + batch_size += seq_len
448
449 seq_data, dummy_multi_modal_data = self.input_registry \
450 .dummy_data_for_profiling(self.model_config,
@@ -465,7 +468,13 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
468 finished_requests_ids = [seq.request_id for seq in seqs]
469 model_input = self.prepare_model_input(
470 seqs, finished_requests_ids=finished_requests_ids)
468 - self.execute_model(model_input, kv_caches)
471 + intermediate_tensors = None
472 + if not get_pp_group().is_first_rank:
473 + intermediate_tensors = self.model.make_empty_intermediate_tensors(
474 + batch_size=batch_size,
475 + dtype=self.model_config.dtype,
476 + device=self.device)
477 + self.execute_model(model_input, kv_caches, intermediate_tensors)
478 torch.xpu.synchronize()
479 return
480
@@ -537,7 +546,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
546 and self.observability_config.collect_model_forward_time):
547 model_forward_start_time = time.time()
548
540 - hidden_states = model_executable(
549 + hidden_or_intermediate_states = model_executable(
550 input_ids=model_input.input_tokens,
551 positions=model_input.input_positions,
552 kv_caches=kv_caches,
@@ -545,12 +554,16 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
554 intermediate_tensors=intermediate_tensors,
555 **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
556 device=self.device))
557 + # Compute the logits in the last pipeline stage.
558 + if not get_pp_group().is_last_rank:
559 + return hidden_or_intermediate_states
560 +
561 if (self.observability_config is not None
562 and self.observability_config.collect_model_forward_time):
563 model_forward_end_time = time.time()
564
565 # Compute the logits.
553 - logits = self.model.compute_logits(hidden_states,
566 + logits = self.model.compute_logits(hidden_or_intermediate_states,
567 model_input.sampling_metadata)
568
569 # Only perform sampling in the driver worker.
vllm/worker/xpu_worker.py
+6
@@ -14,6 +14,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14 SpeculativeConfig)
15 from vllm.distributed import (ensure_model_parallel_initialized,
16 init_distributed_environment)
17 +from vllm.distributed.parallel_state import get_pp_group
18 from vllm.logger import init_logger
19 from vllm.model_executor import set_random_seed
20 from vllm.utils import is_xpu
@@ -198,3 +199,8 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
199 ensure_model_parallel_initialized(
200 parallel_config.tensor_parallel_size,
201 parallel_config.pipeline_parallel_size)
202 +
203 + if parallel_config.pipeline_parallel_size > 1:
204 + # torch-ccl xpu need a collective API warm up
205 + # before calling send/recv API
206 + get_pp_group().all_reduce(torch.zeros(1).xpu())