A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix HPU multiprocessing executor (#12167)
Signed-off-by: Konrad Zawora <kzawora@habana.ai>
Konrad Zawora committed
Jan 22, 2025 at 19:07 UTC
96f6a7596fed0a8a8b5a13ce1ca2a7e06b1e5adf
4 files changed
+22
-4
vllm/config.py
+1
-1
@@ -1293,7 +1293,7 @@ class ParallelConfig:
1293
raise ValueError(f"worker-use-ray can't be used with "
1294
f"distributed executor backend "
1295
f"'{self.distributed_executor_backend}'.")
1296
- ray_only_devices = ["tpu", "hpu"]
1296
+ ray_only_devices = ["tpu"]
1297
from vllm.platforms import current_platform
1298
if (current_platform.device_type in ray_only_devices
1299
and self.world_size > 1):
vllm/engine/arg_utils.py
+1
-1
@@ -397,7 +397,7 @@ class EngineArgs:
397
'or equal to the number of GPUs available, "mp" will be used to '
398
'keep processing on a single host. Otherwise, this will default '
399
'to "ray" if Ray is installed and fail otherwise. Note that tpu '
400
- 'and hpu only support Ray for distributed inference.')
400
+ 'only supports Ray for distributed inference.')
401
402
parser.add_argument(
403
'--worker-use-ray',
vllm/platforms/hpu.py
+18
@@ -1,7 +1,9 @@
1
+import os
2
from typing import TYPE_CHECKING, Optional
3
4
import torch
5
6
+from vllm import envs
7
from vllm.logger import init_logger
8
9
from .interface import Platform, PlatformEnum, _Backend
@@ -58,6 +60,22 @@ class HpuPlatform(Platform):
60
cache_config = vllm_config.cache_config
61
if cache_config and cache_config.block_size is None:
62
cache_config.block_size = 128
63
+ if (parallel_config.distributed_executor_backend == 'mp'
64
+ and envs.VLLM_WORKER_MULTIPROC_METHOD == 'fork'):
65
+ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD",
66
+ None) is not None:
67
+ logger.warning("On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
68
+ "might cause application hangs on exit. Using "
69
+ "VLLM_WORKER_MULTIPROC_METHOD=fork anyway, "
70
+ "as it was explicitly requested.")
71
+ else:
72
+ logger.warning(
73
+ "On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
74
+ "might cause application hangs on exit. Setting "
75
+ "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
76
+ "To override that behavior, please set "
77
+ "VLLM_WORKER_MULTIPROC_METHOD=fork explicitly.")
78
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
79
80
@classmethod
81
def is_pin_memory_available(cls):
vllm/worker/hpu_worker.py
+2
-2
@@ -130,7 +130,6 @@ class HPUWorker(LocalOrDistributedWorkerBase):
130
self,
131
execute_model_req: Optional[ExecuteModelRequest] = None,
132
) -> Optional[List[SamplerOutput]]:
133
- assert execute_model_req is not None
133
# VLLM_HPU_LOG_STEP_GRAPH_COMPILATION - will log graph compilations per engine step, only when there was any - highly recommended to use alongside PT_HPU_METRICS_GC_DETAILS! # noqa:E501
134
# VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL - will log graph compilations per engine step, always, even if there were none # noqa:E501
135
# VLLM_HPU_LOG_STEP_CPU_FALLBACKS - will log cpu fallbacks per engine step, only when there was any # noqa:E501
@@ -144,7 +143,8 @@ class HPUWorker(LocalOrDistributedWorkerBase):
143
'VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL', '0') != '0'
144
log_cpu_fallbacks = os.environ.get('VLLM_HPU_LOG_STEP_CPU_FALLBACKS',
145
'0') != '0' or log_cpu_fallbacks_all
147
- if log_graph_compilation or log_cpu_fallbacks:
146
+ if (log_graph_compilation or log_cpu_fallbacks) and \
147
+ execute_model_req is not None:
148
from habana_frameworks.torch.hpu.metrics import metric_localcontext
149
seq_group_metadata_list = execute_model_req.seq_group_metadata_list
150
is_prompt = any([