@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96f6a7596f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix HPU multiprocessing executor (#12167)

Signed-off-by: Konrad Zawora <kzawora@habana.ai>

Konrad Zawora committed Jan 22, 2025 at 19:07 UTC 96f6a7596fed0a8a8b5a13ce1ca2a7e06b1e5adf
4 files changed +22 -4
vllm/config.py
+1 -1
@@ -1293,7 +1293,7 @@ class ParallelConfig:
1293 raise ValueError(f"worker-use-ray can't be used with "
1294 f"distributed executor backend "
1295 f"'{self.distributed_executor_backend}'.")
1296 - ray_only_devices = ["tpu", "hpu"]
1296 + ray_only_devices = ["tpu"]
1297 from vllm.platforms import current_platform
1298 if (current_platform.device_type in ray_only_devices
1299 and self.world_size > 1):
vllm/engine/arg_utils.py
+1 -1
@@ -397,7 +397,7 @@ class EngineArgs:
397 'or equal to the number of GPUs available, "mp" will be used to '
398 'keep processing on a single host. Otherwise, this will default '
399 'to "ray" if Ray is installed and fail otherwise. Note that tpu '
400 - 'and hpu only support Ray for distributed inference.')
400 + 'only supports Ray for distributed inference.')
401
402 parser.add_argument(
403 '--worker-use-ray',
vllm/platforms/hpu.py
+18
@@ -1,7 +1,9 @@
1 +import os
2 from typing import TYPE_CHECKING, Optional
3
4 import torch
5
6 +from vllm import envs
7 from vllm.logger import init_logger
8
9 from .interface import Platform, PlatformEnum, _Backend
@@ -58,6 +60,22 @@ class HpuPlatform(Platform):
60 cache_config = vllm_config.cache_config
61 if cache_config and cache_config.block_size is None:
62 cache_config.block_size = 128
63 + if (parallel_config.distributed_executor_backend == 'mp'
64 + and envs.VLLM_WORKER_MULTIPROC_METHOD == 'fork'):
65 + if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD",
66 + None) is not None:
67 + logger.warning("On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
68 + "might cause application hangs on exit. Using "
69 + "VLLM_WORKER_MULTIPROC_METHOD=fork anyway, "
70 + "as it was explicitly requested.")
71 + else:
72 + logger.warning(
73 + "On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
74 + "might cause application hangs on exit. Setting "
75 + "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
76 + "To override that behavior, please set "
77 + "VLLM_WORKER_MULTIPROC_METHOD=fork explicitly.")
78 + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
79
80 @classmethod
81 def is_pin_memory_available(cls):
vllm/worker/hpu_worker.py
+2 -2
@@ -130,7 +130,6 @@ class HPUWorker(LocalOrDistributedWorkerBase):
130 self,
131 execute_model_req: Optional[ExecuteModelRequest] = None,
132 ) -> Optional[List[SamplerOutput]]:
133 - assert execute_model_req is not None
133 # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION - will log graph compilations per engine step, only when there was any - highly recommended to use alongside PT_HPU_METRICS_GC_DETAILS! # noqa:E501
134 # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL - will log graph compilations per engine step, always, even if there were none # noqa:E501
135 # VLLM_HPU_LOG_STEP_CPU_FALLBACKS - will log cpu fallbacks per engine step, only when there was any # noqa:E501
@@ -144,7 +143,8 @@ class HPUWorker(LocalOrDistributedWorkerBase):
143 'VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL', '0') != '0'
144 log_cpu_fallbacks = os.environ.get('VLLM_HPU_LOG_STEP_CPU_FALLBACKS',
145 '0') != '0' or log_cpu_fallbacks_all
147 - if log_graph_compilation or log_cpu_fallbacks:
146 + if (log_graph_compilation or log_cpu_fallbacks) and \
147 + execute_model_req is not None:
148 from habana_frameworks.torch.hpu.metrics import metric_localcontext
149 seq_group_metadata_list = execute_model_req.seq_group_metadata_list
150 is_prompt = any([