@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc6e42a9b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[hardware][rocm] allow rocm to override default env var (#7926)

youkaichao committed Aug 27, 2024 at 19:50 UTC bc6e42a9b19364e07da9f279edd81796541d147d
2 files changed +14 -2
vllm/core/scheduler.py
+3 -2
@@ -1088,8 +1088,9 @@ class Scheduler:
1088 )
1089
1090 def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1091 - no_beam_search = (seq_group.sampling_params.best_of == 1
1092 - and not seq_group.sampling_params.use_beam_search)
1091 + no_beam_search = seq_group.sampling_params is None or (
1092 + seq_group.sampling_params.best_of == 1
1093 + and not seq_group.sampling_params.use_beam_search)
1094
1095 return no_beam_search
1096
vllm/platforms/rocm.py
+11
@@ -1,10 +1,21 @@
1 +import os
2 from functools import lru_cache
3 from typing import Tuple
4
5 import torch
6
7 +from vllm.logger import init_logger
8 +
9 from .interface import Platform, PlatformEnum
10
11 +logger = init_logger(__name__)
12 +
13 +if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
14 + logger.warning("`fork` method is not supported by ROCm. "
15 + "VLLM_WORKER_MULTIPROC_METHOD is overridden to"
16 + " `spawn` instead.")
17 + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
18 +
19
20 class RocmPlatform(Platform):
21 _enum = PlatformEnum.ROCM