A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[hardware][rocm] allow rocm to override default env var (#7926)
youkaichao committed
Aug 27, 2024 at 19:50 UTC
bc6e42a9b19364e07da9f279edd81796541d147d
2 files changed
+14
-2
vllm/core/scheduler.py
+3
-2
@@ -1088,8 +1088,9 @@ class Scheduler:
1088
)
1089
1090
def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1091
- no_beam_search = (seq_group.sampling_params.best_of == 1
1092
- and not seq_group.sampling_params.use_beam_search)
1091
+ no_beam_search = seq_group.sampling_params is None or (
1092
+ seq_group.sampling_params.best_of == 1
1093
+ and not seq_group.sampling_params.use_beam_search)
1094
1095
return no_beam_search
1096
vllm/platforms/rocm.py
+11
@@ -1,10 +1,21 @@
1
+import os
2
from functools import lru_cache
3
from typing import Tuple
4
5
import torch
6
7
+from vllm.logger import init_logger
8
+
9
from .interface import Platform, PlatformEnum
10
11
+logger = init_logger(__name__)
12
+
13
+if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
14
+ logger.warning("`fork` method is not supported by ROCm. "
15
+ "VLLM_WORKER_MULTIPROC_METHOD is overridden to"
16
+ " `spawn` instead.")
17
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
18
+
19
20
class RocmPlatform(Platform):
21
_enum = PlatformEnum.ROCM