@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b6c16cf8ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ROCm][AMD] unify CUDA_VISIBLE_DEVICES usage in cuda/rocm (#6352)

Hongxia Yang committed Jul 12, 2024 at 00:30 UTC b6c16cf8ff8d558ec943f1f17342c2c081f3f5af
5 files changed +10 -34
Dockerfile.rocm
+7 -7
@@ -52,25 +52,25 @@ RUN pip install --upgrade pip
52 # Remove sccache so it doesn't interfere with ccache
53 # TODO: implement sccache support across components
54 RUN apt-get purge -y sccache; pip uninstall -y sccache; rm -f "$(which sccache)"
55 -# Install torch == 2.4.0 on ROCm
55 +# Install torch == 2.5.0 on ROCm
56 RUN case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
57 *"rocm-5.7"*) \
58 pip uninstall -y torch torchaudio torchvision \
59 && pip install --no-cache-dir --pre \
60 - torch==2.4.0.dev20240612 torchaudio==2.4.0.dev20240612 \
61 - torchvision==0.19.0.dev20240612 \
60 + torch==2.5.0.dev20240710 torchaudio==2.4.0.dev20240710 \
61 + torchvision==0.20.0.dev20240710 \
62 --index-url https://download.pytorch.org/whl/nightly/rocm5.7;; \
63 *"rocm-6.0"*) \
64 pip uninstall -y torch torchaudio torchvision \
65 && pip install --no-cache-dir --pre \
66 - torch==2.4.0.dev20240612 torchaudio==2.4.0.dev20240612 \
67 - torchvision==0.19.0.dev20240612 \
66 + torch==2.5.0.dev20240710 torchaudio==2.4.0.dev20240710 \
67 + torchvision==0.20.0.dev20240710 \
68 --index-url https://download.pytorch.org/whl/nightly/rocm6.0;; \
69 *"rocm-6.1"*) \
70 pip uninstall -y torch torchaudio torchvision \
71 && pip install --no-cache-dir --pre \
72 - torch==2.4.0.dev20240612 torchaudio==2.4.0.dev20240612 \
73 - torchvision==0.19.0.dev20240612 \
72 + torch==2.5.0.dev20240710 torchaudio==2.4.0.dev20240710 \
73 + torchvision==0.20.0.dev20240710 \
74 --index-url https://download.pytorch.org/whl/nightly/rocm6.1;; \
75 *) ;; esac
76
tests/distributed/test_utils.py
+1 -6
@@ -1,7 +1,7 @@
1 import ray
2
3 import vllm.envs as envs
4 -from vllm.utils import (cuda_device_count_stateless, is_hip,
4 +from vllm.utils import (cuda_device_count_stateless,
5 update_environment_variables)
6
7
@@ -22,11 +22,6 @@ class _CUDADeviceCountStatelessTestActor:
22 def test_cuda_device_count_stateless():
23 """Test that cuda_device_count_stateless changes return value if
24 CUDA_VISIBLE_DEVICES is changed."""
25 - if is_hip():
26 - # Set HIP_VISIBLE_DEVICES == CUDA_VISIBLE_DEVICES. Conversion
27 - # is handled by `update_environment_variables`
28 - update_environment_variables(
29 - {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
25 actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
26 num_gpus=2).remote()
27 assert sorted(ray.get(
vllm/config.py
+1 -8
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
6 import torch
7 from transformers import PretrainedConfig
8
9 -import vllm.envs as envs
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11 from vllm.model_executor.models import ModelRegistry
@@ -14,7 +13,7 @@ from vllm.tracing import is_otel_installed
13 from vllm.transformers_utils.config import get_config, get_hf_text_config
14 from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
15 is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
17 - print_warning_once, update_environment_variables)
16 + print_warning_once)
17
18 if TYPE_CHECKING:
19 from ray.util.placement_group import PlacementGroup
@@ -695,12 +694,6 @@ class ParallelConfig:
694 self.distributed_executor_backend = backend
695 logger.info("Defaulting to use %s for distributed inference",
696 backend)
698 - # If CUDA_VISIBLE_DEVICES is set on ROCm prior to vLLM init,
699 - # propagate changes to HIP_VISIBLE_DEVICES (conversion handled by
700 - # the update_environment_variables function)
701 - if is_hip() and envs.CUDA_VISIBLE_DEVICES:
702 - update_environment_variables(
703 - {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
697
698 self._verify_args()
699 self.rank = 0
vllm/utils.py
-4
@@ -386,10 +386,6 @@ def get_open_port() -> int:
386
387
388 def update_environment_variables(envs: Dict[str, str]):
389 - if is_hip() and "CUDA_VISIBLE_DEVICES" in envs:
390 - # Propagate changes to CUDA_VISIBLE_DEVICES to
391 - # ROCm's HIP_VISIBLE_DEVICES as well
392 - envs["HIP_VISIBLE_DEVICES"] = envs["CUDA_VISIBLE_DEVICES"]
389 for k, v in envs.items():
390 if k in os.environ and os.environ[k] != v:
391 logger.warning(
vllm/worker/worker_base.py
+1 -9
@@ -11,7 +11,7 @@ from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
13 SamplerOutput)
14 -from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
14 +from vllm.utils import (enable_trace_function_call_for_thread,
15 update_environment_variables)
16 from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
17
@@ -309,14 +309,6 @@ class WorkerWrapperBase:
309 # overwriting CUDA_VISIBLE_DEVICES is desired behavior
310 # suppress the warning in `update_environment_variables`
311 del os.environ[key]
312 - if is_hip():
313 - hip_env_var = "HIP_VISIBLE_DEVICES"
314 - if hip_env_var in os.environ:
315 - logger.warning(
316 - "Ignoring pre-set environment variable `%s=%s` as "
317 - "%s has also been set, which takes precedence.",
318 - hip_env_var, os.environ[hip_env_var], key)
319 - os.environ.pop(hip_env_var, None)
312 update_environment_variables(envs)
313
314 def init_worker(self, *args, **kwargs):