A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Use ray[adag] dependency instead of cuda (#7938)
Rui Qiao committed
Sep 6, 2024 at 09:18 UTC
de80783b6907eb084493a76ef9ec3e3941cc2087
5 files changed
+19
-12
Dockerfile
-2
@@ -37,7 +37,6 @@ WORKDIR /workspace
37
38
# install build and runtime dependencies
39
COPY requirements-common.txt requirements-common.txt
40
-COPY requirements-adag.txt requirements-adag.txt
40
COPY requirements-cuda.txt requirements-cuda.txt
41
RUN --mount=type=cache,target=/root/.cache/pip \
42
python3 -m pip install -r requirements-cuda.txt
@@ -66,7 +65,6 @@ COPY setup.py setup.py
65
COPY cmake cmake
66
COPY CMakeLists.txt CMakeLists.txt
67
COPY requirements-common.txt requirements-common.txt
69
-COPY requirements-adag.txt requirements-adag.txt
68
COPY requirements-cuda.txt requirements-cuda.txt
69
COPY pyproject.toml pyproject.toml
70
COPY vllm vllm
MANIFEST.in
-1
@@ -1,5 +1,4 @@
1
include LICENSE
2
-include requirements-adag.txt
2
include requirements-common.txt
3
include requirements-cuda.txt
4
include requirements-rocm.txt
requirements-adag.txt
deleted
-3
@@ -1,3 +0,0 @@
1
-# Dependencies for Ray accelerated DAG
2
-cupy-cuda12x
3
-ray >= 2.32
\ No newline at end of file
requirements-test.txt
+1
-4
@@ -1,6 +1,3 @@
1
-# Needed for Ray accelerated DAG tests
2
--r requirements-adag.txt
3
-
1
# testing
2
pytest
3
tensorizer>=2.9.0
@@ -16,7 +13,7 @@ httpx
13
librosa # required for audio test
14
peft
15
requests
19
-ray
16
+ray[adag]>=2.35
17
sentence-transformers # required for embedding
18
soundfile # required for audio test
19
compressed-tensors==0.4.0 # required for compressed-tensors
vllm/executor/ray_gpu_executor.py
+18
-2
@@ -427,18 +427,34 @@ class RayGPUExecutor(DistributedGPUExecutor):
427
async_run_remote_workers_only to complete."""
428
ray.get(parallel_worker_tasks)
429
430
- def _compiled_ray_dag(self, enable_asyncio: bool):
430
+ def _check_ray_adag_installation(self):
431
import pkg_resources
432
from packaging import version
433
434
- required_version = version.parse("2.32")
434
+ required_version = version.parse("2.35")
435
current_version = version.parse(
436
pkg_resources.get_distribution("ray").version)
437
if current_version < required_version:
438
raise ValueError(f"Ray version {required_version} or greater is "
439
f"required, but found {current_version}")
440
441
+ import importlib.util
442
+ adag_spec = importlib.util.find_spec(
443
+ "ray.experimental.compiled_dag_ref")
444
+ if adag_spec is None:
445
+ raise ValueError("Ray accelerated DAG is not installed. "
446
+ "Run `pip install ray[adag]` to install it.")
447
+
448
+ cupy_spec = importlib.util.find_spec("cupy")
449
+ if cupy_spec is None and envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL:
450
+ raise ValueError(
451
+ "cupy is not installed but required since "
452
+ "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL is set."
453
+ "Run `pip install ray[adag]` and check cupy installation.")
454
+
455
+ def _compiled_ray_dag(self, enable_asyncio: bool):
456
assert self.parallel_config.use_ray
457
+ self._check_ray_adag_installation()
458
from ray.dag import InputNode, MultiOutputNode
459
from ray.experimental.channel.torch_tensor_type import TorchTensorType
460