@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0229c386c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Better integration with Ray Serve (#1821)

Co-authored-by: FlorianJoncour <florian@zetta-sys.com>

FlorianJoncour committed Nov 29, 2023 at 21:25 UTC 0229c386c541a293f18a9ffe1a5cd7735d487158
2 files changed +5 -5
vllm/engine/llm_engine.py
+3 -3
@@ -7,7 +7,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 SchedulerConfig)
8 from vllm.core.scheduler import Scheduler, SchedulerOutputs
9 from vllm.engine.arg_utils import EngineArgs
10 -from vllm.engine.ray_utils import RayWorker, initialize_cluster, ray
10 +from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
11 from vllm.logger import init_logger
12 from vllm.outputs import RequestOutput
13 from vllm.sampling_params import SamplingParams
@@ -162,12 +162,12 @@ class LLMEngine:
162 continue
163 worker = ray.remote(
164 num_cpus=0,
165 - num_gpus=1,
165 + num_gpus=self.cache_config.gpu_memory_utilization,
166 scheduling_strategy=PlacementGroupSchedulingStrategy(
167 placement_group=placement_group,
168 placement_group_capture_child_tasks=True),
169 **ray_remote_kwargs,
170 - )(RayWorker).remote(self.model_config.trust_remote_code)
170 + )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
171 self.workers.append(worker)
172
173 # Initialize torch distributed process group for the workers.
vllm/engine/ray_utils.py
+2 -2
@@ -10,7 +10,7 @@ try:
10 import ray
11 from ray.air.util.torch_dist import TorchDistributedWorker
12
13 - class RayWorker(TorchDistributedWorker):
13 + class RayWorkerVllm(TorchDistributedWorker):
14 """Ray wrapper for vllm.worker.Worker, allowing Worker to be
15 lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
16
@@ -36,7 +36,7 @@ except ImportError as e:
36 "`pip install ray pandas pyarrow`.")
37 ray = None
38 TorchDistributedWorker = None
39 - RayWorker = None
39 + RayWorkerVllm = None
40
41 if TYPE_CHECKING:
42 from ray.util.placement_group import PlacementGroup