A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Better integration with Ray Serve (#1821)
Co-authored-by: FlorianJoncour <florian@zetta-sys.com>
FlorianJoncour committed
Nov 29, 2023 at 21:25 UTC
0229c386c541a293f18a9ffe1a5cd7735d487158
2 files changed
+5
-5
vllm/engine/llm_engine.py
+3
-3
@@ -7,7 +7,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
SchedulerConfig)
8
from vllm.core.scheduler import Scheduler, SchedulerOutputs
9
from vllm.engine.arg_utils import EngineArgs
10
-from vllm.engine.ray_utils import RayWorker, initialize_cluster, ray
10
+from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
11
from vllm.logger import init_logger
12
from vllm.outputs import RequestOutput
13
from vllm.sampling_params import SamplingParams
@@ -162,12 +162,12 @@ class LLMEngine:
162
continue
163
worker = ray.remote(
164
num_cpus=0,
165
- num_gpus=1,
165
+ num_gpus=self.cache_config.gpu_memory_utilization,
166
scheduling_strategy=PlacementGroupSchedulingStrategy(
167
placement_group=placement_group,
168
placement_group_capture_child_tasks=True),
169
**ray_remote_kwargs,
170
- )(RayWorker).remote(self.model_config.trust_remote_code)
170
+ )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
171
self.workers.append(worker)
172
173
# Initialize torch distributed process group for the workers.
vllm/engine/ray_utils.py
+2
-2
@@ -10,7 +10,7 @@ try:
10
import ray
11
from ray.air.util.torch_dist import TorchDistributedWorker
12
13
- class RayWorker(TorchDistributedWorker):
13
+ class RayWorkerVllm(TorchDistributedWorker):
14
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
15
lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
16
@@ -36,7 +36,7 @@ except ImportError as e:
36
"`pip install ray pandas pyarrow`.")
37
ray = None
38
TorchDistributedWorker = None
39
- RayWorker = None
39
+ RayWorkerVllm = None
40
41
if TYPE_CHECKING:
42
from ray.util.placement_group import PlacementGroup