A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
fix typo (#1184)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Wang Ran (汪然) committed
Sep 28, 2023 at 07:22 UTC
30e775281d0ba983e0e2e9904fc9e0fb1a055c20
2 files changed
+5
-6
vllm/engine/llm_engine.py
+2
-2
@@ -54,8 +54,8 @@ class LLMEngine:
54
scheduler_config: The configuration related to the request scheduler.
55
distributed_init_method: The initialization method for distributed
56
execution. See `torch.distributed.init_process_group` for details.
57
- stage_devices: The list of devices for each stage. Each stage is a list
58
- of (rank, node_resource, device) tuples.
57
+ placement_group: Ray placement group for distributed execution.
58
+ Required for distributed execution.
59
log_stats: Whether to log statistics.
60
"""
61
vllm/engine/ray_utils.py
+3
-4
@@ -63,11 +63,10 @@ def initialize_cluster(
63
the default Ray cluster address.
64
65
Returns:
66
- A tuple of (`distributed_init_method`, `all_stage_devices`). The
66
+ A tuple of (`distributed_init_method`, `placement_group`). The
67
`distributed_init_method` is the address for initializing the
68
- distributed backend. `all_stage_devices` includes device IDs for
69
- each worker in each pipeline stage. Each device ID is a tuple of
70
- (rank, node resource, device id).
68
+ distributed backend. `placement_group` includes the specification
69
+ of the resources for each distributed worker.
70
"""
71
if parallel_config.worker_use_ray or engine_use_ray:
72
if ray is None: