@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 30e775281d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

fix typo (#1184)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Wang Ran (汪然) committed Sep 28, 2023 at 07:22 UTC 30e775281d0ba983e0e2e9904fc9e0fb1a055c20
2 files changed +5 -6
vllm/engine/llm_engine.py
+2 -2
@@ -54,8 +54,8 @@ class LLMEngine:
54 scheduler_config: The configuration related to the request scheduler.
55 distributed_init_method: The initialization method for distributed
56 execution. See `torch.distributed.init_process_group` for details.
57 - stage_devices: The list of devices for each stage. Each stage is a list
58 - of (rank, node_resource, device) tuples.
57 + placement_group: Ray placement group for distributed execution.
58 + Required for distributed execution.
59 log_stats: Whether to log statistics.
60 """
61
vllm/engine/ray_utils.py
+3 -4
@@ -63,11 +63,10 @@ def initialize_cluster(
63 the default Ray cluster address.
64
65 Returns:
66 - A tuple of (`distributed_init_method`, `all_stage_devices`). The
66 + A tuple of (`distributed_init_method`, `placement_group`). The
67 `distributed_init_method` is the address for initializing the
68 - distributed backend. `all_stage_devices` includes device IDs for
69 - each worker in each pipeline stage. Each device ID is a tuple of
70 - (rank, node resource, device id).
68 + distributed backend. `placement_group` includes the specification
69 + of the resources for each distributed worker.
70 """
71 if parallel_config.worker_use_ray or engine_use_ray:
72 if ray is None: