A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][core] Implement pipeline parallel on Ray (#12996)
Rui Qiao committed
Feb 13, 2025 at 00:02 UTC
9605c1256ece238c5702c7020e6806eb2c2ebeb0
7 files changed
+110
-45
tests/distributed/test_pipeline_parallel.py
+39
-12
@@ -40,10 +40,23 @@ class PPTestOptions(NamedTuple):
40
@dataclass
41
class PPTestSettings:
42
parallel_setups: List[ParallelSetup]
43
+ # NOTE: the length of distributed_backends and
44
+ # vllm_major_versions should be the same, and they
45
+ # are first zipped together to iterate over all
46
+ # test settings.
47
distributed_backends: List[str]
48
+ # vllm major version: "0" for V0, "1" for V1
49
+ vllm_major_versions: List[str]
50
task: TaskOption
51
test_options: PPTestOptions
52
53
+ def __post_init__(self):
54
+ if len(self.distributed_backends) != len(self.vllm_major_versions):
55
+ raise ValueError(
56
+ f"Length mismatch: distributed_backends "
57
+ f"({len(self.distributed_backends)}) != "
58
+ f"vllm_major_versions ({len(self.vllm_major_versions)})")
59
+
60
@staticmethod
61
def detailed(
62
*,
@@ -79,7 +92,9 @@ class PPTestSettings:
92
eager_mode=True,
93
chunked_prefill=False),
94
],
82
- distributed_backends=["mp", "ray"],
95
+ # only ray is supported for V1
96
+ distributed_backends=["mp", "ray", "ray"],
97
+ vllm_major_versions=["0", "0", "1"],
98
task=task,
99
test_options=PPTestOptions(multi_node_only=multi_node_only,
100
trust_remote_code=trust_remote_code,
@@ -108,6 +123,7 @@ class PPTestSettings:
123
chunked_prefill=False),
124
],
125
distributed_backends=["mp"],
126
+ vllm_major_versions=["0"],
127
task=task,
128
test_options=PPTestOptions(multi_node_only=multi_node_only,
129
trust_remote_code=trust_remote_code,
@@ -120,8 +136,9 @@ class PPTestSettings:
136
opts = self.test_options
137
138
for parallel_setup in self.parallel_setups:
123
- for distributed_backend in self.distributed_backends:
124
- yield (model_name, parallel_setup, distributed_backend,
139
+ for backend, vllm_major_version in zip(self.distributed_backends,
140
+ self.vllm_major_versions):
141
+ yield (model_name, parallel_setup, backend, vllm_major_version,
142
self.task, opts)
143
144
@@ -244,6 +261,7 @@ def _compare_tp(
261
model_name: str,
262
parallel_setup: ParallelSetup,
263
distributed_backend: str,
264
+ vllm_major_version: str,
265
task: TaskOption,
266
test_options: PPTestOptions,
267
num_gpus_available: int,
@@ -296,10 +314,13 @@ def _compare_tp(
314
if hf_overrides:
315
common_args.extend(["--hf-overrides", hf_overrides])
316
299
- if (distributed_backend == "ray" and tp_size == 2 and pp_size == 2
300
- and chunked_prefill):
301
- # Test Ray ADAG for a subset of the tests
317
+ specific_case = tp_size == 2 and pp_size == 2 and chunked_prefill
318
+ if distributed_backend == "ray" and (vllm_major_version == "1"
319
+ or specific_case):
320
+ # For V1, test Ray ADAG for all the tests
321
+ # For V0, test Ray ADAG for a subset of the tests
322
pp_env = {
323
+ "VLLM_USE_V1": vllm_major_version,
324
"VLLM_USE_RAY_COMPILED_DAG": "1",
325
"VLLM_USE_RAY_SPMD_WORKER": "1",
326
"VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": "1",
@@ -348,8 +369,8 @@ def _compare_tp(
369
370
371
@pytest.mark.parametrize(
351
- ("model_name", "parallel_setup", "distributed_backend", "task",
352
- "test_options"),
372
+ ("model_name", "parallel_setup", "distributed_backend",
373
+ "vllm_major_version", "task", "test_options"),
374
[
375
params for model_name, settings in TEXT_GENERATION_MODELS.items()
376
for params in settings.iter_params(model_name)
@@ -361,6 +382,7 @@ def test_tp_language_generation(
382
model_name: str,
383
parallel_setup: ParallelSetup,
384
distributed_backend: str,
385
+ vllm_major_version: str,
386
task: TaskOption,
387
test_options: PPTestOptions,
388
num_gpus_available,
@@ -368,6 +390,7 @@ def test_tp_language_generation(
390
_compare_tp(model_name,
391
parallel_setup,
392
distributed_backend,
393
+ vllm_major_version,
394
task,
395
test_options,
396
num_gpus_available,
@@ -375,8 +398,8 @@ def test_tp_language_generation(
398
399
400
@pytest.mark.parametrize(
378
- ("model_name", "parallel_setup", "distributed_backend", "task",
379
- "test_options"),
401
+ ("model_name", "parallel_setup", "distributed_backend",
402
+ "vllm_major_version", "task", "test_options"),
403
[
404
params for model_name, settings in EMBEDDING_MODELS.items()
405
for params in settings.iter_params(model_name)
@@ -388,6 +411,7 @@ def test_tp_language_embedding(
411
model_name: str,
412
parallel_setup: ParallelSetup,
413
distributed_backend: str,
414
+ vllm_major_version: str,
415
task: TaskOption,
416
test_options: PPTestOptions,
417
num_gpus_available,
@@ -395,6 +419,7 @@ def test_tp_language_embedding(
419
_compare_tp(model_name,
420
parallel_setup,
421
distributed_backend,
422
+ vllm_major_version,
423
task,
424
test_options,
425
num_gpus_available,
@@ -402,8 +427,8 @@ def test_tp_language_embedding(
427
428
429
@pytest.mark.parametrize(
405
- ("model_name", "parallel_setup", "distributed_backend", "task",
406
- "test_options"),
430
+ ("model_name", "parallel_setup", "distributed_backend",
431
+ "vllm_major_version", "task", "test_options"),
432
[
433
params for model_name, settings in MULTIMODAL_MODELS.items()
434
for params in settings.iter_params(model_name)
@@ -415,6 +440,7 @@ def test_tp_multimodal_generation(
440
model_name: str,
441
parallel_setup: ParallelSetup,
442
distributed_backend: str,
443
+ vllm_major_version: str,
444
task: TaskOption,
445
test_options: PPTestOptions,
446
num_gpus_available,
@@ -422,6 +448,7 @@ def test_tp_multimodal_generation(
448
_compare_tp(model_name,
449
parallel_setup,
450
distributed_backend,
451
+ vllm_major_version,
452
task,
453
test_options,
454
num_gpus_available,
vllm/executor/ray_utils.py
+9
-2
@@ -35,7 +35,7 @@ try:
35
36
class RayWorkerWrapper(WorkerWrapperBase):
37
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
38
- lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
38
+ lazily initialized after Ray sets CUDA_VISIBLE_DEVICES."""
39
40
def __init__(self, *args, **kwargs) -> None:
41
super().__init__(*args, **kwargs)
@@ -118,7 +118,14 @@ try:
118
) -> "ModelRunnerOutput":
119
self.setup_device_if_necessary()
120
assert self.worker is not None, "Worker is not initialized"
121
- output = self.worker.model_runner.execute_model(scheduler_output)
121
+ if isinstance(scheduler_output, tuple):
122
+ scheduler_output, intermediate_tensors = scheduler_output
123
+ else:
124
+ scheduler_output, intermediate_tensors = scheduler_output, None
125
+ output = self.worker.model_runner.execute_model(
126
+ scheduler_output, intermediate_tensors)
127
+ if isinstance(output, IntermediateTensors):
128
+ output = scheduler_output, output
129
return output
130
131
def override_env_vars(self, vars: Dict[str, str]):
vllm/v1/core/kv_cache_utils.py
+27
-14
@@ -488,7 +488,8 @@ def is_kv_cache_type_uniform(kv_cache_spec: KVCacheSpec) -> bool:
488
489
def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
490
kv_cache_spec: KVCacheSpec,
491
- available_memory: int) -> KVCacheConfig:
491
+ available_memory: int,
492
+ num_layers: int) -> KVCacheConfig:
493
"""
494
Generates the KV cache configuration for a model with one type of KV cache.
495
Divide the available memory equally among all layers.
@@ -497,6 +498,7 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
498
vllm_config: The global VllmConfig
499
kv_cache_spec: The kv cache spec of the model
500
available_memory: Memory available for KV cache in bytes.
501
+ num_layers: The number of layers in the model.
502
503
Returns:
504
The generated KVCacheConfig
@@ -506,7 +508,7 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
508
assert len(page_sizes) == 1
509
page_size = page_sizes.pop()
510
509
- num_blocks = int(available_memory // page_size // len(kv_cache_spec))
511
+ num_blocks = int(available_memory // page_size // num_layers)
512
num_blocks = max(num_blocks, 0)
513
514
if vllm_config.cache_config.num_gpu_blocks_override is not None:
@@ -536,25 +538,36 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
538
return kv_cache_config
539
540
539
-def get_kv_cache_config(vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec,
540
- available_memory: int) -> KVCacheConfig:
541
+def get_kv_cache_configs(vllm_config: VllmConfig,
542
+ kv_cache_specs: List[KVCacheSpec],
543
+ available_memory: int) -> List[KVCacheConfig]:
544
"""
545
Generates the KV cache configuration for a model
546
TODO: support hybrid models with more than one type of KV cache.
547
548
Args:
549
vllm_config: The global VllmConfig
547
- kv_cache_spec: The kv cache spec of the model
550
+ kv_cache_specs: The kv cache specs of the model
551
available_memory: Memory available for KV cache in bytes.
552
553
Returns:
551
- The generated KVCacheConfig
554
+ The generated KVCacheConfigs
555
"""
553
- check_enough_kv_cache_memory(vllm_config, kv_cache_spec, available_memory)
554
- if is_kv_cache_type_uniform(kv_cache_spec):
555
- # KV cache of all layers are the same, which is true for most models.
556
- # Allocate the same amount of memory for each layer.
557
- return _get_kv_cache_config_uniform_type(vllm_config, kv_cache_spec,
558
- available_memory)
559
- else:
560
- raise NotImplementedError
556
+ # Use the max number of layers to conservatively determine
557
+ # the number of blocks.
558
+ num_layers = max(len(kv_cache_spec) for kv_cache_spec in kv_cache_specs)
559
+ kv_cache_configs = []
560
+ for kv_cache_spec in kv_cache_specs:
561
+ check_enough_kv_cache_memory(vllm_config, kv_cache_spec,
562
+ available_memory)
563
+ if is_kv_cache_type_uniform(kv_cache_spec):
564
+ # KV cache of all layers are the same, which is true for
565
+ # most models. Allocate the same amount of memory for
566
+ # each layer.
567
+ kv_cache_configs.append(
568
+ _get_kv_cache_config_uniform_type(vllm_config, kv_cache_spec,
569
+ available_memory,
570
+ num_layers))
571
+ else:
572
+ raise NotImplementedError
573
+ return kv_cache_configs
vllm/v1/engine/core.py
+12
-7
@@ -16,7 +16,7 @@ from vllm.logger import init_logger
16
from vllm.transformers_utils.config import (
17
maybe_register_config_serialize_by_value)
18
from vllm.utils import get_exception_traceback, zmq_socket_ctx
19
-from vllm.v1.core.kv_cache_utils import get_kv_cache_config
19
+from vllm.v1.core.kv_cache_utils import get_kv_cache_configs
20
from vllm.v1.core.scheduler import Scheduler
21
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
22
EngineCoreRequestType)
@@ -73,20 +73,25 @@ class EngineCore:
73
start = time.time()
74
75
# Get all kv cache needed by the model
76
- kv_cache_spec = self.model_executor.get_kv_cache_spec()
76
+ kv_cache_specs = self.model_executor.get_kv_cache_specs()
77
78
# Profiles the peak memory usage of the model to determine how much
79
# memory can be allocated for kv cache.
80
- availble_gpu_memory = self.model_executor.determine_available_memory()
80
+ available_gpu_memory = self.model_executor.determine_available_memory()
81
82
# Get the kv cache tensor size
83
- kv_cache_config = get_kv_cache_config(vllm_config, kv_cache_spec,
84
- availble_gpu_memory)
85
- num_gpu_blocks = kv_cache_config.num_blocks
83
+ kv_cache_configs = get_kv_cache_configs(vllm_config, kv_cache_specs,
84
+ available_gpu_memory)
85
+ num_gpu_blocks_set = set(config.num_blocks
86
+ for config in kv_cache_configs)
87
+ assert len(num_gpu_blocks_set) == 1, (
88
+ f"num_gpu_blocks need to be the same across workers, "
89
+ f"but they are different: {num_gpu_blocks_set}")
90
+ num_gpu_blocks = num_gpu_blocks_set.pop()
91
num_cpu_blocks = 0
92
93
# Initialize kv cache and warmup the execution
89
- self.model_executor.initialize(kv_cache_config)
94
+ self.model_executor.initialize(kv_cache_configs)
95
96
elapsed = time.time() - start
97
logger.info(("init engine (profile, create kv cache, "
vllm/v1/executor/abstract.py
+5
-7
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import Type
3
+from typing import List, Type
4
5
from vllm.config import VllmConfig
6
from vllm.executor.executor_base import ExecutorBase
@@ -48,12 +48,12 @@ class Executor(ExecutorBase):
48
f"{distributed_executor_backend}")
49
return executor_class
50
51
- def initialize(self, kv_cache_config: KVCacheConfig) -> None:
51
+ def initialize(self, kv_cache_configs: List[KVCacheConfig]) -> None:
52
"""
53
Initialize the KV caches and begin the model execution loop of the
54
underlying workers.
55
"""
56
- self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
56
+ self.collective_rpc("initialize_cache", args=(kv_cache_configs, ))
57
self.collective_rpc("compile_or_warm_up_model")
58
59
def determine_available_memory(self) -> int: # in bytes
@@ -63,11 +63,9 @@ class Executor(ExecutorBase):
63
# operators can be applied to all workers.
64
return min(output)
65
66
- def get_kv_cache_spec(self) -> KVCacheSpec:
66
+ def get_kv_cache_specs(self) -> List[KVCacheSpec]:
67
output = self.collective_rpc("get_kv_cache_spec")
68
- for x in output:
69
- assert x == output[0]
70
- return output[0]
68
+ return output
69
70
def execute_model(
71
self,
vllm/v1/worker/gpu_model_runner.py
+15
-1
@@ -12,7 +12,7 @@ import torch.nn as nn
12
from vllm.attention.backends.abstract import AttentionType
13
from vllm.attention.layer import Attention
14
from vllm.config import CompilationLevel, VllmConfig
15
-from vllm.distributed.parallel_state import graph_capture
15
+from vllm.distributed.parallel_state import get_pp_group, graph_capture
16
from vllm.forward_context import set_forward_context
17
from vllm.inputs import INPUT_REGISTRY
18
from vllm.logger import init_logger
@@ -21,6 +21,7 @@ from vllm.model_executor.model_loader import get_model
21
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
22
from vllm.multimodal.utils import group_mm_inputs_by_modality
23
from vllm.sampling_params import SamplingType
24
+from vllm.sequence import IntermediateTensors
25
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
26
LayerBlockType, cdiv, is_pin_memory_available)
27
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
@@ -773,6 +774,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
774
def execute_model(
775
self,
776
scheduler_output: "SchedulerOutput",
777
+ intermediate_tensors: Optional[IntermediateTensors] = None,
778
) -> ModelRunnerOutput:
779
batch_changed = self._update_states(scheduler_output)
780
@@ -831,8 +833,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
833
positions=positions,
834
kv_caches=self.kv_caches,
835
attn_metadata=None,
836
+ intermediate_tensors=intermediate_tensors,
837
inputs_embeds=inputs_embeds,
838
)
839
+ if not get_pp_group().is_last_rank:
840
+ return hidden_states
841
hidden_states = hidden_states[:num_scheduled_tokens]
842
sample_hidden_states = hidden_states[logits_indices]
843
logits = self.model.compute_logits(sample_hidden_states, None)
@@ -1007,12 +1012,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1012
positions = self.mrope_positions[:, :num_tokens]
1013
else:
1014
positions = self.positions[:num_tokens]
1015
+ intermediate_tensors = None
1016
+ if not get_pp_group().is_first_rank:
1017
+ intermediate_tensors = self.model.make_empty_intermediate_tensors(
1018
+ batch_size=num_tokens,
1019
+ dtype=self.model_config.dtype,
1020
+ device=self.device)
1021
with set_forward_context(None, self.vllm_config):
1022
hidden_states = model(
1023
input_ids=input_ids,
1024
positions=positions,
1025
kv_caches=kv_caches,
1026
attn_metadata=None,
1027
+ intermediate_tensors=intermediate_tensors,
1028
inputs_embeds=inputs_embeds,
1029
)
1030
return hidden_states
@@ -1142,6 +1154,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1154
# Trigger compilation for general shape.
1155
hidden_states = self._dummy_run(self.max_num_tokens,
1156
dummy_kv_caches)
1157
+ if not get_pp_group().is_last_rank:
1158
+ return hidden_states
1159
hidden_states = hidden_states[logit_indices]
1160
logits = self.model.compute_logits(hidden_states, None)
1161
# TODO(woosuk): Consider the memory usage of the sampler.
vllm/v1/worker/gpu_worker.py
+3
-2
@@ -2,7 +2,7 @@
2
"""A GPU worker class."""
3
import gc
4
import os
5
-from typing import TYPE_CHECKING, Optional
5
+from typing import TYPE_CHECKING, List, Optional
6
7
import torch
8
import torch.distributed
@@ -194,8 +194,9 @@ class Worker:
194
def get_kv_cache_spec(self) -> KVCacheSpec:
195
return self.model_runner.get_kv_cache_spec()
196
197
- def initialize_cache(self, kv_cache_config: KVCacheConfig) -> None:
197
+ def initialize_cache(self, kv_cache_configs: List[KVCacheConfig]) -> None:
198
"""Allocate GPU KV cache with the specified kv_cache_config."""
199
+ kv_cache_config = kv_cache_configs[self.rank]
200
if self.vllm_config.model_config.enable_sleep_mode:
201
allocator = CuMemAllocator.get_instance()
202
context = allocator.use_memory_pool(tag="kv_cache")