@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9605c1256e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][core] Implement pipeline parallel on Ray (#12996)

Rui Qiao committed Feb 13, 2025 at 00:02 UTC 9605c1256ece238c5702c7020e6806eb2c2ebeb0
7 files changed +110 -45
tests/distributed/test_pipeline_parallel.py
+39 -12
@@ -40,10 +40,23 @@ class PPTestOptions(NamedTuple):
40 @dataclass
41 class PPTestSettings:
42 parallel_setups: List[ParallelSetup]
43 + # NOTE: the length of distributed_backends and
44 + # vllm_major_versions should be the same, and they
45 + # are first zipped together to iterate over all
46 + # test settings.
47 distributed_backends: List[str]
48 + # vllm major version: "0" for V0, "1" for V1
49 + vllm_major_versions: List[str]
50 task: TaskOption
51 test_options: PPTestOptions
52
53 + def __post_init__(self):
54 + if len(self.distributed_backends) != len(self.vllm_major_versions):
55 + raise ValueError(
56 + f"Length mismatch: distributed_backends "
57 + f"({len(self.distributed_backends)}) != "
58 + f"vllm_major_versions ({len(self.vllm_major_versions)})")
59 +
60 @staticmethod
61 def detailed(
62 *,
@@ -79,7 +92,9 @@ class PPTestSettings:
92 eager_mode=True,
93 chunked_prefill=False),
94 ],
82 - distributed_backends=["mp", "ray"],
95 + # only ray is supported for V1
96 + distributed_backends=["mp", "ray", "ray"],
97 + vllm_major_versions=["0", "0", "1"],
98 task=task,
99 test_options=PPTestOptions(multi_node_only=multi_node_only,
100 trust_remote_code=trust_remote_code,
@@ -108,6 +123,7 @@ class PPTestSettings:
123 chunked_prefill=False),
124 ],
125 distributed_backends=["mp"],
126 + vllm_major_versions=["0"],
127 task=task,
128 test_options=PPTestOptions(multi_node_only=multi_node_only,
129 trust_remote_code=trust_remote_code,
@@ -120,8 +136,9 @@ class PPTestSettings:
136 opts = self.test_options
137
138 for parallel_setup in self.parallel_setups:
123 - for distributed_backend in self.distributed_backends:
124 - yield (model_name, parallel_setup, distributed_backend,
139 + for backend, vllm_major_version in zip(self.distributed_backends,
140 + self.vllm_major_versions):
141 + yield (model_name, parallel_setup, backend, vllm_major_version,
142 self.task, opts)
143
144
@@ -244,6 +261,7 @@ def _compare_tp(
261 model_name: str,
262 parallel_setup: ParallelSetup,
263 distributed_backend: str,
264 + vllm_major_version: str,
265 task: TaskOption,
266 test_options: PPTestOptions,
267 num_gpus_available: int,
@@ -296,10 +314,13 @@ def _compare_tp(
314 if hf_overrides:
315 common_args.extend(["--hf-overrides", hf_overrides])
316
299 - if (distributed_backend == "ray" and tp_size == 2 and pp_size == 2
300 - and chunked_prefill):
301 - # Test Ray ADAG for a subset of the tests
317 + specific_case = tp_size == 2 and pp_size == 2 and chunked_prefill
318 + if distributed_backend == "ray" and (vllm_major_version == "1"
319 + or specific_case):
320 + # For V1, test Ray ADAG for all the tests
321 + # For V0, test Ray ADAG for a subset of the tests
322 pp_env = {
323 + "VLLM_USE_V1": vllm_major_version,
324 "VLLM_USE_RAY_COMPILED_DAG": "1",
325 "VLLM_USE_RAY_SPMD_WORKER": "1",
326 "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": "1",
@@ -348,8 +369,8 @@ def _compare_tp(
369
370
371 @pytest.mark.parametrize(
351 - ("model_name", "parallel_setup", "distributed_backend", "task",
352 - "test_options"),
372 + ("model_name", "parallel_setup", "distributed_backend",
373 + "vllm_major_version", "task", "test_options"),
374 [
375 params for model_name, settings in TEXT_GENERATION_MODELS.items()
376 for params in settings.iter_params(model_name)
@@ -361,6 +382,7 @@ def test_tp_language_generation(
382 model_name: str,
383 parallel_setup: ParallelSetup,
384 distributed_backend: str,
385 + vllm_major_version: str,
386 task: TaskOption,
387 test_options: PPTestOptions,
388 num_gpus_available,
@@ -368,6 +390,7 @@ def test_tp_language_generation(
390 _compare_tp(model_name,
391 parallel_setup,
392 distributed_backend,
393 + vllm_major_version,
394 task,
395 test_options,
396 num_gpus_available,
@@ -375,8 +398,8 @@ def test_tp_language_generation(
398
399
400 @pytest.mark.parametrize(
378 - ("model_name", "parallel_setup", "distributed_backend", "task",
379 - "test_options"),
401 + ("model_name", "parallel_setup", "distributed_backend",
402 + "vllm_major_version", "task", "test_options"),
403 [
404 params for model_name, settings in EMBEDDING_MODELS.items()
405 for params in settings.iter_params(model_name)
@@ -388,6 +411,7 @@ def test_tp_language_embedding(
411 model_name: str,
412 parallel_setup: ParallelSetup,
413 distributed_backend: str,
414 + vllm_major_version: str,
415 task: TaskOption,
416 test_options: PPTestOptions,
417 num_gpus_available,
@@ -395,6 +419,7 @@ def test_tp_language_embedding(
419 _compare_tp(model_name,
420 parallel_setup,
421 distributed_backend,
422 + vllm_major_version,
423 task,
424 test_options,
425 num_gpus_available,
@@ -402,8 +427,8 @@ def test_tp_language_embedding(
427
428
429 @pytest.mark.parametrize(
405 - ("model_name", "parallel_setup", "distributed_backend", "task",
406 - "test_options"),
430 + ("model_name", "parallel_setup", "distributed_backend",
431 + "vllm_major_version", "task", "test_options"),
432 [
433 params for model_name, settings in MULTIMODAL_MODELS.items()
434 for params in settings.iter_params(model_name)
@@ -415,6 +440,7 @@ def test_tp_multimodal_generation(
440 model_name: str,
441 parallel_setup: ParallelSetup,
442 distributed_backend: str,
443 + vllm_major_version: str,
444 task: TaskOption,
445 test_options: PPTestOptions,
446 num_gpus_available,
@@ -422,6 +448,7 @@ def test_tp_multimodal_generation(
448 _compare_tp(model_name,
449 parallel_setup,
450 distributed_backend,
451 + vllm_major_version,
452 task,
453 test_options,
454 num_gpus_available,
vllm/executor/ray_utils.py
+9 -2
@@ -35,7 +35,7 @@ try:
35
36 class RayWorkerWrapper(WorkerWrapperBase):
37 """Ray wrapper for vllm.worker.Worker, allowing Worker to be
38 - lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
38 + lazily initialized after Ray sets CUDA_VISIBLE_DEVICES."""
39
40 def __init__(self, *args, **kwargs) -> None:
41 super().__init__(*args, **kwargs)
@@ -118,7 +118,14 @@ try:
118 ) -> "ModelRunnerOutput":
119 self.setup_device_if_necessary()
120 assert self.worker is not None, "Worker is not initialized"
121 - output = self.worker.model_runner.execute_model(scheduler_output)
121 + if isinstance(scheduler_output, tuple):
122 + scheduler_output, intermediate_tensors = scheduler_output
123 + else:
124 + scheduler_output, intermediate_tensors = scheduler_output, None
125 + output = self.worker.model_runner.execute_model(
126 + scheduler_output, intermediate_tensors)
127 + if isinstance(output, IntermediateTensors):
128 + output = scheduler_output, output
129 return output
130
131 def override_env_vars(self, vars: Dict[str, str]):
vllm/v1/core/kv_cache_utils.py
+27 -14
@@ -488,7 +488,8 @@ def is_kv_cache_type_uniform(kv_cache_spec: KVCacheSpec) -> bool:
488
489 def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
490 kv_cache_spec: KVCacheSpec,
491 - available_memory: int) -> KVCacheConfig:
491 + available_memory: int,
492 + num_layers: int) -> KVCacheConfig:
493 """
494 Generates the KV cache configuration for a model with one type of KV cache.
495 Divide the available memory equally among all layers.
@@ -497,6 +498,7 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
498 vllm_config: The global VllmConfig
499 kv_cache_spec: The kv cache spec of the model
500 available_memory: Memory available for KV cache in bytes.
501 + num_layers: The number of layers in the model.
502
503 Returns:
504 The generated KVCacheConfig
@@ -506,7 +508,7 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
508 assert len(page_sizes) == 1
509 page_size = page_sizes.pop()
510
509 - num_blocks = int(available_memory // page_size // len(kv_cache_spec))
511 + num_blocks = int(available_memory // page_size // num_layers)
512 num_blocks = max(num_blocks, 0)
513
514 if vllm_config.cache_config.num_gpu_blocks_override is not None:
@@ -536,25 +538,36 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
538 return kv_cache_config
539
540
539 -def get_kv_cache_config(vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec,
540 - available_memory: int) -> KVCacheConfig:
541 +def get_kv_cache_configs(vllm_config: VllmConfig,
542 + kv_cache_specs: List[KVCacheSpec],
543 + available_memory: int) -> List[KVCacheConfig]:
544 """
545 Generates the KV cache configuration for a model
546 TODO: support hybrid models with more than one type of KV cache.
547
548 Args:
549 vllm_config: The global VllmConfig
547 - kv_cache_spec: The kv cache spec of the model
550 + kv_cache_specs: The kv cache specs of the model
551 available_memory: Memory available for KV cache in bytes.
552
553 Returns:
551 - The generated KVCacheConfig
554 + The generated KVCacheConfigs
555 """
553 - check_enough_kv_cache_memory(vllm_config, kv_cache_spec, available_memory)
554 - if is_kv_cache_type_uniform(kv_cache_spec):
555 - # KV cache of all layers are the same, which is true for most models.
556 - # Allocate the same amount of memory for each layer.
557 - return _get_kv_cache_config_uniform_type(vllm_config, kv_cache_spec,
558 - available_memory)
559 - else:
560 - raise NotImplementedError
556 + # Use the max number of layers to conservatively determine
557 + # the number of blocks.
558 + num_layers = max(len(kv_cache_spec) for kv_cache_spec in kv_cache_specs)
559 + kv_cache_configs = []
560 + for kv_cache_spec in kv_cache_specs:
561 + check_enough_kv_cache_memory(vllm_config, kv_cache_spec,
562 + available_memory)
563 + if is_kv_cache_type_uniform(kv_cache_spec):
564 + # KV cache of all layers are the same, which is true for
565 + # most models. Allocate the same amount of memory for
566 + # each layer.
567 + kv_cache_configs.append(
568 + _get_kv_cache_config_uniform_type(vllm_config, kv_cache_spec,
569 + available_memory,
570 + num_layers))
571 + else:
572 + raise NotImplementedError
573 + return kv_cache_configs
vllm/v1/engine/core.py
+12 -7
@@ -16,7 +16,7 @@ from vllm.logger import init_logger
16 from vllm.transformers_utils.config import (
17 maybe_register_config_serialize_by_value)
18 from vllm.utils import get_exception_traceback, zmq_socket_ctx
19 -from vllm.v1.core.kv_cache_utils import get_kv_cache_config
19 +from vllm.v1.core.kv_cache_utils import get_kv_cache_configs
20 from vllm.v1.core.scheduler import Scheduler
21 from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
22 EngineCoreRequestType)
@@ -73,20 +73,25 @@ class EngineCore:
73 start = time.time()
74
75 # Get all kv cache needed by the model
76 - kv_cache_spec = self.model_executor.get_kv_cache_spec()
76 + kv_cache_specs = self.model_executor.get_kv_cache_specs()
77
78 # Profiles the peak memory usage of the model to determine how much
79 # memory can be allocated for kv cache.
80 - availble_gpu_memory = self.model_executor.determine_available_memory()
80 + available_gpu_memory = self.model_executor.determine_available_memory()
81
82 # Get the kv cache tensor size
83 - kv_cache_config = get_kv_cache_config(vllm_config, kv_cache_spec,
84 - availble_gpu_memory)
85 - num_gpu_blocks = kv_cache_config.num_blocks
83 + kv_cache_configs = get_kv_cache_configs(vllm_config, kv_cache_specs,
84 + available_gpu_memory)
85 + num_gpu_blocks_set = set(config.num_blocks
86 + for config in kv_cache_configs)
87 + assert len(num_gpu_blocks_set) == 1, (
88 + f"num_gpu_blocks need to be the same across workers, "
89 + f"but they are different: {num_gpu_blocks_set}")
90 + num_gpu_blocks = num_gpu_blocks_set.pop()
91 num_cpu_blocks = 0
92
93 # Initialize kv cache and warmup the execution
89 - self.model_executor.initialize(kv_cache_config)
94 + self.model_executor.initialize(kv_cache_configs)
95
96 elapsed = time.time() - start
97 logger.info(("init engine (profile, create kv cache, "
vllm/v1/executor/abstract.py
+5 -7
@@ -1,6 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from typing import Type
3 +from typing import List, Type
4
5 from vllm.config import VllmConfig
6 from vllm.executor.executor_base import ExecutorBase
@@ -48,12 +48,12 @@ class Executor(ExecutorBase):
48 f"{distributed_executor_backend}")
49 return executor_class
50
51 - def initialize(self, kv_cache_config: KVCacheConfig) -> None:
51 + def initialize(self, kv_cache_configs: List[KVCacheConfig]) -> None:
52 """
53 Initialize the KV caches and begin the model execution loop of the
54 underlying workers.
55 """
56 - self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
56 + self.collective_rpc("initialize_cache", args=(kv_cache_configs, ))
57 self.collective_rpc("compile_or_warm_up_model")
58
59 def determine_available_memory(self) -> int: # in bytes
@@ -63,11 +63,9 @@ class Executor(ExecutorBase):
63 # operators can be applied to all workers.
64 return min(output)
65
66 - def get_kv_cache_spec(self) -> KVCacheSpec:
66 + def get_kv_cache_specs(self) -> List[KVCacheSpec]:
67 output = self.collective_rpc("get_kv_cache_spec")
68 - for x in output:
69 - assert x == output[0]
70 - return output[0]
68 + return output
69
70 def execute_model(
71 self,
vllm/v1/worker/gpu_model_runner.py
+15 -1
@@ -12,7 +12,7 @@ import torch.nn as nn
12 from vllm.attention.backends.abstract import AttentionType
13 from vllm.attention.layer import Attention
14 from vllm.config import CompilationLevel, VllmConfig
15 -from vllm.distributed.parallel_state import graph_capture
15 +from vllm.distributed.parallel_state import get_pp_group, graph_capture
16 from vllm.forward_context import set_forward_context
17 from vllm.inputs import INPUT_REGISTRY
18 from vllm.logger import init_logger
@@ -21,6 +21,7 @@ from vllm.model_executor.model_loader import get_model
21 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
22 from vllm.multimodal.utils import group_mm_inputs_by_modality
23 from vllm.sampling_params import SamplingType
24 +from vllm.sequence import IntermediateTensors
25 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
26 LayerBlockType, cdiv, is_pin_memory_available)
27 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
@@ -773,6 +774,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
774 def execute_model(
775 self,
776 scheduler_output: "SchedulerOutput",
777 + intermediate_tensors: Optional[IntermediateTensors] = None,
778 ) -> ModelRunnerOutput:
779 batch_changed = self._update_states(scheduler_output)
780
@@ -831,8 +833,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
833 positions=positions,
834 kv_caches=self.kv_caches,
835 attn_metadata=None,
836 + intermediate_tensors=intermediate_tensors,
837 inputs_embeds=inputs_embeds,
838 )
839 + if not get_pp_group().is_last_rank:
840 + return hidden_states
841 hidden_states = hidden_states[:num_scheduled_tokens]
842 sample_hidden_states = hidden_states[logits_indices]
843 logits = self.model.compute_logits(sample_hidden_states, None)
@@ -1007,12 +1012,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1012 positions = self.mrope_positions[:, :num_tokens]
1013 else:
1014 positions = self.positions[:num_tokens]
1015 + intermediate_tensors = None
1016 + if not get_pp_group().is_first_rank:
1017 + intermediate_tensors = self.model.make_empty_intermediate_tensors(
1018 + batch_size=num_tokens,
1019 + dtype=self.model_config.dtype,
1020 + device=self.device)
1021 with set_forward_context(None, self.vllm_config):
1022 hidden_states = model(
1023 input_ids=input_ids,
1024 positions=positions,
1025 kv_caches=kv_caches,
1026 attn_metadata=None,
1027 + intermediate_tensors=intermediate_tensors,
1028 inputs_embeds=inputs_embeds,
1029 )
1030 return hidden_states
@@ -1142,6 +1154,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1154 # Trigger compilation for general shape.
1155 hidden_states = self._dummy_run(self.max_num_tokens,
1156 dummy_kv_caches)
1157 + if not get_pp_group().is_last_rank:
1158 + return hidden_states
1159 hidden_states = hidden_states[logit_indices]
1160 logits = self.model.compute_logits(hidden_states, None)
1161 # TODO(woosuk): Consider the memory usage of the sampler.
vllm/v1/worker/gpu_worker.py
+3 -2
@@ -2,7 +2,7 @@
2 """A GPU worker class."""
3 import gc
4 import os
5 -from typing import TYPE_CHECKING, Optional
5 +from typing import TYPE_CHECKING, List, Optional
6
7 import torch
8 import torch.distributed
@@ -194,8 +194,9 @@ class Worker:
194 def get_kv_cache_spec(self) -> KVCacheSpec:
195 return self.model_runner.get_kv_cache_spec()
196
197 - def initialize_cache(self, kv_cache_config: KVCacheConfig) -> None:
197 + def initialize_cache(self, kv_cache_configs: List[KVCacheConfig]) -> None:
198 """Allocate GPU KV cache with the specified kv_cache_config."""
199 + kv_cache_config = kv_cache_configs[self.rank]
200 if self.vllm_config.model_config.enable_sleep_mode:
201 allocator = CuMemAllocator.get_instance()
202 context = allocator.use_memory_pool(tag="kv_cache")