@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69672f116c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[core][distributed] simplify code to support pipeline parallel (#6406)

youkaichao committed Jul 14, 2024 at 21:20 UTC 69672f116cf83dbcfd2d470a959dfe123df4d301
5 files changed +107 -61
.buildkite/test-pipeline.yaml
+1 -3
@@ -46,9 +46,7 @@ steps:
46 fast_check: true
47 commands:
48 - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
49 - - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
50 - - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
51 - - VLLM_ATTENTION_BACKEND=FLASHINFER pytest -v -s basic_correctness/test_basic_correctness.py
49 + - pytest -v -s basic_correctness/test_basic_correctness.py
50 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
51 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
52 - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
tests/basic_correctness/test_basic_correctness.py
+8 -3
@@ -28,10 +28,8 @@ def test_vllm_gc_ed():
28 assert weak_llm() is None
29
30
31 -@pytest.mark.skipif(is_hip()
32 - and os.getenv("VLLM_ATTENTION_BACKEND") == "FLASHINFER",
33 - reason="Flashinfer does not support ROCm/HIP.")
31 @pytest.mark.parametrize("model", MODELS)
32 +@pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
33 @pytest.mark.parametrize("dtype", ["half"])
34 @pytest.mark.parametrize("max_tokens", [5])
35 @pytest.mark.parametrize("enforce_eager", [False, True])
@@ -40,10 +38,17 @@ def test_models(
38 vllm_runner,
39 example_prompts,
40 model: str,
41 + backend: str,
42 dtype: str,
43 max_tokens: int,
44 enforce_eager: bool,
45 ) -> None:
46 +
47 + if backend == "FLASHINFER" and is_hip():
48 + pytest.skip("Flashinfer does not support ROCm/HIP.")
49 +
50 + os.environ["VLLM_ATTENTION_BACKEND"] = backend
51 +
52 with hf_runner(model, dtype=dtype) as hf_model:
53 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
54
vllm/model_executor/models/gpt2.py
+20 -27
@@ -27,7 +27,6 @@ from vllm.attention import Attention, AttentionMetadata
27 from vllm.config import CacheConfig
28 from vllm.distributed.parallel_state import (
29 get_pp_group, get_tensor_model_parallel_world_size)
30 -from vllm.distributed.utils import get_pp_indices
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 QKVParallelLinear,
@@ -42,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, SamplerOutput
43
44 +from .utils import is_pp_missing_parameter, make_layers
45 +
46
47 class GPT2Attention(nn.Module):
48
@@ -183,18 +184,9 @@ class GPT2Model(nn.Module):
184 self.embed_dim = config.hidden_size
185 self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
186 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
186 - self.start_layer, self.end_layer = get_pp_indices(
187 + self.start_layer, self.end_layer, self.h = make_layers(
188 config.num_hidden_layers,
188 - get_pp_group().rank_in_group,
189 - get_pp_group().world_size)
190 - self.h = nn.ModuleList(
191 - [nn.Identity() for _ in range(self.start_layer)] + [
192 - GPT2Block(config, cache_config, quant_config)
193 - for _ in range(self.start_layer, self.end_layer)
194 - ] + [
195 - nn.Identity()
196 - for _ in range(self.end_layer, config.num_hidden_layers)
197 - ])
189 + lambda: GPT2Block(config, cache_config, quant_config))
190 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
191
192 def forward(
@@ -291,19 +283,20 @@ class GPT2LMHeadModel(nn.Module):
283 continue
284 if not name.startswith("transformer."):
285 name = "transformer." + name
294 - try:
295 - param = params_dict[name]
296 - # The HF's GPT-2 implementation uses Conv1D instead of Linear.
297 - # Because of this, we need to transpose the weights.
298 - # Note(zhuohan): the logic below might break quantized models.
299 - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
300 - if conv1d_weight_name not in name:
301 - continue
302 - if not name.endswith(".weight"):
303 - continue
304 - loaded_weight = loaded_weight.t()
305 - weight_loader = getattr(param, "weight_loader",
306 - default_weight_loader)
307 - weight_loader(param, loaded_weight)
308 - except KeyError:
286 +
287 + if is_pp_missing_parameter(name, self):
288 continue
289 +
290 + param = params_dict[name]
291 + # The HF's GPT-2 implementation uses Conv1D instead of Linear.
292 + # Because of this, we need to transpose the weights.
293 + # Note(zhuohan): the logic below might break quantized models.
294 + for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
295 + if conv1d_weight_name not in name:
296 + continue
297 + if not name.endswith(".weight"):
298 + continue
299 + loaded_weight = loaded_weight.t()
300 + weight_loader = getattr(param, "weight_loader",
301 + default_weight_loader)
302 + weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+22 -28
@@ -29,8 +29,7 @@ from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig, LoRAConfig
32 -from vllm.distributed import (get_pp_group, get_pp_indices,
33 - get_tensor_model_parallel_rank,
32 +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -51,6 +50,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
50 from vllm.utils import is_hip, print_warning_once
51
52 from .interfaces import SupportsLoRA
53 +from .utils import is_pp_missing_parameter, make_layers
54
55
56 class LlamaMLP(nn.Module):
@@ -262,20 +262,11 @@ class LlamaModel(nn.Module):
262 config.hidden_size,
263 org_num_embeddings=config.vocab_size,
264 )
265 - self.start_layer, self.end_layer = get_pp_indices(
265 + self.start_layer, self.end_layer, self.layers = make_layers(
266 config.num_hidden_layers,
267 - get_pp_group().rank_in_group,
268 - get_pp_group().world_size)
269 - self.layers = nn.ModuleList(
270 - [nn.Identity() for _ in range(self.start_layer)] + [
271 - LlamaDecoderLayer(config=config,
272 - cache_config=cache_config,
273 - quant_config=quant_config)
274 - for _ in range(self.start_layer, self.end_layer)
275 - ] + [
276 - nn.Identity()
277 - for _ in range(self.end_layer, config.num_hidden_layers)
278 - ])
267 + lambda: LlamaDecoderLayer(config=config,
268 + cache_config=cache_config,
269 + quant_config=quant_config))
270 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
271
272 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -455,12 +446,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
446 # Skip loading extra bias for GPTQ models.
447 if name.endswith(".bias") and name not in params_dict:
448 continue
458 - try:
459 - param = params_dict[name]
460 - weight_loader = param.weight_loader
461 - weight_loader(param, loaded_weight, shard_id)
462 - except KeyError:
463 - pass
449 +
450 + if is_pp_missing_parameter(name, self):
451 + continue
452 +
453 + param = params_dict[name]
454 + weight_loader = param.weight_loader
455 + weight_loader(param, loaded_weight, shard_id)
456 +
457 break
458 else:
459 # Skip loading extra bias for GPTQ models.
@@ -479,13 +472,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
472 continue
473 else:
474 name = remapped_kv_scale_name
482 - try:
483 - param = params_dict[name]
484 - weight_loader = getattr(param, "weight_loader",
485 - default_weight_loader)
486 - weight_loader(param, loaded_weight)
487 - except KeyError:
488 - pass
475 +
476 + if is_pp_missing_parameter(name, self):
477 + continue
478 +
479 + param = params_dict[name]
480 + weight_loader = getattr(param, "weight_loader",
481 + default_weight_loader)
482 + weight_loader(param, loaded_weight)
483
484 # If this function is called, it should always initialize KV cache scale
485 # factors (or else raise an exception). Thus, handled exceptions should
vllm/model_executor/models/utils.py
+56
@@ -1,3 +1,5 @@
1 +from typing import Callable, Dict, List, Tuple
2 +
3 import torch
4
5 from vllm.multimodal import BatchedTensors
@@ -39,3 +41,57 @@ def merge_vision_embeddings(input_ids: torch.Tensor,
41 inputs_embeds[mask] = torch.cat(vision_embeddings)
42
43 return inputs_embeds
44 +
45 +
46 +class PPMissingLayer(torch.nn.Identity):
47 + """
48 + A placeholder layer for missing layers in a pipeline parallel model.
49 + """
50 +
51 + def __init__(self, *args, **kwargs):
52 + super().__init__()
53 +
54 +
55 +def make_layers(
56 + num_hidden_layers: int, layer_fn: Callable[[], torch.nn.Module]
57 +) -> Tuple[int, int, torch.nn.ModuleList]:
58 + """Make a list of layers with the given layer function, taking
59 + pipeline parallelism into account.
60 + """
61 + from vllm.distributed.parallel_state import get_pp_group
62 + from vllm.distributed.utils import get_pp_indices
63 + start_layer, end_layer = get_pp_indices(num_hidden_layers,
64 + get_pp_group().rank_in_group,
65 + get_pp_group().world_size)
66 + modules = torch.nn.ModuleList(
67 + [PPMissingLayer() for _ in range(start_layer)] +
68 + [layer_fn() for _ in range(start_layer, end_layer)] +
69 + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
70 + return start_layer, end_layer, modules
71 +
72 +
73 +# NOTE: don't use lru_cache here because it can prevent garbage collection
74 +_model_to_pp_missing_layer_names: Dict[int, List[str]] = {}
75 +
76 +
77 +def get_pp_missing_layer_names(model: torch.nn.Module) -> List[str]:
78 + """Get the names of the missing layers in a pipeline parallel model."""
79 + model_id = id(model)
80 + if model_id in _model_to_pp_missing_layer_names:
81 + return _model_to_pp_missing_layer_names[model_id]
82 +
83 + missing_layer_names = []
84 + for name, module in model.named_modules():
85 + if isinstance(module, PPMissingLayer):
86 + missing_layer_names.append(name)
87 + _model_to_pp_missing_layer_names[model_id] = missing_layer_names
88 +
89 + return missing_layer_names
90 +
91 +
92 +def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool:
93 + """Check if a parameter is missing in a pipeline parallel model."""
94 + for missing_layer_name in get_pp_missing_layer_names(model):
95 + if name.startswith(missing_layer_name):
96 + return True
97 + return False