A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Implement sharded state loader (#4690)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Aurick Qiao committed
May 16, 2024 at 01:11 UTC
30e754390c2a8a7198f472386d35ee1ec9443e4a
7 files changed
+351
examples/save_sharded_state.py
new
+75
@@ -0,0 +1,75 @@
1
+"""
2
+Saves each worker's model state dict directly to a checkpoint, which enables a
3
+fast load path for large tensor-parallel models where each worker only needs to
4
+read its own shard rather than the entire checkpoint.
5
+
6
+Example usage:
7
+
8
+python save_sharded_state.py \
9
+ --model /path/to/load \
10
+ --quantization deepspeedfp \
11
+ --tensor-parallel-size 8 \
12
+ --output /path/to/save
13
+
14
+Then, the model can be loaded with
15
+
16
+llm = LLM(
17
+ model="/path/to/save",
18
+ load_format="sharded_state",
19
+ quantization="deepspeedfp",
20
+ tensor_parallel_size=8,
21
+)
22
+"""
23
+import argparse
24
+import dataclasses
25
+import os
26
+import shutil
27
+from pathlib import Path
28
+
29
+from vllm import LLM, EngineArgs
30
+
31
+parser = argparse.ArgumentParser()
32
+EngineArgs.add_cli_args(parser)
33
+parser.add_argument("--output",
34
+ "-o",
35
+ required=True,
36
+ type=str,
37
+ help="path to output checkpoint")
38
+parser.add_argument("--file-pattern",
39
+ type=str,
40
+ help="string pattern of saved filenames")
41
+parser.add_argument("--max-file-size",
42
+ type=str,
43
+ default=5 * 1024**3,
44
+ help="max size (in bytes) of each safetensors file")
45
+
46
+
47
+def main(args):
48
+ engine_args = EngineArgs.from_cli_args(args)
49
+ if engine_args.enable_lora:
50
+ raise ValueError("Saving with enable_lora=True is not supported!")
51
+ model_path = engine_args.model
52
+ if not Path(model_path).is_dir():
53
+ raise ValueError("model path must be a local directory")
54
+ # Create LLM instance from arguments
55
+ llm = LLM(**dataclasses.asdict(engine_args))
56
+ # Prepare output directory
57
+ Path(args.output).mkdir(exist_ok=True)
58
+ # Dump worker states to output directory
59
+ model_executor = llm.llm_engine.model_executor
60
+ model_executor.save_sharded_state(path=args.output,
61
+ pattern=args.file_pattern,
62
+ max_size=args.max_file_size)
63
+ # Copy metadata files to output directory
64
+ for file in os.listdir(model_path):
65
+ if os.path.splitext(file)[1] not in (".bin", ".pt", ".safetensors"):
66
+ if os.path.isdir(os.path.join(model_path, file)):
67
+ shutil.copytree(os.path.join(model_path, file),
68
+ os.path.join(args.output, file))
69
+ else:
70
+ shutil.copy(os.path.join(model_path, file), args.output)
71
+
72
+
73
+if __name__ == "__main__":
74
+ args = parser.parse_args()
75
+ main(args)
tests/test_sharded_state_loader.py
new
+90
@@ -0,0 +1,90 @@
1
+import os
2
+import shutil
3
+from tempfile import TemporaryDirectory
4
+
5
+import pytest
6
+import torch
7
+from huggingface_hub import snapshot_download
8
+
9
+from vllm import LLM, SamplingParams
10
+from vllm.model_executor.model_loader.loader import ShardedStateLoader
11
+
12
+prompts = [
13
+ "Hello, my name is",
14
+ "The president of the United States is",
15
+ "The capital of France is",
16
+ "The future of AI is",
17
+]
18
+
19
+# Create a sampling params object.
20
+sampling_params = SamplingParams(
21
+ temperature=0.8,
22
+ top_p=0.95,
23
+ seed=0,
24
+ max_tokens=256,
25
+ ignore_eos=True,
26
+)
27
+
28
+
29
+def test_filter_subtensors():
30
+ state_dict = {
31
+ "a": torch.empty(2),
32
+ "b": torch.empty((2, 4)),
33
+ "c": torch.empty((2, 4, 8)),
34
+ }
35
+ state_dict.update({
36
+ "x": state_dict["b"],
37
+ "y": state_dict["c"][1, 2, :],
38
+ "z": state_dict["c"][1, :, 4],
39
+ })
40
+ filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
41
+ assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
42
+ for key, tensor in filtered_state_dict.items():
43
+ assert tensor.equal(state_dict[key])
44
+
45
+
46
+@pytest.mark.parametrize("enable_lora", [False, True])
47
+def test_sharded_state_loader(enable_lora):
48
+ weights_patterns = ("*.bin", "*.pt", "*.safetensors")
49
+
50
+ with TemporaryDirectory() as cache_dir, TemporaryDirectory() as output_dir:
51
+ input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
52
+ cache_dir=cache_dir)
53
+
54
+ llm = LLM(
55
+ model=input_dir,
56
+ worker_use_ray=True,
57
+ gpu_memory_utilization=0.3,
58
+ )
59
+
60
+ # Dump worker states to output directory
61
+ model_executor = llm.llm_engine.model_executor
62
+ model_executor.save_sharded_state(path=output_dir)
63
+ # Copy metadata files to output directory
64
+ for file in os.listdir(input_dir):
65
+ if not any(file.endswith(ext) for ext in weights_patterns):
66
+ shutil.copy(f"{input_dir}/{file}", output_dir)
67
+ del llm.llm_engine.model_executor
68
+
69
+ llm_before = LLM(
70
+ model=input_dir,
71
+ worker_use_ray=True,
72
+ enable_lora=enable_lora,
73
+ gpu_memory_utilization=0.3,
74
+ )
75
+ gen_before = llm_before.generate(prompts, sampling_params)
76
+ out_before = [gen.outputs[0].__dict__ for gen in gen_before]
77
+ del llm_before.llm_engine.model_executor
78
+
79
+ llm_after = LLM(
80
+ model=output_dir,
81
+ worker_use_ray=True,
82
+ enable_lora=enable_lora,
83
+ gpu_memory_utilization=0.3,
84
+ load_format="sharded_state",
85
+ )
86
+ gen_after = llm_after.generate(prompts, sampling_params)
87
+ out_after = [gen.outputs[0].__dict__ for gen in gen_after]
88
+ del llm_after.llm_engine.model_executor
89
+
90
+ assert out_before == out_after
vllm/config.py
+1
@@ -463,6 +463,7 @@ class LoadFormat(str, enum.Enum):
463
NPCACHE = "npcache"
464
DUMMY = "dummy"
465
TENSORIZER = "tensorizer"
466
+ SHARDED_STATE = "sharded_state"
467
468
469
@dataclass
vllm/executor/distributed_gpu_executor.py
+11
@@ -77,6 +77,17 @@ class DistributedGPUExecutor(GPUExecutor):
77
def list_loras(self) -> Set[int]:
78
return self._run_workers("list_loras")
79
80
+ def save_sharded_state(
81
+ self,
82
+ path: str,
83
+ pattern: Optional[str] = None,
84
+ max_size: Optional[int] = None,
85
+ ) -> None:
86
+ self._run_workers("save_sharded_state",
87
+ path=path,
88
+ pattern=pattern,
89
+ max_size=max_size)
90
+
91
@abstractmethod
92
def _run_workers(
93
self,
vllm/model_executor/model_loader/loader.py
+148
@@ -1,4 +1,5 @@
1
# ruff: noqa: SIM117
2
+import collections
3
import copy
4
import glob
5
import os
@@ -366,6 +367,150 @@ class TensorizerLoader(BaseModelLoader):
367
cache_config)
368
369
370
+class ShardedStateLoader(BaseModelLoader):
371
+ """
372
+ Model loader that directly loads each worker's model state dict, which
373
+ enables a fast load path for large tensor-parallel models where each worker
374
+ only needs to read its own shard rather than the entire checkpoint. See
375
+ `examples/save_sharded_states.py` for creating a sharded checkpoint.
376
+ """
377
+
378
+ DEFAULT_PATTERN = "model-rank-{rank}-part-{part}.safetensors"
379
+
380
+ def __init__(self, load_config: LoadConfig):
381
+ super().__init__(load_config)
382
+ extra_config = ({} if load_config.model_loader_extra_config is None
383
+ else load_config.model_loader_extra_config.copy())
384
+ self.pattern = extra_config.pop("pattern", self.DEFAULT_PATTERN)
385
+ if extra_config:
386
+ raise ValueError(f"Unexpected extra config keys for load format "
387
+ f"{load_config.load_format}: "
388
+ f"{load_config.model_loader_extra_config.keys()}")
389
+
390
+ @staticmethod
391
+ def _filter_subtensors(
392
+ tensors: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
393
+ """
394
+ Filter out all tensors that share the same memory or a subset of the
395
+ memory of another tensor.
396
+ """
397
+ same_storage_groups = collections.defaultdict(list)
398
+ for key, tensor in tensors.items():
399
+ if tensor.numel():
400
+ ptr = tensor.untyped_storage().data_ptr()
401
+ same_storage_groups[tensor.device, ptr].append((key, tensor))
402
+
403
+ def get_end_ptr(tensor: torch.Tensor) -> int:
404
+ return tensor.view(-1)[-1].data_ptr() + tensor.element_size()
405
+
406
+ result = {}
407
+ for group in same_storage_groups.values():
408
+ for k, t in group:
409
+ a, b = t.data_ptr(), get_end_ptr(t)
410
+ for k2, t2 in group:
411
+ if not t2.is_contiguous():
412
+ continue
413
+ a2, b2 = t2.data_ptr(), get_end_ptr(t2)
414
+ if a < a2 or b2 < b:
415
+ continue
416
+ if a2 < a or b < b2 or not t.is_contiguous():
417
+ break # t2 covers strictly more memory than t.
418
+ if k2 < k:
419
+ # Same tensors, keep the one with the smaller key.
420
+ break
421
+ else:
422
+ result[k] = t
423
+ return result
424
+
425
+ def load_model(self, *, model_config: ModelConfig,
426
+ device_config: DeviceConfig,
427
+ lora_config: Optional[LoRAConfig],
428
+ vision_language_config: Optional[VisionLanguageConfig],
429
+ parallel_config: ParallelConfig,
430
+ scheduler_config: SchedulerConfig,
431
+ cache_config: CacheConfig) -> nn.Module:
432
+ from safetensors.torch import safe_open
433
+
434
+ from vllm.distributed import get_tensor_model_parallel_rank
435
+ with set_default_torch_dtype(model_config.dtype):
436
+ with torch.device(device_config.device):
437
+ model = _initialize_model(model_config, self.load_config,
438
+ lora_config, vision_language_config,
439
+ cache_config)
440
+ rank = get_tensor_model_parallel_rank()
441
+ pattern = os.path.join(
442
+ model_config.model,
443
+ self.pattern.format(rank=rank, part="*"),
444
+ )
445
+ filepaths = glob.glob(pattern)
446
+ if not filepaths:
447
+ # TODO: support un-sharded checkpoints too
448
+ raise ValueError(
449
+ f"Could not find checkpoint files '{pattern}', only "
450
+ f"pre-sharded checkpoints are currently supported!")
451
+ state_dict = self._filter_subtensors(model.state_dict())
452
+ for path in filepaths:
453
+ with safe_open(path, framework="pt") as f:
454
+ for key in f.keys(): # noqa: SIM118
455
+ tensor = f.get_tensor(key)
456
+ # If loading with LoRA enabled, additional padding may
457
+ # be added to certain parameters. We only load into a
458
+ # narrowed view of the parameter data.
459
+ param_data = state_dict[key].data
460
+ param_shape = state_dict[key].shape
461
+ for dim, size in enumerate(tensor.shape):
462
+ if size < param_shape[dim]:
463
+ param_data = param_data.narrow(dim, 0, size)
464
+ if tensor.shape != param_shape:
465
+ logger.warning(
466
+ "loading tensor of shape %s into "
467
+ "parameter '%s' of shape %s", tensor.shape,
468
+ key, param_shape)
469
+ param_data.copy_(tensor)
470
+ state_dict.pop(key)
471
+ if state_dict:
472
+ raise ValueError(
473
+ f"Missing keys {tuple(state_dict)} in loaded state!")
474
+ return model.eval()
475
+
476
+ @staticmethod
477
+ def save_model(
478
+ model: torch.nn.Module,
479
+ path: str,
480
+ pattern: Optional[str] = None,
481
+ max_size: Optional[int] = None,
482
+ ) -> None:
483
+ from safetensors.torch import save_file
484
+
485
+ from vllm.distributed import get_tensor_model_parallel_rank
486
+ if pattern is None:
487
+ pattern = ShardedStateLoader.DEFAULT_PATTERN
488
+ rank = get_tensor_model_parallel_rank()
489
+ part_idx = 0
490
+ total_size = 0
491
+ state_dict = ShardedStateLoader._filter_subtensors(model.state_dict())
492
+ state_dict_part: Dict[str, torch.Tensor] = {}
493
+ for key, tensor in state_dict.items():
494
+ param_size = tensor.nelement() * tensor.element_size()
495
+ if max_size is not None and total_size + param_size > max_size:
496
+ filename = pattern.format(rank=rank, part=part_idx)
497
+ save_file(
498
+ state_dict_part,
499
+ os.path.join(path, filename),
500
+ )
501
+ part_idx += 1
502
+ total_size = 0
503
+ state_dict_part = {}
504
+ state_dict_part[key] = tensor
505
+ total_size += param_size
506
+ if len(state_dict_part) > 0:
507
+ filename = pattern.format(rank=rank, part=part_idx)
508
+ save_file(
509
+ state_dict_part,
510
+ os.path.join(path, filename),
511
+ )
512
+
513
+
514
def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
515
"""Get a model loader based on the load format."""
516
@@ -378,4 +523,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
523
if load_config.load_format == LoadFormat.TENSORIZER:
524
return TensorizerLoader(load_config)
525
526
+ if load_config.load_format == LoadFormat.SHARDED_STATE:
527
+ return ShardedStateLoader(load_config)
528
+
529
return DefaultModelLoader(load_config)
vllm/worker/model_runner.py
+14
@@ -182,6 +182,20 @@ class ModelRunner:
182
"but the KV cache data type is not FP8. "
183
"KV cache scaling factors will not be used.")
184
185
+ def save_sharded_state(
186
+ self,
187
+ path: str,
188
+ pattern: Optional[str] = None,
189
+ max_size: Optional[int] = None,
190
+ ) -> None:
191
+ from vllm.model_executor.model_loader.loader import ShardedStateLoader
192
+ ShardedStateLoader.save_model(
193
+ self.model,
194
+ path,
195
+ pattern=pattern,
196
+ max_size=max_size,
197
+ )
198
+
199
def get_max_block_per_batch(self) -> int:
200
block_size = self.block_size
201
return (self.max_seq_len_to_capture + block_size - 1) // block_size
vllm/worker/worker.py
+12
@@ -119,6 +119,18 @@ class Worker(WorkerBase):
119
def load_model(self):
120
self.model_runner.load_model()
121
122
+ def save_sharded_state(
123
+ self,
124
+ path: str,
125
+ pattern: Optional[str] = None,
126
+ max_size: Optional[int] = None,
127
+ ) -> None:
128
+ self.model_runner.save_sharded_state(
129
+ path,
130
+ pattern=pattern,
131
+ max_size=max_size,
132
+ )
133
+
134
@torch.inference_mode()
135
def determine_num_available_blocks(self) -> Tuple[int, int]:
136
"""Profiles the peak memory usage of the model to determine how many