@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 30e754390c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Implement sharded state loader (#4690)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Aurick Qiao committed May 16, 2024 at 01:11 UTC 30e754390c2a8a7198f472386d35ee1ec9443e4a
7 files changed +351
examples/save_sharded_state.py new
+75
@@ -0,0 +1,75 @@
1 +"""
2 +Saves each worker's model state dict directly to a checkpoint, which enables a
3 +fast load path for large tensor-parallel models where each worker only needs to
4 +read its own shard rather than the entire checkpoint.
5 +
6 +Example usage:
7 +
8 +python save_sharded_state.py \
9 + --model /path/to/load \
10 + --quantization deepspeedfp \
11 + --tensor-parallel-size 8 \
12 + --output /path/to/save
13 +
14 +Then, the model can be loaded with
15 +
16 +llm = LLM(
17 + model="/path/to/save",
18 + load_format="sharded_state",
19 + quantization="deepspeedfp",
20 + tensor_parallel_size=8,
21 +)
22 +"""
23 +import argparse
24 +import dataclasses
25 +import os
26 +import shutil
27 +from pathlib import Path
28 +
29 +from vllm import LLM, EngineArgs
30 +
31 +parser = argparse.ArgumentParser()
32 +EngineArgs.add_cli_args(parser)
33 +parser.add_argument("--output",
34 + "-o",
35 + required=True,
36 + type=str,
37 + help="path to output checkpoint")
38 +parser.add_argument("--file-pattern",
39 + type=str,
40 + help="string pattern of saved filenames")
41 +parser.add_argument("--max-file-size",
42 + type=str,
43 + default=5 * 1024**3,
44 + help="max size (in bytes) of each safetensors file")
45 +
46 +
47 +def main(args):
48 + engine_args = EngineArgs.from_cli_args(args)
49 + if engine_args.enable_lora:
50 + raise ValueError("Saving with enable_lora=True is not supported!")
51 + model_path = engine_args.model
52 + if not Path(model_path).is_dir():
53 + raise ValueError("model path must be a local directory")
54 + # Create LLM instance from arguments
55 + llm = LLM(**dataclasses.asdict(engine_args))
56 + # Prepare output directory
57 + Path(args.output).mkdir(exist_ok=True)
58 + # Dump worker states to output directory
59 + model_executor = llm.llm_engine.model_executor
60 + model_executor.save_sharded_state(path=args.output,
61 + pattern=args.file_pattern,
62 + max_size=args.max_file_size)
63 + # Copy metadata files to output directory
64 + for file in os.listdir(model_path):
65 + if os.path.splitext(file)[1] not in (".bin", ".pt", ".safetensors"):
66 + if os.path.isdir(os.path.join(model_path, file)):
67 + shutil.copytree(os.path.join(model_path, file),
68 + os.path.join(args.output, file))
69 + else:
70 + shutil.copy(os.path.join(model_path, file), args.output)
71 +
72 +
73 +if __name__ == "__main__":
74 + args = parser.parse_args()
75 + main(args)
tests/test_sharded_state_loader.py new
+90
@@ -0,0 +1,90 @@
1 +import os
2 +import shutil
3 +from tempfile import TemporaryDirectory
4 +
5 +import pytest
6 +import torch
7 +from huggingface_hub import snapshot_download
8 +
9 +from vllm import LLM, SamplingParams
10 +from vllm.model_executor.model_loader.loader import ShardedStateLoader
11 +
12 +prompts = [
13 + "Hello, my name is",
14 + "The president of the United States is",
15 + "The capital of France is",
16 + "The future of AI is",
17 +]
18 +
19 +# Create a sampling params object.
20 +sampling_params = SamplingParams(
21 + temperature=0.8,
22 + top_p=0.95,
23 + seed=0,
24 + max_tokens=256,
25 + ignore_eos=True,
26 +)
27 +
28 +
29 +def test_filter_subtensors():
30 + state_dict = {
31 + "a": torch.empty(2),
32 + "b": torch.empty((2, 4)),
33 + "c": torch.empty((2, 4, 8)),
34 + }
35 + state_dict.update({
36 + "x": state_dict["b"],
37 + "y": state_dict["c"][1, 2, :],
38 + "z": state_dict["c"][1, :, 4],
39 + })
40 + filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
41 + assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
42 + for key, tensor in filtered_state_dict.items():
43 + assert tensor.equal(state_dict[key])
44 +
45 +
46 +@pytest.mark.parametrize("enable_lora", [False, True])
47 +def test_sharded_state_loader(enable_lora):
48 + weights_patterns = ("*.bin", "*.pt", "*.safetensors")
49 +
50 + with TemporaryDirectory() as cache_dir, TemporaryDirectory() as output_dir:
51 + input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
52 + cache_dir=cache_dir)
53 +
54 + llm = LLM(
55 + model=input_dir,
56 + worker_use_ray=True,
57 + gpu_memory_utilization=0.3,
58 + )
59 +
60 + # Dump worker states to output directory
61 + model_executor = llm.llm_engine.model_executor
62 + model_executor.save_sharded_state(path=output_dir)
63 + # Copy metadata files to output directory
64 + for file in os.listdir(input_dir):
65 + if not any(file.endswith(ext) for ext in weights_patterns):
66 + shutil.copy(f"{input_dir}/{file}", output_dir)
67 + del llm.llm_engine.model_executor
68 +
69 + llm_before = LLM(
70 + model=input_dir,
71 + worker_use_ray=True,
72 + enable_lora=enable_lora,
73 + gpu_memory_utilization=0.3,
74 + )
75 + gen_before = llm_before.generate(prompts, sampling_params)
76 + out_before = [gen.outputs[0].__dict__ for gen in gen_before]
77 + del llm_before.llm_engine.model_executor
78 +
79 + llm_after = LLM(
80 + model=output_dir,
81 + worker_use_ray=True,
82 + enable_lora=enable_lora,
83 + gpu_memory_utilization=0.3,
84 + load_format="sharded_state",
85 + )
86 + gen_after = llm_after.generate(prompts, sampling_params)
87 + out_after = [gen.outputs[0].__dict__ for gen in gen_after]
88 + del llm_after.llm_engine.model_executor
89 +
90 + assert out_before == out_after
vllm/config.py
+1
@@ -463,6 +463,7 @@ class LoadFormat(str, enum.Enum):
463 NPCACHE = "npcache"
464 DUMMY = "dummy"
465 TENSORIZER = "tensorizer"
466 + SHARDED_STATE = "sharded_state"
467
468
469 @dataclass
vllm/executor/distributed_gpu_executor.py
+11
@@ -77,6 +77,17 @@ class DistributedGPUExecutor(GPUExecutor):
77 def list_loras(self) -> Set[int]:
78 return self._run_workers("list_loras")
79
80 + def save_sharded_state(
81 + self,
82 + path: str,
83 + pattern: Optional[str] = None,
84 + max_size: Optional[int] = None,
85 + ) -> None:
86 + self._run_workers("save_sharded_state",
87 + path=path,
88 + pattern=pattern,
89 + max_size=max_size)
90 +
91 @abstractmethod
92 def _run_workers(
93 self,
vllm/model_executor/model_loader/loader.py
+148
@@ -1,4 +1,5 @@
1 # ruff: noqa: SIM117
2 +import collections
3 import copy
4 import glob
5 import os
@@ -366,6 +367,150 @@ class TensorizerLoader(BaseModelLoader):
367 cache_config)
368
369
370 +class ShardedStateLoader(BaseModelLoader):
371 + """
372 + Model loader that directly loads each worker's model state dict, which
373 + enables a fast load path for large tensor-parallel models where each worker
374 + only needs to read its own shard rather than the entire checkpoint. See
375 + `examples/save_sharded_states.py` for creating a sharded checkpoint.
376 + """
377 +
378 + DEFAULT_PATTERN = "model-rank-{rank}-part-{part}.safetensors"
379 +
380 + def __init__(self, load_config: LoadConfig):
381 + super().__init__(load_config)
382 + extra_config = ({} if load_config.model_loader_extra_config is None
383 + else load_config.model_loader_extra_config.copy())
384 + self.pattern = extra_config.pop("pattern", self.DEFAULT_PATTERN)
385 + if extra_config:
386 + raise ValueError(f"Unexpected extra config keys for load format "
387 + f"{load_config.load_format}: "
388 + f"{load_config.model_loader_extra_config.keys()}")
389 +
390 + @staticmethod
391 + def _filter_subtensors(
392 + tensors: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
393 + """
394 + Filter out all tensors that share the same memory or a subset of the
395 + memory of another tensor.
396 + """
397 + same_storage_groups = collections.defaultdict(list)
398 + for key, tensor in tensors.items():
399 + if tensor.numel():
400 + ptr = tensor.untyped_storage().data_ptr()
401 + same_storage_groups[tensor.device, ptr].append((key, tensor))
402 +
403 + def get_end_ptr(tensor: torch.Tensor) -> int:
404 + return tensor.view(-1)[-1].data_ptr() + tensor.element_size()
405 +
406 + result = {}
407 + for group in same_storage_groups.values():
408 + for k, t in group:
409 + a, b = t.data_ptr(), get_end_ptr(t)
410 + for k2, t2 in group:
411 + if not t2.is_contiguous():
412 + continue
413 + a2, b2 = t2.data_ptr(), get_end_ptr(t2)
414 + if a < a2 or b2 < b:
415 + continue
416 + if a2 < a or b < b2 or not t.is_contiguous():
417 + break # t2 covers strictly more memory than t.
418 + if k2 < k:
419 + # Same tensors, keep the one with the smaller key.
420 + break
421 + else:
422 + result[k] = t
423 + return result
424 +
425 + def load_model(self, *, model_config: ModelConfig,
426 + device_config: DeviceConfig,
427 + lora_config: Optional[LoRAConfig],
428 + vision_language_config: Optional[VisionLanguageConfig],
429 + parallel_config: ParallelConfig,
430 + scheduler_config: SchedulerConfig,
431 + cache_config: CacheConfig) -> nn.Module:
432 + from safetensors.torch import safe_open
433 +
434 + from vllm.distributed import get_tensor_model_parallel_rank
435 + with set_default_torch_dtype(model_config.dtype):
436 + with torch.device(device_config.device):
437 + model = _initialize_model(model_config, self.load_config,
438 + lora_config, vision_language_config,
439 + cache_config)
440 + rank = get_tensor_model_parallel_rank()
441 + pattern = os.path.join(
442 + model_config.model,
443 + self.pattern.format(rank=rank, part="*"),
444 + )
445 + filepaths = glob.glob(pattern)
446 + if not filepaths:
447 + # TODO: support un-sharded checkpoints too
448 + raise ValueError(
449 + f"Could not find checkpoint files '{pattern}', only "
450 + f"pre-sharded checkpoints are currently supported!")
451 + state_dict = self._filter_subtensors(model.state_dict())
452 + for path in filepaths:
453 + with safe_open(path, framework="pt") as f:
454 + for key in f.keys(): # noqa: SIM118
455 + tensor = f.get_tensor(key)
456 + # If loading with LoRA enabled, additional padding may
457 + # be added to certain parameters. We only load into a
458 + # narrowed view of the parameter data.
459 + param_data = state_dict[key].data
460 + param_shape = state_dict[key].shape
461 + for dim, size in enumerate(tensor.shape):
462 + if size < param_shape[dim]:
463 + param_data = param_data.narrow(dim, 0, size)
464 + if tensor.shape != param_shape:
465 + logger.warning(
466 + "loading tensor of shape %s into "
467 + "parameter '%s' of shape %s", tensor.shape,
468 + key, param_shape)
469 + param_data.copy_(tensor)
470 + state_dict.pop(key)
471 + if state_dict:
472 + raise ValueError(
473 + f"Missing keys {tuple(state_dict)} in loaded state!")
474 + return model.eval()
475 +
476 + @staticmethod
477 + def save_model(
478 + model: torch.nn.Module,
479 + path: str,
480 + pattern: Optional[str] = None,
481 + max_size: Optional[int] = None,
482 + ) -> None:
483 + from safetensors.torch import save_file
484 +
485 + from vllm.distributed import get_tensor_model_parallel_rank
486 + if pattern is None:
487 + pattern = ShardedStateLoader.DEFAULT_PATTERN
488 + rank = get_tensor_model_parallel_rank()
489 + part_idx = 0
490 + total_size = 0
491 + state_dict = ShardedStateLoader._filter_subtensors(model.state_dict())
492 + state_dict_part: Dict[str, torch.Tensor] = {}
493 + for key, tensor in state_dict.items():
494 + param_size = tensor.nelement() * tensor.element_size()
495 + if max_size is not None and total_size + param_size > max_size:
496 + filename = pattern.format(rank=rank, part=part_idx)
497 + save_file(
498 + state_dict_part,
499 + os.path.join(path, filename),
500 + )
501 + part_idx += 1
502 + total_size = 0
503 + state_dict_part = {}
504 + state_dict_part[key] = tensor
505 + total_size += param_size
506 + if len(state_dict_part) > 0:
507 + filename = pattern.format(rank=rank, part=part_idx)
508 + save_file(
509 + state_dict_part,
510 + os.path.join(path, filename),
511 + )
512 +
513 +
514 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
515 """Get a model loader based on the load format."""
516
@@ -378,4 +523,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
523 if load_config.load_format == LoadFormat.TENSORIZER:
524 return TensorizerLoader(load_config)
525
526 + if load_config.load_format == LoadFormat.SHARDED_STATE:
527 + return ShardedStateLoader(load_config)
528 +
529 return DefaultModelLoader(load_config)
vllm/worker/model_runner.py
+14
@@ -182,6 +182,20 @@ class ModelRunner:
182 "but the KV cache data type is not FP8. "
183 "KV cache scaling factors will not be used.")
184
185 + def save_sharded_state(
186 + self,
187 + path: str,
188 + pattern: Optional[str] = None,
189 + max_size: Optional[int] = None,
190 + ) -> None:
191 + from vllm.model_executor.model_loader.loader import ShardedStateLoader
192 + ShardedStateLoader.save_model(
193 + self.model,
194 + path,
195 + pattern=pattern,
196 + max_size=max_size,
197 + )
198 +
199 def get_max_block_per_batch(self) -> int:
200 block_size = self.block_size
201 return (self.max_seq_len_to_capture + block_size - 1) // block_size
vllm/worker/worker.py
+12
@@ -119,6 +119,18 @@ class Worker(WorkerBase):
119 def load_model(self):
120 self.model_runner.load_model()
121
122 + def save_sharded_state(
123 + self,
124 + path: str,
125 + pattern: Optional[str] = None,
126 + max_size: Optional[int] = None,
127 + ) -> None:
128 + self.model_runner.save_sharded_state(
129 + path,
130 + pattern=pattern,
131 + max_size=max_size,
132 + )
133 +
134 @torch.inference_mode()
135 def determine_num_available_blocks(self) -> Tuple[int, int]:
136 """Profiles the peak memory usage of the model to determine how many