@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3682e33f9f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[v1] fix compilation cache (#11598)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 30, 2024 at 12:24 UTC 3682e33f9ff9d8baade6112a8e75a77da898f504
4 files changed +69 -14
tests/compile/piecewise/test_toy_llama.py
+13 -2
@@ -7,7 +7,7 @@ if the config `tractable_init` is set to True. Otherwise, the weights are
7 initialized randomly with a fixed seed.
8 """
9 from dataclasses import dataclass
10 -from typing import Optional, Tuple
10 +from typing import Any, List, Optional, Tuple
11
12 import torch
13 from torch import nn
@@ -54,6 +54,16 @@ class LlamaConfig:
54 tractable_init: bool = False
55 random_seed: int = 0
56
57 + def compute_hash(self) -> str:
58 + factors: List[Any] = []
59 + for k, v in self.__dict__.items():
60 + if k == "random_seed":
61 + continue
62 + factors.append((k, v))
63 + factors.sort()
64 + import hashlib
65 + return hashlib.md5(str(factors).encode()).hexdigest()
66 +
67 def __post_init__(self):
68 assert self.mlp_size >= self.hidden_size
69
@@ -263,7 +273,8 @@ def run_model(llama_config,
273 compilation_config = CompilationConfig(
274 level=CompilationLevel.NO_COMPILATION, )
275
266 - vllm_config = VllmConfig(compilation_config=compilation_config)
276 + vllm_config = VllmConfig(compilation_config=compilation_config,
277 + additional_config=llama_config)
278 with set_current_vllm_config(vllm_config):
279 model = LlamaModel(config=llama_config,
280 vllm_config=vllm_config,
vllm/compilation/backends.py
+13 -9
@@ -619,8 +619,10 @@ class PiecewiseBackend:
619 # the entries for different shapes that we need to either
620 # compile or capture cudagraph
621 self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
622 - self.to_be_compiled_sizes: Set[int] = self.compile_sizes.union(
623 - self.capture_sizes)
622 +
623 + # to_be_compiled_sizes tracks the remaining sizes to compile,
624 + # and updates during the compilation process, so we need to copy it
625 + self.to_be_compiled_sizes: Set[int] = self.compile_sizes.copy()
626 for shape in self.compile_sizes.union(self.capture_sizes):
627 self.concrete_size_entries[shape] = ConcreteSizeEntry(
628 runtime_shape=shape,
@@ -628,12 +630,17 @@ class PiecewiseBackend:
630 use_cudagraph=shape in self.capture_sizes,
631 )
632
633 + def check_for_ending_compilation(self):
634 + if self.is_last_graph and not self.to_be_compiled_sizes:
635 + # no specific sizes to compile
636 + # save the hash of the inductor graph for the next run
637 + self.compilation_config.inductor_hash_cache.save_to_file()
638 + end_monitoring_torch_compile(self.vllm_config)
639 +
640 def __call__(self, *args) -> Any:
641 if not self.first_run_finished:
642 self.first_run_finished = True
634 - # no specific sizes to compile
635 - if self.is_last_graph and not self.to_be_compiled_sizes:
636 - end_monitoring_torch_compile(self.vllm_config)
643 + self.check_for_ending_compilation()
644 return self.compiled_graph_for_general_shape(*args)
645
646 runtime_shape = args[self.sym_shape_indices[0]]
@@ -662,10 +669,7 @@ class PiecewiseBackend:
669
670 # finished compilations for all required shapes
671 if self.is_last_graph and not self.to_be_compiled_sizes:
665 -
666 - # save the hash of the inductor graph for the next run
667 - self.compilation_config.inductor_hash_cache.save_to_file()
668 - end_monitoring_torch_compile(self.vllm_config)
672 + self.check_for_ending_compilation()
673
674 if not entry.use_cudagraph:
675 return entry.runnable(*args)
vllm/config.py
+42 -3
@@ -9,8 +9,8 @@ from contextlib import contextmanager
9 from dataclasses import dataclass, field, replace
10 from pathlib import Path
11 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,
12 - Final, List, Literal, Mapping, Optional, Set, Tuple, Type,
13 - Union)
12 + Final, List, Literal, Mapping, Optional, Protocol, Set,
13 + Tuple, Type, Union)
14
15 import torch
16 from pydantic import BaseModel, Field, PrivateAttr
@@ -75,6 +75,12 @@ HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],
75 PretrainedConfig]]
76
77
78 +class SupportsHash(Protocol):
79 +
80 + def compute_hash(self) -> str:
81 + ...
82 +
83 +
84 class ModelConfig:
85 """Configuration for the model.
86
@@ -2969,6 +2975,10 @@ class VllmConfig:
2975 init=True) # type: ignore
2976 kv_transfer_config: KVTransferConfig = field(default=None,
2977 init=True) # type: ignore
2978 + # some opaque config, only used to provide additional information
2979 + # for the hash computation, mainly used for testing and debugging.
2980 + additional_config: SupportsHash = field(default=None,
2981 + init=True) # type: ignore
2982 instance_id: str = ""
2983
2984 def compute_hash(self) -> str:
@@ -3000,33 +3010,62 @@ class VllmConfig:
3010 vllm_factors.append(__version__)
3011 if self.model_config:
3012 vllm_factors.append(self.model_config.compute_hash())
3013 + else:
3014 + vllm_factors.append("None")
3015 if self.cache_config:
3016 vllm_factors.append(self.cache_config.compute_hash())
3017 + else:
3018 + vllm_factors.append("None")
3019 if self.parallel_config:
3020 vllm_factors.append(self.parallel_config.compute_hash())
3021 + else:
3022 + vllm_factors.append("None")
3023 if self.scheduler_config:
3024 vllm_factors.append(self.scheduler_config.compute_hash())
3025 + else:
3026 + vllm_factors.append("None")
3027 if self.device_config:
3028 vllm_factors.append(self.device_config.compute_hash())
3029 + else:
3030 + vllm_factors.append("None")
3031 if self.load_config:
3032 vllm_factors.append(self.load_config.compute_hash())
3033 + else:
3034 + vllm_factors.append("None")
3035 if self.lora_config:
3036 vllm_factors.append(self.lora_config.compute_hash())
3037 + else:
3038 + vllm_factors.append("None")
3039 if self.speculative_config:
3040 vllm_factors.append(self.speculative_config.compute_hash())
3041 + else:
3042 + vllm_factors.append("None")
3043 if self.decoding_config:
3044 vllm_factors.append(self.decoding_config.compute_hash())
3045 + else:
3046 + vllm_factors.append("None")
3047 if self.observability_config:
3048 vllm_factors.append(self.observability_config.compute_hash())
3049 + else:
3050 + vllm_factors.append("None")
3051 if self.prompt_adapter_config:
3052 vllm_factors.append(self.prompt_adapter_config.compute_hash())
3053 + else:
3054 + vllm_factors.append("None")
3055 if self.quant_config:
3056 pass # should be captured by model_config.quantization
3057 if self.compilation_config:
3058 vllm_factors.append(self.compilation_config.compute_hash())
3059 + else:
3060 + vllm_factors.append("None")
3061 if self.kv_transfer_config:
3062 vllm_factors.append(self.kv_transfer_config.compute_hash())
3029 -
3063 + else:
3064 + vllm_factors.append("None")
3065 + if self.additional_config:
3066 + vllm_factors.append(self.additional_config.compute_hash())
3067 + else:
3068 + vllm_factors.append("None")
3069 factors.append(vllm_factors)
3070
3071 hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10]
vllm/v1/worker/gpu_worker.py
+1
@@ -48,6 +48,7 @@ class Worker:
48 self.prompt_adapter_config = vllm_config.prompt_adapter_config
49 self.observability_config = vllm_config.observability_config
50
51 + self.parallel_config.rank = rank
52 self.local_rank = local_rank
53 self.rank = rank
54 self.distributed_init_method = distributed_init_method