A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[v1] fix compilation cache (#11598)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 30, 2024 at 12:24 UTC
3682e33f9ff9d8baade6112a8e75a77da898f504
4 files changed
+69
-14
tests/compile/piecewise/test_toy_llama.py
+13
-2
@@ -7,7 +7,7 @@ if the config `tractable_init` is set to True. Otherwise, the weights are
7
initialized randomly with a fixed seed.
8
"""
9
from dataclasses import dataclass
10
-from typing import Optional, Tuple
10
+from typing import Any, List, Optional, Tuple
11
12
import torch
13
from torch import nn
@@ -54,6 +54,16 @@ class LlamaConfig:
54
tractable_init: bool = False
55
random_seed: int = 0
56
57
+ def compute_hash(self) -> str:
58
+ factors: List[Any] = []
59
+ for k, v in self.__dict__.items():
60
+ if k == "random_seed":
61
+ continue
62
+ factors.append((k, v))
63
+ factors.sort()
64
+ import hashlib
65
+ return hashlib.md5(str(factors).encode()).hexdigest()
66
+
67
def __post_init__(self):
68
assert self.mlp_size >= self.hidden_size
69
@@ -263,7 +273,8 @@ def run_model(llama_config,
273
compilation_config = CompilationConfig(
274
level=CompilationLevel.NO_COMPILATION, )
275
266
- vllm_config = VllmConfig(compilation_config=compilation_config)
276
+ vllm_config = VllmConfig(compilation_config=compilation_config,
277
+ additional_config=llama_config)
278
with set_current_vllm_config(vllm_config):
279
model = LlamaModel(config=llama_config,
280
vllm_config=vllm_config,
vllm/compilation/backends.py
+13
-9
@@ -619,8 +619,10 @@ class PiecewiseBackend:
619
# the entries for different shapes that we need to either
620
# compile or capture cudagraph
621
self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
622
- self.to_be_compiled_sizes: Set[int] = self.compile_sizes.union(
623
- self.capture_sizes)
622
+
623
+ # to_be_compiled_sizes tracks the remaining sizes to compile,
624
+ # and updates during the compilation process, so we need to copy it
625
+ self.to_be_compiled_sizes: Set[int] = self.compile_sizes.copy()
626
for shape in self.compile_sizes.union(self.capture_sizes):
627
self.concrete_size_entries[shape] = ConcreteSizeEntry(
628
runtime_shape=shape,
@@ -628,12 +630,17 @@ class PiecewiseBackend:
630
use_cudagraph=shape in self.capture_sizes,
631
)
632
633
+ def check_for_ending_compilation(self):
634
+ if self.is_last_graph and not self.to_be_compiled_sizes:
635
+ # no specific sizes to compile
636
+ # save the hash of the inductor graph for the next run
637
+ self.compilation_config.inductor_hash_cache.save_to_file()
638
+ end_monitoring_torch_compile(self.vllm_config)
639
+
640
def __call__(self, *args) -> Any:
641
if not self.first_run_finished:
642
self.first_run_finished = True
634
- # no specific sizes to compile
635
- if self.is_last_graph and not self.to_be_compiled_sizes:
636
- end_monitoring_torch_compile(self.vllm_config)
643
+ self.check_for_ending_compilation()
644
return self.compiled_graph_for_general_shape(*args)
645
646
runtime_shape = args[self.sym_shape_indices[0]]
@@ -662,10 +669,7 @@ class PiecewiseBackend:
669
670
# finished compilations for all required shapes
671
if self.is_last_graph and not self.to_be_compiled_sizes:
665
-
666
- # save the hash of the inductor graph for the next run
667
- self.compilation_config.inductor_hash_cache.save_to_file()
668
- end_monitoring_torch_compile(self.vllm_config)
672
+ self.check_for_ending_compilation()
673
674
if not entry.use_cudagraph:
675
return entry.runnable(*args)
vllm/config.py
+42
-3
@@ -9,8 +9,8 @@ from contextlib import contextmanager
9
from dataclasses import dataclass, field, replace
10
from pathlib import Path
11
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,
12
- Final, List, Literal, Mapping, Optional, Set, Tuple, Type,
13
- Union)
12
+ Final, List, Literal, Mapping, Optional, Protocol, Set,
13
+ Tuple, Type, Union)
14
15
import torch
16
from pydantic import BaseModel, Field, PrivateAttr
@@ -75,6 +75,12 @@ HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],
75
PretrainedConfig]]
76
77
78
+class SupportsHash(Protocol):
79
+
80
+ def compute_hash(self) -> str:
81
+ ...
82
+
83
+
84
class ModelConfig:
85
"""Configuration for the model.
86
@@ -2969,6 +2975,10 @@ class VllmConfig:
2975
init=True) # type: ignore
2976
kv_transfer_config: KVTransferConfig = field(default=None,
2977
init=True) # type: ignore
2978
+ # some opaque config, only used to provide additional information
2979
+ # for the hash computation, mainly used for testing and debugging.
2980
+ additional_config: SupportsHash = field(default=None,
2981
+ init=True) # type: ignore
2982
instance_id: str = ""
2983
2984
def compute_hash(self) -> str:
@@ -3000,33 +3010,62 @@ class VllmConfig:
3010
vllm_factors.append(__version__)
3011
if self.model_config:
3012
vllm_factors.append(self.model_config.compute_hash())
3013
+ else:
3014
+ vllm_factors.append("None")
3015
if self.cache_config:
3016
vllm_factors.append(self.cache_config.compute_hash())
3017
+ else:
3018
+ vllm_factors.append("None")
3019
if self.parallel_config:
3020
vllm_factors.append(self.parallel_config.compute_hash())
3021
+ else:
3022
+ vllm_factors.append("None")
3023
if self.scheduler_config:
3024
vllm_factors.append(self.scheduler_config.compute_hash())
3025
+ else:
3026
+ vllm_factors.append("None")
3027
if self.device_config:
3028
vllm_factors.append(self.device_config.compute_hash())
3029
+ else:
3030
+ vllm_factors.append("None")
3031
if self.load_config:
3032
vllm_factors.append(self.load_config.compute_hash())
3033
+ else:
3034
+ vllm_factors.append("None")
3035
if self.lora_config:
3036
vllm_factors.append(self.lora_config.compute_hash())
3037
+ else:
3038
+ vllm_factors.append("None")
3039
if self.speculative_config:
3040
vllm_factors.append(self.speculative_config.compute_hash())
3041
+ else:
3042
+ vllm_factors.append("None")
3043
if self.decoding_config:
3044
vllm_factors.append(self.decoding_config.compute_hash())
3045
+ else:
3046
+ vllm_factors.append("None")
3047
if self.observability_config:
3048
vllm_factors.append(self.observability_config.compute_hash())
3049
+ else:
3050
+ vllm_factors.append("None")
3051
if self.prompt_adapter_config:
3052
vllm_factors.append(self.prompt_adapter_config.compute_hash())
3053
+ else:
3054
+ vllm_factors.append("None")
3055
if self.quant_config:
3056
pass # should be captured by model_config.quantization
3057
if self.compilation_config:
3058
vllm_factors.append(self.compilation_config.compute_hash())
3059
+ else:
3060
+ vllm_factors.append("None")
3061
if self.kv_transfer_config:
3062
vllm_factors.append(self.kv_transfer_config.compute_hash())
3029
-
3063
+ else:
3064
+ vllm_factors.append("None")
3065
+ if self.additional_config:
3066
+ vllm_factors.append(self.additional_config.compute_hash())
3067
+ else:
3068
+ vllm_factors.append("None")
3069
factors.append(vllm_factors)
3070
3071
hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10]
vllm/v1/worker/gpu_worker.py
+1
@@ -48,6 +48,7 @@ class Worker:
48
self.prompt_adapter_config = vllm_config.prompt_adapter_config
49
self.observability_config = vllm_config.observability_config
50
51
+ self.parallel_config.rank = rank
52
self.local_rank = local_rank
53
self.rank = rank
54
self.distributed_init_method = distributed_init_method