A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Standardize RoPE handling for Qwen2-VL (#9250)
Cyrus Leung committed
Oct 16, 2024 at 13:56 UTC
7e7eae338d2774f90ba4b5a04d6c53e7299f40de
16 files changed
+102
-200
benchmarks/kernels/benchmark_rope.py
+2
-2
@@ -31,7 +31,7 @@ def benchmark_rope_kernels_multi_lora(
31
# batched RoPE can take multiple scaling factors
32
batched_rope = get_rope(head_size, rotary_dim, max_position, base,
33
is_neox_style, {
34
- "type": "linear",
34
+ "rope_type": "linear",
35
"factor": tuple(scaling_factors)
36
})
37
# non-batched RoPE takes only one scaling factor, we create multiple
@@ -41,7 +41,7 @@ def benchmark_rope_kernels_multi_lora(
41
non_batched_ropes.append(
42
get_rope(head_size, rotary_dim, max_position, base, is_neox_style,
43
{
44
- "type": "linear",
44
+ "rope_type": "linear",
45
"factor": (scaling_factor, )
46
}))
47
requirements-common.txt
+1
-1
@@ -4,7 +4,7 @@ numpy < 2.0.0
4
requests >= 2.26.0
5
tqdm
6
py-cpuinfo
7
-transformers >= 4.45.0 # Required for Llama 3.2.
7
+transformers >= 4.45.2 # Required for Llama 3.2 and Qwen2-VL.
8
tokenizers >= 0.19.1 # Required for Llama 3.
9
protobuf # Required by LlamaTokenizer.
10
fastapi >= 0.107.0, < 0.113.0; python_version < '3.9'
tests/kernels/test_pos_encoding.py
+4
-4
@@ -105,7 +105,7 @@ def test_batched_rotary_embedding(
105
if rotary_dim is None:
106
rotary_dim = head_size
107
rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
108
- "type": "linear",
108
+ "rope_type": "linear",
109
"factor": (1, )
110
})
111
rope = rope.to(dtype=dtype)
@@ -166,7 +166,7 @@ def test_batched_rotary_embedding_multi_lora(
166
rotary_dim = head_size
167
scaling_factors: List[int] = [1, 2, 4]
168
rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
169
- "type": "linear",
169
+ "rope_type": "linear",
170
"factor": tuple(scaling_factors)
171
})
172
rope = rope.to(dtype=dtype)
@@ -211,10 +211,10 @@ def test_rope_module_cache():
211
MAX_POSITIONS = [123, 1234]
212
BASES = [10000, 1000000]
213
ROPE_SCALINGS = (None, {
214
- "type": "linear",
214
+ "rope_type": "linear",
215
"factor": (1, )
216
}, {
217
- "type": "dynamic",
217
+ "rope_type": "dynamic",
218
"factor": 1
219
})
220
settings = (HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
tests/lora/test_layers.py
+1
-1
@@ -951,7 +951,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
951
lora_rope.create_lora_weights(max_loras, lora_config)
952
linear_rope = get_rope(head_size, rotary_dim, max_position, base,
953
is_neox_style, {
954
- "type": "linear",
954
+ "rope_type": "linear",
955
"factor": scaling_factors
956
})
957
linear_rope = linear_rope.to(dtype=dtype)
tests/test_config.py
+2
-2
@@ -64,9 +64,9 @@ def test_get_sliding_window():
64
65
66
def test_rope_customization():
67
- TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
67
+ TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
68
TEST_ROPE_THETA = 16_000_000.0
69
- LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
69
+ LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
70
71
llama_model_config = ModelConfig(
72
"meta-llama/Meta-Llama-3-8B-Instruct",
vllm/config.py
+7
-14
@@ -1739,16 +1739,10 @@ def _get_and_verify_max_len(
1739
1740
rope_scaling = getattr(hf_config, "rope_scaling", None)
1741
if rope_scaling is not None:
1742
- if "type" in rope_scaling:
1743
- rope_type = rope_scaling["type"]
1744
- elif "rope_type" in rope_scaling:
1745
- rope_type = rope_scaling["rope_type"]
1746
- else:
1747
- raise ValueError(
1748
- "rope_scaling must have a 'type' or 'rope_type' key.")
1742
+ # No need to consider "type" key because of patch_rope_scaling when
1743
+ # loading HF config
1744
+ rope_type = rope_scaling["rope_type"]
1745
1750
- # The correct one should be "longrope", kept "su" here
1751
- # to be backward compatible
1746
if rope_type not in ("su", "longrope", "llama3"):
1747
if disable_sliding_window:
1748
# TODO(robertgshaw): Find a model that supports rope_scaling
@@ -1758,11 +1752,10 @@ def _get_and_verify_max_len(
1752
"with rope_scaling. Please raise an issue so we can "
1753
"investigate.")
1754
1761
- if rope_type == "mrope":
1762
- scaling_factor = 1
1763
- else:
1764
- assert "factor" in rope_scaling
1765
- scaling_factor = rope_scaling["factor"]
1755
+ # NOTE: rope_type == "default" does not define factor
1756
+ # https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
1757
+ scaling_factor = rope_scaling.get("factor", 1.0)
1758
+
1759
if rope_type == "yarn":
1760
derived_max_model_len = rope_scaling[
1761
"original_max_position_embeddings"]
vllm/engine/arg_utils.py
+6
-5
@@ -454,11 +454,12 @@ class EngineArgs:
454
'None, we assume the model weights are not '
455
'quantized and use `dtype` to determine the data '
456
'type of the weights.')
457
- parser.add_argument('--rope-scaling',
458
- default=None,
459
- type=json.loads,
460
- help='RoPE scaling configuration in JSON format. '
461
- 'For example, {"type":"dynamic","factor":2.0}')
457
+ parser.add_argument(
458
+ '--rope-scaling',
459
+ default=None,
460
+ type=json.loads,
461
+ help='RoPE scaling configuration in JSON format. '
462
+ 'For example, {"rope_type":"dynamic","factor":2.0}')
463
parser.add_argument('--rope-theta',
464
default=None,
465
type=float,
vllm/model_executor/layers/rotary_embedding.py
+28
-19
@@ -920,13 +920,10 @@ def get_rope(
920
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
921
is_neox_style, dtype)
922
else:
923
- scaling_type = rope_scaling[
924
- "type"] if "type" in rope_scaling else rope_scaling["rope_type"]
925
- # The correct one should be "longrope" but keep "su" here
926
- # for backward compatible
927
- if scaling_type not in {"su", "longrope"}:
928
- scaling_factor = rope_scaling.get("factor", 1.0)
923
+ scaling_type = rope_scaling["rope_type"]
924
+
925
if scaling_type == "llama3":
926
+ scaling_factor = rope_scaling["factor"]
927
low_freq_factor = rope_scaling["low_freq_factor"]
928
high_freq_factor = rope_scaling["high_freq_factor"]
929
original_max_position = rope_scaling[
@@ -937,16 +934,39 @@ def get_rope(
934
scaling_factor, low_freq_factor,
935
high_freq_factor,
936
original_max_position)
937
+ elif scaling_type == "default":
938
+ if "mrope_section" in rope_scaling:
939
+ rotary_emb = MRotaryEmbedding(
940
+ head_size,
941
+ rotary_dim,
942
+ max_position,
943
+ base,
944
+ is_neox_style,
945
+ dtype,
946
+ mrope_section=rope_scaling["mrope_section"],
947
+ )
948
+ else:
949
+ rotary_emb = RotaryEmbedding(
950
+ head_size,
951
+ rotary_dim,
952
+ max_position,
953
+ base,
954
+ is_neox_style,
955
+ dtype,
956
+ )
957
elif scaling_type == "linear":
958
+ scaling_factor = rope_scaling["factor"]
959
rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
960
max_position, base,
961
is_neox_style,
962
scaling_factor, dtype)
963
elif scaling_type == "dynamic":
964
+ scaling_factor = rope_scaling["factor"]
965
rotary_emb = DynamicNTKScalingRotaryEmbedding(
966
head_size, rotary_dim, max_position, base, is_neox_style,
967
scaling_factor, dtype)
968
elif scaling_type == "yarn":
969
+ scaling_factor = rope_scaling["factor"]
970
original_max_position = rope_scaling[
971
"original_max_position_embeddings"]
972
extra_kwargs = {
@@ -961,6 +981,7 @@ def get_rope(
981
scaling_factor, dtype,
982
**extra_kwargs)
983
elif scaling_type == "deepseek_yarn":
984
+ scaling_factor = rope_scaling["factor"]
985
original_max_position = rope_scaling[
986
"original_max_position_embeddings"]
987
# assert max_position == original_max_position * scaling_factor
@@ -973,9 +994,7 @@ def get_rope(
994
rotary_emb = DeepseekScalingRotaryEmbedding(
995
head_size, rotary_dim, original_max_position, base,
996
is_neox_style, scaling_factor, dtype, **extra_kwargs)
976
- # The correct one should be "longrope" but keep "su" here
977
- # for backward compatible
978
- elif scaling_type == "su" or scaling_type == "longrope":
997
+ elif scaling_type == "longrope":
998
short_factor = rope_scaling["short_factor"]
999
long_factor = rope_scaling["long_factor"]
1000
original_max_position = rope_scaling[
@@ -989,16 +1008,6 @@ def get_rope(
1008
head_size, rotary_dim, max_position, original_max_position,
1009
base, is_neox_style, dtype, short_factor, long_factor,
1010
**extra_kwargs)
992
- elif scaling_type == "mrope":
993
- rotary_emb = MRotaryEmbedding(
994
- head_size,
995
- rotary_dim,
996
- max_position,
997
- base,
998
- is_neox_style,
999
- dtype,
1000
- mrope_section=rope_scaling["mrope_section"],
1001
- )
1011
else:
1012
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
1013
_ROPE_DICT[key] = rotary_emb
vllm/model_executor/models/deepseek_v2.py
+1
-1
@@ -242,7 +242,7 @@ class DeepseekV2Attention(nn.Module):
242
bias=False,
243
quant_config=quant_config,
244
prefix=f"{prefix}.o_proj")
245
- rope_scaling['type'] = 'deepseek_yarn'
245
+ rope_scaling["rope_type"] = 'deepseek_yarn'
246
self.rotary_emb = get_rope(qk_rope_head_dim,
247
rotary_dim=qk_rope_head_dim,
248
max_position=max_position_embeddings,
vllm/model_executor/models/phi3_small.py
+1
-1
@@ -179,7 +179,7 @@ class Phi3SmallSelfAttention(nn.Module):
179
rope_scaling["factor"] = self.rope_position_scale
180
else:
181
rope_scaling = {
182
- "type": "linear",
182
+ "rope_type": "linear",
183
"factor": self.rope_position_scale,
184
}
185
vllm/model_executor/models/qwen2_vl.py
+4
-4
@@ -34,6 +34,8 @@ from PIL import Image
34
from transformers.image_utils import (get_image_size,
35
infer_channel_dimension_format,
36
to_numpy_array)
37
+from transformers.models.qwen2_vl.configuration_qwen2_vl import (
38
+ Qwen2VLConfig, Qwen2VLVisionConfig)
39
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
make_batched_images, make_batched_videos, smart_resize)
41
@@ -62,8 +64,7 @@ from vllm.multimodal.base import MultiModalData
64
from vllm.multimodal.image import cached_get_image_processor
65
from vllm.platforms import current_platform
66
from vllm.sequence import IntermediateTensors, SequenceData
65
-from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
66
- Qwen2VLVisionConfig)
67
+from vllm.transformers_utils.config import uses_mrope
68
from vllm.transformers_utils.processor import get_processor
69
from vllm.utils import is_cpu
70
@@ -1061,8 +1062,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1062
if image_input is None and video_input is None:
1063
inputs_embeds = None
1064
else:
1064
- rope_scaling = getattr(self.config, "rope_scaling", {})
1065
- if rope_scaling.get("type", None) == "mrope":
1065
+ if uses_mrope(self.config):
1066
assert positions.ndim == 2 and positions.size(0) == 3, (
1067
"multimodal section rotary embedding requires "
1068
f"(3, seq_len) positions, but got {positions.size()}")
vllm/transformers_utils/config.py
+41
-3
@@ -23,8 +23,8 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
23
MedusaConfig, MllamaConfig,
24
MLPSpeculatorConfig, MPTConfig,
25
NemotronConfig, NVLM_D_Config,
26
- Qwen2VLConfig, RWConfig,
27
- SolarConfig, UltravoxConfig)
26
+ RWConfig, SolarConfig,
27
+ UltravoxConfig)
28
# yapf: enable
29
from vllm.transformers_utils.utils import check_gguf_file
30
@@ -57,7 +57,6 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
57
"NVLM_D": NVLM_D_Config,
58
"solar": SolarConfig,
59
"ultravox": UltravoxConfig,
60
- "qwen2_vl": Qwen2VLConfig,
60
**_CONFIG_REGISTRY_OVERRIDE_HF
61
}
62
@@ -91,6 +90,43 @@ def file_or_path_exists(model: Union[str, Path], config_name, revision,
90
return False
91
92
93
+def patch_rope_scaling(config: PretrainedConfig) -> None:
94
+ """Provide backwards compatibility for RoPE."""
95
+ text_config = getattr(config, "text_config", None)
96
+ if text_config is not None:
97
+ patch_rope_scaling(text_config)
98
+
99
+ rope_scaling = getattr(config, "rope_scaling", None)
100
+ if rope_scaling is not None:
101
+ patch_rope_scaling_dict(rope_scaling)
102
+
103
+
104
+def patch_rope_scaling_dict(rope_scaling: Dict[str, Any]) -> None:
105
+ if "rope_type" not in rope_scaling and "type" in rope_scaling:
106
+ rope_scaling["rope_type"] = rope_scaling["type"]
107
+ logger.info("Replacing legacy 'type' key with 'rope_type'")
108
+
109
+ if "rope_type" not in rope_scaling:
110
+ raise ValueError("rope_scaling should have a 'rope_type' key")
111
+
112
+ if rope_scaling["rope_type"] == "su":
113
+ rope_scaling["rope_type"] = "longrope"
114
+ logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
115
+ elif rope_scaling["rope_type"] == "mrope":
116
+ assert "mrope_section" in rope_scaling
117
+ rope_scaling["rope_type"] = "default"
118
+ logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
119
+
120
+
121
+def uses_mrope(config: PretrainedConfig) -> bool:
122
+ """Detect if the model with this config uses M-ROPE."""
123
+ rope_scaling = getattr(config, "rope_scaling", None)
124
+ if rope_scaling is None:
125
+ return False
126
+
127
+ return "mrope_section" in rope_scaling
128
+
129
+
130
def get_config(
131
model: Union[str, Path],
132
trust_remote_code: bool,
@@ -191,6 +227,8 @@ def get_config(
227
)
228
config.update({key: value})
229
230
+ patch_rope_scaling(config)
231
+
232
return config
233
234
vllm/transformers_utils/configs/__init__.py
-4
@@ -14,8 +14,6 @@ from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
14
from vllm.transformers_utils.configs.mpt import MPTConfig
15
from vllm.transformers_utils.configs.nemotron import NemotronConfig
16
from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
17
-from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
18
- Qwen2VLVisionConfig)
17
from vllm.transformers_utils.configs.solar import SolarConfig
18
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
19
@@ -35,6 +33,4 @@ __all__ = [
33
"NVLM_D_Config",
34
"SolarConfig",
35
"UltravoxConfig",
38
- "Qwen2VLConfig",
39
- "Qwen2VLVisionConfig",
36
]
vllm/transformers_utils/configs/qwen2vl.py
deleted
-131
@@ -1,131 +0,0 @@
1
-# coding=utf-8
2
-# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team.
3
-# All rights reserved.
4
-#
5
-# Licensed under the Apache License, Version 2.0 (the "License");
6
-# you may not use this file except in compliance with the License.
7
-# You may obtain a copy of the License at
8
-#
9
-# http://www.apache.org/licenses/LICENSE-2.0
10
-#
11
-# Unless required by applicable law or agreed to in writing, software
12
-# distributed under the License is distributed on an "AS IS" BASIS,
13
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
-# See the License for the specific language governing permissions and
15
-# limitations under the License.
16
-"""Qwen2VL model configuration"""
17
-
18
-import os
19
-from typing import Union
20
-
21
-from transformers import PretrainedConfig
22
-
23
-
24
-class Qwen2VLVisionConfig(PretrainedConfig):
25
- model_type = "qwen2_vl"
26
-
27
- def __init__(
28
- self,
29
- depth=32,
30
- embed_dim=1280,
31
- hidden_size=3584,
32
- hidden_act="quick_gelu",
33
- mlp_ratio=4,
34
- num_heads=16,
35
- in_channels=3,
36
- patch_size=14,
37
- spatial_merge_size=2,
38
- temporal_patch_size=2,
39
- **kwargs,
40
- ):
41
- super().__init__(**kwargs)
42
-
43
- self.depth = depth
44
- self.embed_dim = embed_dim
45
- self.hidden_size = hidden_size
46
- self.hidden_act = hidden_act
47
- self.mlp_ratio = mlp_ratio
48
- self.num_heads = num_heads
49
- self.in_channels = in_channels
50
- self.patch_size = patch_size
51
- self.spatial_merge_size = spatial_merge_size
52
- self.temporal_patch_size = temporal_patch_size
53
-
54
- @classmethod
55
- def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
56
- os.PathLike],
57
- **kwargs) -> "PretrainedConfig":
58
- cls._set_token_in_kwargs(kwargs)
59
-
60
- config_dict, kwargs = cls.get_config_dict(
61
- pretrained_model_name_or_path, **kwargs)
62
-
63
- if config_dict.get("model_type") == "qwen2_vl":
64
- config_dict = config_dict["vision_config"]
65
-
66
- return cls.from_dict(config_dict, **kwargs)
67
-
68
-
69
-class Qwen2VLConfig(PretrainedConfig):
70
-
71
- def __init__(
72
- self,
73
- vocab_size=152064,
74
- hidden_size=8192,
75
- intermediate_size=29568,
76
- num_hidden_layers=80,
77
- num_attention_heads=64,
78
- num_key_value_heads=8,
79
- hidden_act="silu",
80
- max_position_embeddings=32768,
81
- initializer_range=0.02,
82
- rms_norm_eps=1e-05,
83
- use_cache=True,
84
- tie_word_embeddings=False,
85
- rope_theta=1000000.0,
86
- use_sliding_window=False,
87
- sliding_window=4096,
88
- max_window_layers=80,
89
- attention_dropout=0.0,
90
- vision_config=None,
91
- rope_scaling=None,
92
- **kwargs,
93
- ):
94
- if isinstance(vision_config, dict):
95
- self.vision_config = Qwen2VLVisionConfig(**vision_config)
96
- elif vision_config is None:
97
- self.vision_config = Qwen2VLVisionConfig()
98
-
99
- self.vocab_size = vocab_size
100
- self.max_position_embeddings = max_position_embeddings
101
- self.hidden_size = hidden_size
102
- self.intermediate_size = intermediate_size
103
- self.num_hidden_layers = num_hidden_layers
104
- self.num_attention_heads = num_attention_heads
105
- self.use_sliding_window = use_sliding_window
106
- self.sliding_window = sliding_window
107
- self.max_window_layers = max_window_layers
108
-
109
- # for backward compatibility
110
- if num_key_value_heads is None:
111
- num_key_value_heads = num_attention_heads
112
-
113
- self.num_key_value_heads = num_key_value_heads
114
- self.hidden_act = hidden_act
115
- self.initializer_range = initializer_range
116
- self.rms_norm_eps = rms_norm_eps
117
- self.use_cache = use_cache
118
- self.rope_theta = rope_theta
119
- self.attention_dropout = attention_dropout
120
- self.rope_scaling = rope_scaling
121
-
122
- # NOTE: the following section from original transformers config
123
- # for Qwen2-VL is commented out to address rope config loading issue
124
- #
125
- # if self.rope_scaling is not None and "type" in self.rope_scaling:
126
- # if self.rope_scaling["type"] == "mrope":
127
- # self.rope_scaling["type"] = "default"
128
- # self.rope_scaling["rope_type"] = self.rope_scaling["type"]
129
- # rope_config_validation(self)
130
-
131
- super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
vllm/worker/cpu_model_runner.py
+2
-4
@@ -19,6 +19,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19
MultiModalInputs)
20
from vllm.sequence import (IntermediateTensors, SequenceData,
21
SequenceGroupMetadata)
22
+from vllm.transformers_utils.config import uses_mrope
23
from vllm.utils import make_tensor_with_pad
24
from vllm.worker.model_runner_base import (
25
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -439,10 +440,7 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
440
def model_is_mrope(self) -> bool:
441
"""Detect if the model has "mrope" rope_scaling type.
442
mrope requires keep "rope_deltas" between prompt and decoding phases."""
442
- rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
443
- if rope_scaling is None:
444
- return False
445
- return rope_scaling.get("type", None) == "mrope"
443
+ return uses_mrope(self.model_config.hf_config)
444
445
def load_model(self) -> None:
446
self.model = get_model(model_config=self.model_config,
vllm/worker/model_runner.py
+2
-4
@@ -47,6 +47,7 @@ from vllm.prompt_adapter.worker_manager import (
47
LRUCacheWorkerPromptAdapterManager)
48
from vllm.sampling_params import SamplingParams
49
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50
+from vllm.transformers_utils.config import uses_mrope
51
from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52
flatten_2d_lists, is_hip, is_pin_memory_available,
53
supports_dynamo)
@@ -1379,10 +1380,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1380
def model_is_mrope(self) -> bool:
1381
"""Detect if the model has "mrope" rope_scaling type.
1382
mrope requires keep "rope_deltas" between prompt and decoding phases."""
1382
- rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
1383
- if rope_scaling is None:
1384
- return False
1385
- return rope_scaling.get("type", None) == "mrope"
1383
+ return uses_mrope(self.model_config.hf_config)
1384
1385
@torch.inference_mode()
1386
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None: