@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e7eae338d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Standardize RoPE handling for Qwen2-VL (#9250)

Cyrus Leung committed Oct 16, 2024 at 13:56 UTC 7e7eae338d2774f90ba4b5a04d6c53e7299f40de
16 files changed +102 -200
benchmarks/kernels/benchmark_rope.py
+2 -2
@@ -31,7 +31,7 @@ def benchmark_rope_kernels_multi_lora(
31 # batched RoPE can take multiple scaling factors
32 batched_rope = get_rope(head_size, rotary_dim, max_position, base,
33 is_neox_style, {
34 - "type": "linear",
34 + "rope_type": "linear",
35 "factor": tuple(scaling_factors)
36 })
37 # non-batched RoPE takes only one scaling factor, we create multiple
@@ -41,7 +41,7 @@ def benchmark_rope_kernels_multi_lora(
41 non_batched_ropes.append(
42 get_rope(head_size, rotary_dim, max_position, base, is_neox_style,
43 {
44 - "type": "linear",
44 + "rope_type": "linear",
45 "factor": (scaling_factor, )
46 }))
47
requirements-common.txt
+1 -1
@@ -4,7 +4,7 @@ numpy < 2.0.0
4 requests >= 2.26.0
5 tqdm
6 py-cpuinfo
7 -transformers >= 4.45.0 # Required for Llama 3.2.
7 +transformers >= 4.45.2 # Required for Llama 3.2 and Qwen2-VL.
8 tokenizers >= 0.19.1 # Required for Llama 3.
9 protobuf # Required by LlamaTokenizer.
10 fastapi >= 0.107.0, < 0.113.0; python_version < '3.9'
tests/kernels/test_pos_encoding.py
+4 -4
@@ -105,7 +105,7 @@ def test_batched_rotary_embedding(
105 if rotary_dim is None:
106 rotary_dim = head_size
107 rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
108 - "type": "linear",
108 + "rope_type": "linear",
109 "factor": (1, )
110 })
111 rope = rope.to(dtype=dtype)
@@ -166,7 +166,7 @@ def test_batched_rotary_embedding_multi_lora(
166 rotary_dim = head_size
167 scaling_factors: List[int] = [1, 2, 4]
168 rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
169 - "type": "linear",
169 + "rope_type": "linear",
170 "factor": tuple(scaling_factors)
171 })
172 rope = rope.to(dtype=dtype)
@@ -211,10 +211,10 @@ def test_rope_module_cache():
211 MAX_POSITIONS = [123, 1234]
212 BASES = [10000, 1000000]
213 ROPE_SCALINGS = (None, {
214 - "type": "linear",
214 + "rope_type": "linear",
215 "factor": (1, )
216 }, {
217 - "type": "dynamic",
217 + "rope_type": "dynamic",
218 "factor": 1
219 })
220 settings = (HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
tests/lora/test_layers.py
+1 -1
@@ -951,7 +951,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
951 lora_rope.create_lora_weights(max_loras, lora_config)
952 linear_rope = get_rope(head_size, rotary_dim, max_position, base,
953 is_neox_style, {
954 - "type": "linear",
954 + "rope_type": "linear",
955 "factor": scaling_factors
956 })
957 linear_rope = linear_rope.to(dtype=dtype)
tests/test_config.py
+2 -2
@@ -64,9 +64,9 @@ def test_get_sliding_window():
64
65
66 def test_rope_customization():
67 - TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
67 + TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
68 TEST_ROPE_THETA = 16_000_000.0
69 - LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
69 + LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
70
71 llama_model_config = ModelConfig(
72 "meta-llama/Meta-Llama-3-8B-Instruct",
vllm/config.py
+7 -14
@@ -1739,16 +1739,10 @@ def _get_and_verify_max_len(
1739
1740 rope_scaling = getattr(hf_config, "rope_scaling", None)
1741 if rope_scaling is not None:
1742 - if "type" in rope_scaling:
1743 - rope_type = rope_scaling["type"]
1744 - elif "rope_type" in rope_scaling:
1745 - rope_type = rope_scaling["rope_type"]
1746 - else:
1747 - raise ValueError(
1748 - "rope_scaling must have a 'type' or 'rope_type' key.")
1742 + # No need to consider "type" key because of patch_rope_scaling when
1743 + # loading HF config
1744 + rope_type = rope_scaling["rope_type"]
1745
1750 - # The correct one should be "longrope", kept "su" here
1751 - # to be backward compatible
1746 if rope_type not in ("su", "longrope", "llama3"):
1747 if disable_sliding_window:
1748 # TODO(robertgshaw): Find a model that supports rope_scaling
@@ -1758,11 +1752,10 @@ def _get_and_verify_max_len(
1752 "with rope_scaling. Please raise an issue so we can "
1753 "investigate.")
1754
1761 - if rope_type == "mrope":
1762 - scaling_factor = 1
1763 - else:
1764 - assert "factor" in rope_scaling
1765 - scaling_factor = rope_scaling["factor"]
1755 + # NOTE: rope_type == "default" does not define factor
1756 + # https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
1757 + scaling_factor = rope_scaling.get("factor", 1.0)
1758 +
1759 if rope_type == "yarn":
1760 derived_max_model_len = rope_scaling[
1761 "original_max_position_embeddings"]
vllm/engine/arg_utils.py
+6 -5
@@ -454,11 +454,12 @@ class EngineArgs:
454 'None, we assume the model weights are not '
455 'quantized and use `dtype` to determine the data '
456 'type of the weights.')
457 - parser.add_argument('--rope-scaling',
458 - default=None,
459 - type=json.loads,
460 - help='RoPE scaling configuration in JSON format. '
461 - 'For example, {"type":"dynamic","factor":2.0}')
457 + parser.add_argument(
458 + '--rope-scaling',
459 + default=None,
460 + type=json.loads,
461 + help='RoPE scaling configuration in JSON format. '
462 + 'For example, {"rope_type":"dynamic","factor":2.0}')
463 parser.add_argument('--rope-theta',
464 default=None,
465 type=float,
vllm/model_executor/layers/rotary_embedding.py
+28 -19
@@ -920,13 +920,10 @@ def get_rope(
920 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
921 is_neox_style, dtype)
922 else:
923 - scaling_type = rope_scaling[
924 - "type"] if "type" in rope_scaling else rope_scaling["rope_type"]
925 - # The correct one should be "longrope" but keep "su" here
926 - # for backward compatible
927 - if scaling_type not in {"su", "longrope"}:
928 - scaling_factor = rope_scaling.get("factor", 1.0)
923 + scaling_type = rope_scaling["rope_type"]
924 +
925 if scaling_type == "llama3":
926 + scaling_factor = rope_scaling["factor"]
927 low_freq_factor = rope_scaling["low_freq_factor"]
928 high_freq_factor = rope_scaling["high_freq_factor"]
929 original_max_position = rope_scaling[
@@ -937,16 +934,39 @@ def get_rope(
934 scaling_factor, low_freq_factor,
935 high_freq_factor,
936 original_max_position)
937 + elif scaling_type == "default":
938 + if "mrope_section" in rope_scaling:
939 + rotary_emb = MRotaryEmbedding(
940 + head_size,
941 + rotary_dim,
942 + max_position,
943 + base,
944 + is_neox_style,
945 + dtype,
946 + mrope_section=rope_scaling["mrope_section"],
947 + )
948 + else:
949 + rotary_emb = RotaryEmbedding(
950 + head_size,
951 + rotary_dim,
952 + max_position,
953 + base,
954 + is_neox_style,
955 + dtype,
956 + )
957 elif scaling_type == "linear":
958 + scaling_factor = rope_scaling["factor"]
959 rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
960 max_position, base,
961 is_neox_style,
962 scaling_factor, dtype)
963 elif scaling_type == "dynamic":
964 + scaling_factor = rope_scaling["factor"]
965 rotary_emb = DynamicNTKScalingRotaryEmbedding(
966 head_size, rotary_dim, max_position, base, is_neox_style,
967 scaling_factor, dtype)
968 elif scaling_type == "yarn":
969 + scaling_factor = rope_scaling["factor"]
970 original_max_position = rope_scaling[
971 "original_max_position_embeddings"]
972 extra_kwargs = {
@@ -961,6 +981,7 @@ def get_rope(
981 scaling_factor, dtype,
982 **extra_kwargs)
983 elif scaling_type == "deepseek_yarn":
984 + scaling_factor = rope_scaling["factor"]
985 original_max_position = rope_scaling[
986 "original_max_position_embeddings"]
987 # assert max_position == original_max_position * scaling_factor
@@ -973,9 +994,7 @@ def get_rope(
994 rotary_emb = DeepseekScalingRotaryEmbedding(
995 head_size, rotary_dim, original_max_position, base,
996 is_neox_style, scaling_factor, dtype, **extra_kwargs)
976 - # The correct one should be "longrope" but keep "su" here
977 - # for backward compatible
978 - elif scaling_type == "su" or scaling_type == "longrope":
997 + elif scaling_type == "longrope":
998 short_factor = rope_scaling["short_factor"]
999 long_factor = rope_scaling["long_factor"]
1000 original_max_position = rope_scaling[
@@ -989,16 +1008,6 @@ def get_rope(
1008 head_size, rotary_dim, max_position, original_max_position,
1009 base, is_neox_style, dtype, short_factor, long_factor,
1010 **extra_kwargs)
992 - elif scaling_type == "mrope":
993 - rotary_emb = MRotaryEmbedding(
994 - head_size,
995 - rotary_dim,
996 - max_position,
997 - base,
998 - is_neox_style,
999 - dtype,
1000 - mrope_section=rope_scaling["mrope_section"],
1001 - )
1011 else:
1012 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
1013 _ROPE_DICT[key] = rotary_emb
vllm/model_executor/models/deepseek_v2.py
+1 -1
@@ -242,7 +242,7 @@ class DeepseekV2Attention(nn.Module):
242 bias=False,
243 quant_config=quant_config,
244 prefix=f"{prefix}.o_proj")
245 - rope_scaling['type'] = 'deepseek_yarn'
245 + rope_scaling["rope_type"] = 'deepseek_yarn'
246 self.rotary_emb = get_rope(qk_rope_head_dim,
247 rotary_dim=qk_rope_head_dim,
248 max_position=max_position_embeddings,
vllm/model_executor/models/phi3_small.py
+1 -1
@@ -179,7 +179,7 @@ class Phi3SmallSelfAttention(nn.Module):
179 rope_scaling["factor"] = self.rope_position_scale
180 else:
181 rope_scaling = {
182 - "type": "linear",
182 + "rope_type": "linear",
183 "factor": self.rope_position_scale,
184 }
185
vllm/model_executor/models/qwen2_vl.py
+4 -4
@@ -34,6 +34,8 @@ from PIL import Image
34 from transformers.image_utils import (get_image_size,
35 infer_channel_dimension_format,
36 to_numpy_array)
37 +from transformers.models.qwen2_vl.configuration_qwen2_vl import (
38 + Qwen2VLConfig, Qwen2VLVisionConfig)
39 from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40 make_batched_images, make_batched_videos, smart_resize)
41
@@ -62,8 +64,7 @@ from vllm.multimodal.base import MultiModalData
64 from vllm.multimodal.image import cached_get_image_processor
65 from vllm.platforms import current_platform
66 from vllm.sequence import IntermediateTensors, SequenceData
65 -from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
66 - Qwen2VLVisionConfig)
67 +from vllm.transformers_utils.config import uses_mrope
68 from vllm.transformers_utils.processor import get_processor
69 from vllm.utils import is_cpu
70
@@ -1061,8 +1062,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1062 if image_input is None and video_input is None:
1063 inputs_embeds = None
1064 else:
1064 - rope_scaling = getattr(self.config, "rope_scaling", {})
1065 - if rope_scaling.get("type", None) == "mrope":
1065 + if uses_mrope(self.config):
1066 assert positions.ndim == 2 and positions.size(0) == 3, (
1067 "multimodal section rotary embedding requires "
1068 f"(3, seq_len) positions, but got {positions.size()}")
vllm/transformers_utils/config.py
+41 -3
@@ -23,8 +23,8 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
23 MedusaConfig, MllamaConfig,
24 MLPSpeculatorConfig, MPTConfig,
25 NemotronConfig, NVLM_D_Config,
26 - Qwen2VLConfig, RWConfig,
27 - SolarConfig, UltravoxConfig)
26 + RWConfig, SolarConfig,
27 + UltravoxConfig)
28 # yapf: enable
29 from vllm.transformers_utils.utils import check_gguf_file
30
@@ -57,7 +57,6 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
57 "NVLM_D": NVLM_D_Config,
58 "solar": SolarConfig,
59 "ultravox": UltravoxConfig,
60 - "qwen2_vl": Qwen2VLConfig,
60 **_CONFIG_REGISTRY_OVERRIDE_HF
61 }
62
@@ -91,6 +90,43 @@ def file_or_path_exists(model: Union[str, Path], config_name, revision,
90 return False
91
92
93 +def patch_rope_scaling(config: PretrainedConfig) -> None:
94 + """Provide backwards compatibility for RoPE."""
95 + text_config = getattr(config, "text_config", None)
96 + if text_config is not None:
97 + patch_rope_scaling(text_config)
98 +
99 + rope_scaling = getattr(config, "rope_scaling", None)
100 + if rope_scaling is not None:
101 + patch_rope_scaling_dict(rope_scaling)
102 +
103 +
104 +def patch_rope_scaling_dict(rope_scaling: Dict[str, Any]) -> None:
105 + if "rope_type" not in rope_scaling and "type" in rope_scaling:
106 + rope_scaling["rope_type"] = rope_scaling["type"]
107 + logger.info("Replacing legacy 'type' key with 'rope_type'")
108 +
109 + if "rope_type" not in rope_scaling:
110 + raise ValueError("rope_scaling should have a 'rope_type' key")
111 +
112 + if rope_scaling["rope_type"] == "su":
113 + rope_scaling["rope_type"] = "longrope"
114 + logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
115 + elif rope_scaling["rope_type"] == "mrope":
116 + assert "mrope_section" in rope_scaling
117 + rope_scaling["rope_type"] = "default"
118 + logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
119 +
120 +
121 +def uses_mrope(config: PretrainedConfig) -> bool:
122 + """Detect if the model with this config uses M-ROPE."""
123 + rope_scaling = getattr(config, "rope_scaling", None)
124 + if rope_scaling is None:
125 + return False
126 +
127 + return "mrope_section" in rope_scaling
128 +
129 +
130 def get_config(
131 model: Union[str, Path],
132 trust_remote_code: bool,
@@ -191,6 +227,8 @@ def get_config(
227 )
228 config.update({key: value})
229
230 + patch_rope_scaling(config)
231 +
232 return config
233
234
vllm/transformers_utils/configs/__init__.py
-4
@@ -14,8 +14,6 @@ from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
14 from vllm.transformers_utils.configs.mpt import MPTConfig
15 from vllm.transformers_utils.configs.nemotron import NemotronConfig
16 from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
17 -from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
18 - Qwen2VLVisionConfig)
17 from vllm.transformers_utils.configs.solar import SolarConfig
18 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
19
@@ -35,6 +33,4 @@ __all__ = [
33 "NVLM_D_Config",
34 "SolarConfig",
35 "UltravoxConfig",
38 - "Qwen2VLConfig",
39 - "Qwen2VLVisionConfig",
36 ]
vllm/transformers_utils/configs/qwen2vl.py deleted
-131
@@ -1,131 +0,0 @@
1 -# coding=utf-8
2 -# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team.
3 -# All rights reserved.
4 -#
5 -# Licensed under the Apache License, Version 2.0 (the "License");
6 -# you may not use this file except in compliance with the License.
7 -# You may obtain a copy of the License at
8 -#
9 -# http://www.apache.org/licenses/LICENSE-2.0
10 -#
11 -# Unless required by applicable law or agreed to in writing, software
12 -# distributed under the License is distributed on an "AS IS" BASIS,
13 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 -# See the License for the specific language governing permissions and
15 -# limitations under the License.
16 -"""Qwen2VL model configuration"""
17 -
18 -import os
19 -from typing import Union
20 -
21 -from transformers import PretrainedConfig
22 -
23 -
24 -class Qwen2VLVisionConfig(PretrainedConfig):
25 - model_type = "qwen2_vl"
26 -
27 - def __init__(
28 - self,
29 - depth=32,
30 - embed_dim=1280,
31 - hidden_size=3584,
32 - hidden_act="quick_gelu",
33 - mlp_ratio=4,
34 - num_heads=16,
35 - in_channels=3,
36 - patch_size=14,
37 - spatial_merge_size=2,
38 - temporal_patch_size=2,
39 - **kwargs,
40 - ):
41 - super().__init__(**kwargs)
42 -
43 - self.depth = depth
44 - self.embed_dim = embed_dim
45 - self.hidden_size = hidden_size
46 - self.hidden_act = hidden_act
47 - self.mlp_ratio = mlp_ratio
48 - self.num_heads = num_heads
49 - self.in_channels = in_channels
50 - self.patch_size = patch_size
51 - self.spatial_merge_size = spatial_merge_size
52 - self.temporal_patch_size = temporal_patch_size
53 -
54 - @classmethod
55 - def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
56 - os.PathLike],
57 - **kwargs) -> "PretrainedConfig":
58 - cls._set_token_in_kwargs(kwargs)
59 -
60 - config_dict, kwargs = cls.get_config_dict(
61 - pretrained_model_name_or_path, **kwargs)
62 -
63 - if config_dict.get("model_type") == "qwen2_vl":
64 - config_dict = config_dict["vision_config"]
65 -
66 - return cls.from_dict(config_dict, **kwargs)
67 -
68 -
69 -class Qwen2VLConfig(PretrainedConfig):
70 -
71 - def __init__(
72 - self,
73 - vocab_size=152064,
74 - hidden_size=8192,
75 - intermediate_size=29568,
76 - num_hidden_layers=80,
77 - num_attention_heads=64,
78 - num_key_value_heads=8,
79 - hidden_act="silu",
80 - max_position_embeddings=32768,
81 - initializer_range=0.02,
82 - rms_norm_eps=1e-05,
83 - use_cache=True,
84 - tie_word_embeddings=False,
85 - rope_theta=1000000.0,
86 - use_sliding_window=False,
87 - sliding_window=4096,
88 - max_window_layers=80,
89 - attention_dropout=0.0,
90 - vision_config=None,
91 - rope_scaling=None,
92 - **kwargs,
93 - ):
94 - if isinstance(vision_config, dict):
95 - self.vision_config = Qwen2VLVisionConfig(**vision_config)
96 - elif vision_config is None:
97 - self.vision_config = Qwen2VLVisionConfig()
98 -
99 - self.vocab_size = vocab_size
100 - self.max_position_embeddings = max_position_embeddings
101 - self.hidden_size = hidden_size
102 - self.intermediate_size = intermediate_size
103 - self.num_hidden_layers = num_hidden_layers
104 - self.num_attention_heads = num_attention_heads
105 - self.use_sliding_window = use_sliding_window
106 - self.sliding_window = sliding_window
107 - self.max_window_layers = max_window_layers
108 -
109 - # for backward compatibility
110 - if num_key_value_heads is None:
111 - num_key_value_heads = num_attention_heads
112 -
113 - self.num_key_value_heads = num_key_value_heads
114 - self.hidden_act = hidden_act
115 - self.initializer_range = initializer_range
116 - self.rms_norm_eps = rms_norm_eps
117 - self.use_cache = use_cache
118 - self.rope_theta = rope_theta
119 - self.attention_dropout = attention_dropout
120 - self.rope_scaling = rope_scaling
121 -
122 - # NOTE: the following section from original transformers config
123 - # for Qwen2-VL is commented out to address rope config loading issue
124 - #
125 - # if self.rope_scaling is not None and "type" in self.rope_scaling:
126 - # if self.rope_scaling["type"] == "mrope":
127 - # self.rope_scaling["type"] = "default"
128 - # self.rope_scaling["rope_type"] = self.rope_scaling["type"]
129 - # rope_config_validation(self)
130 -
131 - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
vllm/worker/cpu_model_runner.py
+2 -4
@@ -19,6 +19,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19 MultiModalInputs)
20 from vllm.sequence import (IntermediateTensors, SequenceData,
21 SequenceGroupMetadata)
22 +from vllm.transformers_utils.config import uses_mrope
23 from vllm.utils import make_tensor_with_pad
24 from vllm.worker.model_runner_base import (
25 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -439,10 +440,7 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
440 def model_is_mrope(self) -> bool:
441 """Detect if the model has "mrope" rope_scaling type.
442 mrope requires keep "rope_deltas" between prompt and decoding phases."""
442 - rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
443 - if rope_scaling is None:
444 - return False
445 - return rope_scaling.get("type", None) == "mrope"
443 + return uses_mrope(self.model_config.hf_config)
444
445 def load_model(self) -> None:
446 self.model = get_model(model_config=self.model_config,
vllm/worker/model_runner.py
+2 -4
@@ -47,6 +47,7 @@ from vllm.prompt_adapter.worker_manager import (
47 LRUCacheWorkerPromptAdapterManager)
48 from vllm.sampling_params import SamplingParams
49 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50 +from vllm.transformers_utils.config import uses_mrope
51 from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52 flatten_2d_lists, is_hip, is_pin_memory_available,
53 supports_dynamo)
@@ -1379,10 +1380,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1380 def model_is_mrope(self) -> bool:
1381 """Detect if the model has "mrope" rope_scaling type.
1382 mrope requires keep "rope_deltas" between prompt and decoding phases."""
1382 - rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
1383 - if rope_scaling is None:
1384 - return False
1385 - return rope_scaling.get("type", None) == "mrope"
1383 + return uses_mrope(self.model_config.hf_config)
1384
1385 @torch.inference_mode()
1386 def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None: