A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[VLM] Use `SequenceData.from_token_counts` to create dummy data (#8687)
Cyrus Leung committed
Sep 21, 2024 at 14:28 UTC
5e85f4f82a5b6eaad6869198d6ac76a0c12cf6d0
12 files changed
+73
-80
vllm/inputs/registry.py
+1
-1
@@ -125,7 +125,7 @@ class InputRegistry:
125
# Avoid circular import
126
from vllm.sequence import SequenceData
127
128
- dummy_seq_data = SequenceData.from_counts({0: seq_len})
128
+ dummy_seq_data = SequenceData.from_token_counts((0, seq_len))
129
dummy_multi_modal_data = None
130
131
return dummy_seq_data, dummy_multi_modal_data
vllm/model_executor/models/blip.py
+6
-7
@@ -1,6 +1,5 @@
1
"""Minimal implementation of BlipVisionModel intended to be only used
2
within a vision language model."""
3
-from array import array
3
from typing import Optional, Union
4
5
import torch
@@ -19,7 +18,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.multimodal.utils import (cached_get_tokenizer,
20
repeat_and_pad_placeholder_tokens)
22
-from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
21
+from vllm.sequence import SequenceData
22
23
try:
24
from xformers import ops as xops
@@ -53,6 +52,7 @@ def get_max_blip_image_tokens(
52
def dummy_seq_data_for_blip(
53
hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
54
seq_len: int,
55
+ num_images: int,
56
*,
57
image_token_id: int,
58
image_feature_size_override: Optional[int] = None,
@@ -62,11 +62,10 @@ def dummy_seq_data_for_blip(
62
else:
63
image_feature_size = image_feature_size_override
64
65
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
66
- [image_token_id]) * image_feature_size
67
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
68
- [0]) * (seq_len - image_feature_size)
69
- return SequenceData(token_ids)
65
+ return SequenceData.from_token_counts(
66
+ (image_token_id, image_feature_size * num_images),
67
+ (0, seq_len - image_feature_size * num_images),
68
+ )
69
70
71
def dummy_image_for_blip(
vllm/model_executor/models/blip2.py
+5
-8
@@ -1,4 +1,3 @@
1
-from array import array
1
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
TypedDict, Union)
3
@@ -18,8 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.models.opt import OPTModel
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
21
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
22
- SequenceData)
20
+from vllm.sequence import IntermediateTensors, SequenceData
21
22
from .blip import (BlipVisionModel, dummy_image_for_blip,
23
get_max_blip_image_tokens)
@@ -429,11 +427,10 @@ def dummy_seq_data_for_blip2(
427
else:
428
image_feature_size = image_feature_size_override
429
432
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
433
- [image_token_id]) * image_feature_size * num_images
434
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
435
- [0]) * (seq_len - image_feature_size * num_images)
436
- return SequenceData(token_ids)
430
+ return SequenceData.from_token_counts(
431
+ (image_token_id, image_feature_size * num_images),
432
+ (0, seq_len - image_feature_size * num_images),
433
+ )
434
435
436
def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
vllm/model_executor/models/chameleon.py
+5
-8
@@ -1,4 +1,3 @@
1
-from array import array
1
from functools import cached_property
2
from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
3
Tuple, TypedDict)
@@ -32,8 +31,7 @@ from vllm.model_executor.utils import set_weight_attrs
31
from vllm.multimodal import MULTIMODAL_REGISTRY
32
from vllm.multimodal.utils import (cached_get_tokenizer,
33
repeat_and_pad_placeholder_tokens)
35
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36
- SequenceData)
34
+from vllm.sequence import IntermediateTensors, SequenceData
35
from vllm.utils import print_warning_once
36
37
from .interfaces import SupportsMultiModal
@@ -72,11 +70,10 @@ def dummy_seq_data_for_chameleon(
70
else:
71
image_feature_size = image_feature_size_override
72
75
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
76
- [image_token_id]) * image_feature_size * num_images
77
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
78
- [0]) * (seq_len - image_feature_size * num_images)
79
- return SequenceData(token_ids)
73
+ return SequenceData.from_token_counts(
74
+ (image_token_id, image_feature_size * num_images),
75
+ (0, seq_len - image_feature_size * num_images),
76
+ )
77
78
79
def dummy_image_for_chameleon(
vllm/model_executor/models/clip.py
+5
-7
@@ -1,6 +1,5 @@
1
"""Minimal implementation of CLIPVisionModel intended to be only used
2
within a vision language model."""
3
-from array import array
3
from typing import Iterable, List, Optional, Tuple, Union
4
5
import torch
@@ -20,7 +19,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.multimodal.utils import (cached_get_tokenizer,
21
repeat_and_pad_placeholder_tokens)
23
-from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
22
+from vllm.sequence import SequenceData
23
24
try:
25
from xformers import ops as xops
@@ -62,11 +61,10 @@ def dummy_seq_data_for_clip(
61
else:
62
image_feature_size = image_feature_size_override
63
65
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
66
- [image_token_id]) * image_feature_size * num_images
67
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
68
- [0]) * (seq_len - image_feature_size * num_images)
69
- return SequenceData(token_ids)
64
+ return SequenceData.from_token_counts(
65
+ (image_token_id, image_feature_size * num_images),
66
+ (0, seq_len - image_feature_size * num_images),
67
+ )
68
69
70
def dummy_image_for_clip(
vllm/model_executor/models/minicpmv.py
+2
-5
@@ -23,7 +23,6 @@
23
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
24
import math
25
import re
26
-from array import array
26
from functools import partial
27
from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
28
TypedDict)
@@ -56,8 +55,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
55
from vllm.multimodal import MULTIMODAL_REGISTRY
56
from vllm.multimodal.image import cached_get_image_processor
57
from vllm.multimodal.utils import cached_get_tokenizer
59
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60
- SequenceData)
58
+from vllm.sequence import IntermediateTensors, SequenceData
59
60
from .idefics2_vision_model import Idefics2VisionTransformer
61
@@ -259,8 +257,7 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
257
258
259
def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
262
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len
263
- return SequenceData(token_ids)
260
+ return SequenceData.from_token_counts((0, seq_len))
261
262
263
def dummy_image_for_minicpmv(hf_config: PretrainedConfig, num_images: int):
vllm/model_executor/models/pixtral.py
+5
-9
@@ -1,4 +1,3 @@
1
-from array import array
1
from dataclasses import dataclass, fields
2
from itertools import tee
3
from typing import Iterable, List, Mapping, Optional, Tuple, Union
@@ -24,8 +23,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.base import MultiModalInputs
25
from vllm.multimodal.utils import cached_get_tokenizer
27
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
28
- SequenceData)
26
+from vllm.sequence import IntermediateTensors, SequenceData
27
28
from .interfaces import SupportsMultiModal
29
from .utils import init_vllm_registered_model
@@ -63,13 +61,11 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
61
image_feature_size = (size**2) // (patch_size**2)
62
63
num_image_tokens = image_feature_size * num_images
64
+ seq_data = SequenceData.from_token_counts(
65
+ (image_token_id, num_image_tokens),
66
+ (0, seq_len - num_image_tokens),
67
+ )
68
67
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
68
- [image_token_id]) * num_image_tokens
69
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
70
- [0]) * (seq_len - num_image_tokens)
71
-
72
- seq_data = SequenceData(token_ids)
69
mm_data = {"image": num_images * [image]}
70
return seq_data, mm_data
71
vllm/model_executor/models/qwen.py
+5
-5
@@ -7,7 +7,6 @@
7
8
import math
9
import re
10
-from array import array
10
from functools import partial
11
from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
12
Optional, Tuple, TypedDict, Union)
@@ -45,8 +44,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
44
from vllm.multimodal import MULTIMODAL_REGISTRY
45
from vllm.multimodal.base import MultiModalInputs
46
from vllm.multimodal.utils import cached_get_tokenizer
48
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
49
- SequenceData)
47
+from vllm.sequence import IntermediateTensors, SequenceData
48
from vllm.utils import is_list_of
49
50
from .utils import flatten_bn, is_pp_missing_parameter, make_layers
@@ -819,7 +817,7 @@ def dummy_data_for_qwen(
817
# The presence of a visual config indicates this is a multimodal model.
818
# If we don't have it, the model is considered an LLM for warmup purposes.
819
if not hasattr(hf_config, "visual"):
822
- seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len))
820
+ seq_data = SequenceData.from_token_counts((0, seq_len))
821
mm_data = None
822
return seq_data, mm_data
823
@@ -846,11 +844,13 @@ def dummy_data_for_qwen(
844
if len(toks) < seq_len:
845
toks += [0] * (seq_len - len(toks))
846
847
+ seq_data = SequenceData.from_seqs(toks)
848
+
849
# Build the input images; width/height doesn't actually matter here since
850
# the data will get resized and the # of tokens per image is constant
851
image = Image.new("RGB", (224, 224), color=0)
852
mm_data = {"image": image if num_images == 1 else [image] * num_images}
853
- return SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, toks)), mm_data
853
+ return seq_data, mm_data
854
855
856
@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
vllm/model_executor/models/qwen2_vl.py
+9
-12
@@ -22,7 +22,6 @@
22
# See the License for the specific language governing permissions and
23
# limitations under the License.
24
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25
-from array import array
25
from functools import lru_cache, partial
26
from typing import (Iterable, List, Mapping, Optional, Tuple, Type, TypedDict,
27
Union)
@@ -66,8 +65,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
65
from vllm.multimodal.base import MultiModalData
66
from vllm.multimodal.image import cached_get_image_processor
67
from vllm.platforms import current_platform
69
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
70
- SequenceData)
68
+from vllm.sequence import IntermediateTensors, SequenceData
69
from vllm.transformers_utils.processor import get_processor
70
71
logger = init_logger(__name__)
@@ -681,15 +679,14 @@ def dummy_data_for_qwen2_vl(
679
"--limit-mm-per-prompt.")
680
681
hf_config = ctx.get_hf_config(Qwen2VLConfig)
684
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
685
- [hf_config.vision_start_token_id])
686
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
687
- [hf_config.image_token_id]) * max_llm_image_tokens
688
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
689
- [hf_config.vision_end_token_id])
690
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
691
- [0]) * (seq_len - max_llm_image_tokens - 2)
692
- dummy_seqdata = SequenceData(token_ids)
682
+
683
+ dummy_seqdata = SequenceData.from_token_counts(
684
+ (hf_config.vision_start_token_id, 1),
685
+ (hf_config.image_token_id, max_llm_image_tokens),
686
+ (hf_config.vision_end_token_id, 1),
687
+ (0, seq_len - max_llm_image_tokens - 2),
688
+ )
689
+
690
dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
691
color=0)
692
vllm/model_executor/models/siglip.py
+5
-7
@@ -2,7 +2,6 @@
2
within a vision language model."""
3
4
import math
5
-from array import array
5
from typing import Iterable, List, Optional, Tuple, Union
6
7
import torch
@@ -24,7 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.multimodal.utils import (cached_get_tokenizer,
25
repeat_and_pad_placeholder_tokens)
27
-from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
26
+from vllm.sequence import SequenceData
27
28
try:
29
from xformers import ops as xops
@@ -67,11 +66,10 @@ def dummy_seq_data_for_siglip(
66
else:
67
image_feature_size = image_feature_size_override
68
70
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
71
- [image_token_id]) * image_feature_size
72
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
73
- [0]) * (seq_len - image_feature_size)
74
- return SequenceData(token_ids)
69
+ return SequenceData.from_token_counts(
70
+ (image_token_id, image_feature_size * num_images),
71
+ (0, seq_len - image_feature_size * num_images),
72
+ )
73
74
75
def dummy_image_for_siglip(
vllm/model_executor/models/ultravox.py
+22
-8
@@ -77,15 +77,11 @@ def get_ultravox_max_audio_tokens(ctx: InputContext):
77
return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND)
78
79
80
-def dummy_data_for_ultravox(
80
+def dummy_seq_data_for_ultravox(
81
ctx: InputContext,
82
seq_len: int,
83
- mm_counts: Mapping[str, int],
83
+ audio_count: int,
84
):
85
- feature_extractor = whisper_feature_extractor(ctx)
86
-
87
- audio_count = mm_counts["audio"]
88
-
85
audio_placeholder = array(
86
VLLM_TOKEN_ID_ARRAY_TYPE,
87
[_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx)
@@ -96,10 +92,28 @@ def dummy_data_for_ultravox(
92
other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
93
[0]) * (seq_len - len(audio_token_ids))
94
95
+ return SequenceData(audio_token_ids + other_token_ids)
96
+
97
+
98
+def dummy_audio_for_ultravox(
99
+ ctx: InputContext,
100
+ audio_count: int,
101
+):
102
+ feature_extractor = whisper_feature_extractor(ctx)
103
audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1)
100
- mm_dict = {"audio": [audio_and_sr] * audio_count}
104
+ return {"audio": [audio_and_sr] * audio_count}
105
+
106
+
107
+def dummy_data_for_ultravox(
108
+ ctx: InputContext,
109
+ seq_len: int,
110
+ mm_counts: Mapping[str, int],
111
+):
112
+ audio_count = mm_counts["audio"]
113
+ seq_data = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count)
114
+ mm_dict = dummy_audio_for_ultravox(ctx, audio_count)
115
102
- return (SequenceData(audio_token_ids + other_token_ids), mm_dict)
116
+ return (seq_data, mm_dict)
117
118
119
def input_mapper_for_ultravox(ctx: InputContext, data: object):
vllm/sequence.py
+3
-3
@@ -171,13 +171,13 @@ class SequenceData(msgspec.Struct,
171
_mrope_position_delta: Optional[int] = None
172
173
@staticmethod
174
- def from_counts(counts_by_token: Mapping[int, int]) -> "SequenceData":
175
- if len(counts_by_token) == 0:
174
+ def from_token_counts(*token_counts: Tuple[int, int]) -> "SequenceData":
175
+ if len(token_counts) == 0:
176
return SequenceData.from_seqs([])
177
178
arrs = [
179
array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
180
- for token_id, count in counts_by_token.items()
180
+ for token_id, count in token_counts
181
]
182
183
return SequenceData(reduce(array.__add__, arrs))