@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e85f4f82a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[VLM] Use `SequenceData.from_token_counts` to create dummy data (#8687)

Cyrus Leung committed Sep 21, 2024 at 14:28 UTC 5e85f4f82a5b6eaad6869198d6ac76a0c12cf6d0
12 files changed +73 -80
vllm/inputs/registry.py
+1 -1
@@ -125,7 +125,7 @@ class InputRegistry:
125 # Avoid circular import
126 from vllm.sequence import SequenceData
127
128 - dummy_seq_data = SequenceData.from_counts({0: seq_len})
128 + dummy_seq_data = SequenceData.from_token_counts((0, seq_len))
129 dummy_multi_modal_data = None
130
131 return dummy_seq_data, dummy_multi_modal_data
vllm/model_executor/models/blip.py
+6 -7
@@ -1,6 +1,5 @@
1 """Minimal implementation of BlipVisionModel intended to be only used
2 within a vision language model."""
3 -from array import array
3 from typing import Optional, Union
4
5 import torch
@@ -19,7 +18,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 from vllm.multimodal.utils import (cached_get_tokenizer,
20 repeat_and_pad_placeholder_tokens)
22 -from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
21 +from vllm.sequence import SequenceData
22
23 try:
24 from xformers import ops as xops
@@ -53,6 +52,7 @@ def get_max_blip_image_tokens(
52 def dummy_seq_data_for_blip(
53 hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
54 seq_len: int,
55 + num_images: int,
56 *,
57 image_token_id: int,
58 image_feature_size_override: Optional[int] = None,
@@ -62,11 +62,10 @@ def dummy_seq_data_for_blip(
62 else:
63 image_feature_size = image_feature_size_override
64
65 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
66 - [image_token_id]) * image_feature_size
67 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
68 - [0]) * (seq_len - image_feature_size)
69 - return SequenceData(token_ids)
65 + return SequenceData.from_token_counts(
66 + (image_token_id, image_feature_size * num_images),
67 + (0, seq_len - image_feature_size * num_images),
68 + )
69
70
71 def dummy_image_for_blip(
vllm/model_executor/models/blip2.py
+5 -8
@@ -1,4 +1,3 @@
1 -from array import array
1 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 TypedDict, Union)
3
@@ -18,8 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.model_executor.models.opt import OPTModel
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
21 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
22 - SequenceData)
20 +from vllm.sequence import IntermediateTensors, SequenceData
21
22 from .blip import (BlipVisionModel, dummy_image_for_blip,
23 get_max_blip_image_tokens)
@@ -429,11 +427,10 @@ def dummy_seq_data_for_blip2(
427 else:
428 image_feature_size = image_feature_size_override
429
432 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
433 - [image_token_id]) * image_feature_size * num_images
434 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
435 - [0]) * (seq_len - image_feature_size * num_images)
436 - return SequenceData(token_ids)
430 + return SequenceData.from_token_counts(
431 + (image_token_id, image_feature_size * num_images),
432 + (0, seq_len - image_feature_size * num_images),
433 + )
434
435
436 def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
vllm/model_executor/models/chameleon.py
+5 -8
@@ -1,4 +1,3 @@
1 -from array import array
1 from functools import cached_property
2 from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
3 Tuple, TypedDict)
@@ -32,8 +31,7 @@ from vllm.model_executor.utils import set_weight_attrs
31 from vllm.multimodal import MULTIMODAL_REGISTRY
32 from vllm.multimodal.utils import (cached_get_tokenizer,
33 repeat_and_pad_placeholder_tokens)
35 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36 - SequenceData)
34 +from vllm.sequence import IntermediateTensors, SequenceData
35 from vllm.utils import print_warning_once
36
37 from .interfaces import SupportsMultiModal
@@ -72,11 +70,10 @@ def dummy_seq_data_for_chameleon(
70 else:
71 image_feature_size = image_feature_size_override
72
75 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
76 - [image_token_id]) * image_feature_size * num_images
77 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
78 - [0]) * (seq_len - image_feature_size * num_images)
79 - return SequenceData(token_ids)
73 + return SequenceData.from_token_counts(
74 + (image_token_id, image_feature_size * num_images),
75 + (0, seq_len - image_feature_size * num_images),
76 + )
77
78
79 def dummy_image_for_chameleon(
vllm/model_executor/models/clip.py
+5 -7
@@ -1,6 +1,5 @@
1 """Minimal implementation of CLIPVisionModel intended to be only used
2 within a vision language model."""
3 -from array import array
3 from typing import Iterable, List, Optional, Tuple, Union
4
5 import torch
@@ -20,7 +19,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 from vllm.multimodal.utils import (cached_get_tokenizer,
21 repeat_and_pad_placeholder_tokens)
23 -from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
22 +from vllm.sequence import SequenceData
23
24 try:
25 from xformers import ops as xops
@@ -62,11 +61,10 @@ def dummy_seq_data_for_clip(
61 else:
62 image_feature_size = image_feature_size_override
63
65 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
66 - [image_token_id]) * image_feature_size * num_images
67 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
68 - [0]) * (seq_len - image_feature_size * num_images)
69 - return SequenceData(token_ids)
64 + return SequenceData.from_token_counts(
65 + (image_token_id, image_feature_size * num_images),
66 + (0, seq_len - image_feature_size * num_images),
67 + )
68
69
70 def dummy_image_for_clip(
vllm/model_executor/models/minicpmv.py
+2 -5
@@ -23,7 +23,6 @@
23 """Inference-only MiniCPM-V model compatible with HuggingFace weights."""
24 import math
25 import re
26 -from array import array
26 from functools import partial
27 from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
28 TypedDict)
@@ -56,8 +55,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
55 from vllm.multimodal import MULTIMODAL_REGISTRY
56 from vllm.multimodal.image import cached_get_image_processor
57 from vllm.multimodal.utils import cached_get_tokenizer
59 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60 - SequenceData)
58 +from vllm.sequence import IntermediateTensors, SequenceData
59
60 from .idefics2_vision_model import Idefics2VisionTransformer
61
@@ -259,8 +257,7 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
257
258
259 def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
262 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len
263 - return SequenceData(token_ids)
260 + return SequenceData.from_token_counts((0, seq_len))
261
262
263 def dummy_image_for_minicpmv(hf_config: PretrainedConfig, num_images: int):
vllm/model_executor/models/pixtral.py
+5 -9
@@ -1,4 +1,3 @@
1 -from array import array
1 from dataclasses import dataclass, fields
2 from itertools import tee
3 from typing import Iterable, List, Mapping, Optional, Tuple, Union
@@ -24,8 +23,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 from vllm.multimodal.base import MultiModalInputs
25 from vllm.multimodal.utils import cached_get_tokenizer
27 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
28 - SequenceData)
26 +from vllm.sequence import IntermediateTensors, SequenceData
27
28 from .interfaces import SupportsMultiModal
29 from .utils import init_vllm_registered_model
@@ -63,13 +61,11 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
61 image_feature_size = (size**2) // (patch_size**2)
62
63 num_image_tokens = image_feature_size * num_images
64 + seq_data = SequenceData.from_token_counts(
65 + (image_token_id, num_image_tokens),
66 + (0, seq_len - num_image_tokens),
67 + )
68
67 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
68 - [image_token_id]) * num_image_tokens
69 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
70 - [0]) * (seq_len - num_image_tokens)
71 -
72 - seq_data = SequenceData(token_ids)
69 mm_data = {"image": num_images * [image]}
70 return seq_data, mm_data
71
vllm/model_executor/models/qwen.py
+5 -5
@@ -7,7 +7,6 @@
7
8 import math
9 import re
10 -from array import array
10 from functools import partial
11 from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
12 Optional, Tuple, TypedDict, Union)
@@ -45,8 +44,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
44 from vllm.multimodal import MULTIMODAL_REGISTRY
45 from vllm.multimodal.base import MultiModalInputs
46 from vllm.multimodal.utils import cached_get_tokenizer
48 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
49 - SequenceData)
47 +from vllm.sequence import IntermediateTensors, SequenceData
48 from vllm.utils import is_list_of
49
50 from .utils import flatten_bn, is_pp_missing_parameter, make_layers
@@ -819,7 +817,7 @@ def dummy_data_for_qwen(
817 # The presence of a visual config indicates this is a multimodal model.
818 # If we don't have it, the model is considered an LLM for warmup purposes.
819 if not hasattr(hf_config, "visual"):
822 - seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len))
820 + seq_data = SequenceData.from_token_counts((0, seq_len))
821 mm_data = None
822 return seq_data, mm_data
823
@@ -846,11 +844,13 @@ def dummy_data_for_qwen(
844 if len(toks) < seq_len:
845 toks += [0] * (seq_len - len(toks))
846
847 + seq_data = SequenceData.from_seqs(toks)
848 +
849 # Build the input images; width/height doesn't actually matter here since
850 # the data will get resized and the # of tokens per image is constant
851 image = Image.new("RGB", (224, 224), color=0)
852 mm_data = {"image": image if num_images == 1 else [image] * num_images}
853 - return SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, toks)), mm_data
853 + return seq_data, mm_data
854
855
856 @MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
vllm/model_executor/models/qwen2_vl.py
+9 -12
@@ -22,7 +22,6 @@
22 # See the License for the specific language governing permissions and
23 # limitations under the License.
24 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25 -from array import array
25 from functools import lru_cache, partial
26 from typing import (Iterable, List, Mapping, Optional, Tuple, Type, TypedDict,
27 Union)
@@ -66,8 +65,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
65 from vllm.multimodal.base import MultiModalData
66 from vllm.multimodal.image import cached_get_image_processor
67 from vllm.platforms import current_platform
69 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
70 - SequenceData)
68 +from vllm.sequence import IntermediateTensors, SequenceData
69 from vllm.transformers_utils.processor import get_processor
70
71 logger = init_logger(__name__)
@@ -681,15 +679,14 @@ def dummy_data_for_qwen2_vl(
679 "--limit-mm-per-prompt.")
680
681 hf_config = ctx.get_hf_config(Qwen2VLConfig)
684 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
685 - [hf_config.vision_start_token_id])
686 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
687 - [hf_config.image_token_id]) * max_llm_image_tokens
688 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
689 - [hf_config.vision_end_token_id])
690 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
691 - [0]) * (seq_len - max_llm_image_tokens - 2)
692 - dummy_seqdata = SequenceData(token_ids)
682 +
683 + dummy_seqdata = SequenceData.from_token_counts(
684 + (hf_config.vision_start_token_id, 1),
685 + (hf_config.image_token_id, max_llm_image_tokens),
686 + (hf_config.vision_end_token_id, 1),
687 + (0, seq_len - max_llm_image_tokens - 2),
688 + )
689 +
690 dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
691 color=0)
692
vllm/model_executor/models/siglip.py
+5 -7
@@ -2,7 +2,6 @@
2 within a vision language model."""
3
4 import math
5 -from array import array
5 from typing import Iterable, List, Optional, Tuple, Union
6
7 import torch
@@ -24,7 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.multimodal.utils import (cached_get_tokenizer,
25 repeat_and_pad_placeholder_tokens)
27 -from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
26 +from vllm.sequence import SequenceData
27
28 try:
29 from xformers import ops as xops
@@ -67,11 +66,10 @@ def dummy_seq_data_for_siglip(
66 else:
67 image_feature_size = image_feature_size_override
68
70 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
71 - [image_token_id]) * image_feature_size
72 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
73 - [0]) * (seq_len - image_feature_size)
74 - return SequenceData(token_ids)
69 + return SequenceData.from_token_counts(
70 + (image_token_id, image_feature_size * num_images),
71 + (0, seq_len - image_feature_size * num_images),
72 + )
73
74
75 def dummy_image_for_siglip(
vllm/model_executor/models/ultravox.py
+22 -8
@@ -77,15 +77,11 @@ def get_ultravox_max_audio_tokens(ctx: InputContext):
77 return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND)
78
79
80 -def dummy_data_for_ultravox(
80 +def dummy_seq_data_for_ultravox(
81 ctx: InputContext,
82 seq_len: int,
83 - mm_counts: Mapping[str, int],
83 + audio_count: int,
84 ):
85 - feature_extractor = whisper_feature_extractor(ctx)
86 -
87 - audio_count = mm_counts["audio"]
88 -
85 audio_placeholder = array(
86 VLLM_TOKEN_ID_ARRAY_TYPE,
87 [_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx)
@@ -96,10 +92,28 @@ def dummy_data_for_ultravox(
92 other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
93 [0]) * (seq_len - len(audio_token_ids))
94
95 + return SequenceData(audio_token_ids + other_token_ids)
96 +
97 +
98 +def dummy_audio_for_ultravox(
99 + ctx: InputContext,
100 + audio_count: int,
101 +):
102 + feature_extractor = whisper_feature_extractor(ctx)
103 audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1)
100 - mm_dict = {"audio": [audio_and_sr] * audio_count}
104 + return {"audio": [audio_and_sr] * audio_count}
105 +
106 +
107 +def dummy_data_for_ultravox(
108 + ctx: InputContext,
109 + seq_len: int,
110 + mm_counts: Mapping[str, int],
111 +):
112 + audio_count = mm_counts["audio"]
113 + seq_data = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count)
114 + mm_dict = dummy_audio_for_ultravox(ctx, audio_count)
115
102 - return (SequenceData(audio_token_ids + other_token_ids), mm_dict)
116 + return (seq_data, mm_dict)
117
118
119 def input_mapper_for_ultravox(ctx: InputContext, data: object):
vllm/sequence.py
+3 -3
@@ -171,13 +171,13 @@ class SequenceData(msgspec.Struct,
171 _mrope_position_delta: Optional[int] = None
172
173 @staticmethod
174 - def from_counts(counts_by_token: Mapping[int, int]) -> "SequenceData":
175 - if len(counts_by_token) == 0:
174 + def from_token_counts(*token_counts: Tuple[int, int]) -> "SequenceData":
175 + if len(token_counts) == 0:
176 return SequenceData.from_seqs([])
177
178 arrs = [
179 array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
180 - for token_id, count in counts_by_token.items()
180 + for token_id, count in token_counts
181 ]
182
183 return SequenceData(reduce(array.__add__, arrs))