A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] Remove unused utils and clean up imports (#15708)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 29, 2025 at 00:41 UTC
c6bc0034d0b8d3960d0a44565812ca253fc95943
4 files changed
+5
-195
tests/multimodal/test_utils.py
+1
-68
@@ -9,12 +9,10 @@ from typing import TYPE_CHECKING, NamedTuple, Optional
9
import numpy as np
10
import pytest
11
from PIL import Image, ImageChops
12
-from transformers import AutoConfig, AutoTokenizer
12
13
from vllm.multimodal.inputs import PlaceholderRange
14
from vllm.multimodal.utils import (MediaConnector,
16
- merge_and_sort_multimodal_metadata,
17
- repeat_and_pad_placeholder_tokens)
15
+ merge_and_sort_multimodal_metadata)
16
17
if TYPE_CHECKING:
18
from vllm.multimodal.hasher import MultiModalHashDict
@@ -136,71 +134,6 @@ async def test_fetch_image_local_files(image_url: str):
134
f"file://{temp_dir}/../{os.path.basename(image_url)}")
135
136
139
-@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"])
140
-def test_repeat_and_pad_placeholder_tokens(model):
141
- config = AutoConfig.from_pretrained(model)
142
- image_token_id = config.image_token_index
143
-
144
- tokenizer = AutoTokenizer.from_pretrained(model)
145
-
146
- test_cases = [
147
- (
148
- "<image>",
149
- 2,
150
- "<image><image>",
151
- [32000, 32000],
152
- [{ "offset": 0, "length": 2 }],
153
- ),
154
- (
155
- "<image><image>",
156
- 2,
157
- "<image><image><image>",
158
- [32000, 32000, 32000],
159
- [{ "offset": 0, "length": 2 }],
160
- ),
161
- (
162
- "<image><image>",
163
- [3, 2],
164
- "<image><image><image><image><image>",
165
- [32000, 32000, 32000, 32000, 32000],
166
- [{ "offset": 0, "length": 3 }, { "offset": 3, "length": 2 }],
167
- ),
168
- (
169
- "Image:<image>Image:<image>!",
170
- [3, 2],
171
- "Image:<image><image><image>Image:<image><image>!",
172
- [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
173
- [{ "offset": 2, "length": 3 }, { "offset": 7, "length": 2 }],
174
- ),
175
- (
176
- "<image>",
177
- [3, 2],
178
- "<image><image><image>",
179
- [32000, 32000, 32000],
180
- [{ "offset": 0, "length": 3 }],
181
- ),
182
- ] # yapf: disable
183
-
184
- for (
185
- prompt,
186
- repeat_count,
187
- expected_prompt,
188
- expected_token_ids,
189
- expected_ranges,
190
- ) in test_cases:
191
- new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
192
- tokenizer=tokenizer,
193
- prompt=prompt,
194
- prompt_token_ids=tokenizer.encode(prompt,
195
- add_special_tokens=False),
196
- placeholder_token_id=image_token_id,
197
- repeat_count=repeat_count,
198
- )
199
- assert new_prompt == expected_prompt
200
- assert new_token_ids == expected_token_ids
201
- assert ranges == expected_ranges
202
-
203
-
137
# Used for the next two tests related to `merge_and_sort_multimodal_metadata`.
138
class TestCase(NamedTuple):
139
mm_positions: "MultiModalPlaceholderDict"
vllm/multimodal/utils.py
-119
@@ -12,8 +12,6 @@ from PIL import Image
12
13
import vllm.envs as envs
14
from vllm.connections import HTTPConnection, global_http_connection
15
-from vllm.logger import init_logger
16
-from vllm.transformers_utils.tokenizer import AnyTokenizer
15
16
from .audio import AudioMediaIO
17
from .base import MediaIO
@@ -21,8 +19,6 @@ from .image import ImageEmbeddingMediaIO, ImageMediaIO
19
from .inputs import PlaceholderRange
20
from .video import VideoMediaIO
21
24
-logger = init_logger(__name__)
25
-
22
_M = TypeVar("_M")
23
24
if TYPE_CHECKING:
@@ -296,121 +292,6 @@ def encode_video_base64(frames: npt.NDArray) -> str:
292
return video_io.encode_base64(frames)
293
294
299
-# Utilities for input processors
300
-_T = TypeVar("_T", str, int)
301
-
302
-
303
-def repeat_and_pad_token(
304
- token: _T,
305
- *,
306
- repeat_count: int = 1,
307
- pad_token_left: Optional[_T] = None,
308
- pad_token_right: Optional[_T] = None,
309
-) -> list[_T]:
310
- replacement = [token] * repeat_count
311
- if pad_token_left is not None:
312
- replacement = [pad_token_left] + replacement
313
- if pad_token_right is not None:
314
- replacement = replacement + [pad_token_right]
315
-
316
- return replacement
317
-
318
-
319
-def repeat_and_pad_placeholder_tokens(
320
- tokenizer: AnyTokenizer,
321
- prompt: Optional[str],
322
- prompt_token_ids: list[int],
323
- *,
324
- placeholder_token_id: int,
325
- repeat_count: Union[int, list[int]],
326
- pad_token_left: Optional[int] = None,
327
- pad_token_right: Optional[int] = None,
328
-) -> tuple[Optional[str], list[int], list[PlaceholderRange]]:
329
- if isinstance(repeat_count, int):
330
- repeat_count = [repeat_count]
331
-
332
- if prompt is None:
333
- new_prompt = None
334
- else:
335
- placeholder_token_str = tokenizer.decode(placeholder_token_id)
336
- pad_token_str_left = (None if pad_token_left is None else
337
- tokenizer.decode(pad_token_left))
338
- pad_token_str_right = (None if pad_token_right is None else
339
- tokenizer.decode(pad_token_right))
340
-
341
- placeholder_token_count = prompt.count(placeholder_token_str)
342
- # This is an arbitrary number to distinguish between the two cases
343
- if placeholder_token_count > 16:
344
- logger.warning(
345
- "Please follow the prompt format that is "
346
- "documented on HuggingFace which does not involve "
347
- "repeating %s tokens.", placeholder_token_str)
348
- if placeholder_token_count < len(repeat_count):
349
- logger.warning(
350
- "The number of multi-modal placeholder tokens in the prompt "
351
- "is less than the number of multi-modal inputs. Extra "
352
- "placeholder tokens will be treated as plain text")
353
- repeat_count = repeat_count[:placeholder_token_count]
354
-
355
- prompt_parts = prompt.split(placeholder_token_str,
356
- maxsplit=len(repeat_count))
357
- new_prompt = ""
358
- for i, repeat_count_item in enumerate(repeat_count):
359
- replacement_str = "".join(
360
- repeat_and_pad_token(
361
- placeholder_token_str,
362
- repeat_count=repeat_count_item,
363
- pad_token_left=pad_token_str_left,
364
- pad_token_right=pad_token_str_right,
365
- ))
366
- # The image tokens are removed to be consistent with HuggingFace
367
- new_prompt += prompt_parts[i] + replacement_str
368
- new_prompt += prompt_parts[-1]
369
-
370
- new_token_ids = list[int]()
371
- placeholder_ranges = list[PlaceholderRange]()
372
- placeholder_token_idx = 0
373
- for i, token in enumerate(prompt_token_ids):
374
- if token == placeholder_token_id:
375
- curr_repeat_count = repeat_count[placeholder_token_idx]
376
- replacement_ids = repeat_and_pad_token(
377
- placeholder_token_id,
378
- repeat_count=curr_repeat_count,
379
- pad_token_left=pad_token_left,
380
- pad_token_right=pad_token_right,
381
- )
382
- offset = len(new_token_ids)
383
- if pad_token_left is not None:
384
- offset += 1
385
- placeholder_ranges.append({
386
- "offset": offset,
387
- "length": curr_repeat_count,
388
- })
389
- new_token_ids.extend(replacement_ids)
390
- placeholder_token_idx += 1
391
-
392
- # No need to further scan the list since we replaced all tokens
393
- if placeholder_token_idx >= len(repeat_count):
394
- new_token_ids.extend(prompt_token_ids[i + 1:])
395
- break
396
- else:
397
- new_token_ids.append(token)
398
-
399
- return new_prompt, new_token_ids, placeholder_ranges
400
-
401
-
402
-def consecutive_placeholder_ranges(
403
- num_items: int,
404
- item_size: int,
405
- initial_offset: int = 0) -> list[PlaceholderRange]:
406
- """Returns a list of consecutive PlaceholderRanges of a fixed size"""
407
-
408
- return [
409
- PlaceholderRange(offset=initial_offset + i * item_size,
410
- length=item_size) for i in range(num_items)
411
- ]
412
-
413
-
295
def merge_and_sort_multimodal_metadata(
296
mm_positions: "MultiModalPlaceholderDict",
297
mm_hashes: Optional["MultiModalHashDict"],
vllm/v1/core/sched/output.py
+1
-2
@@ -10,8 +10,7 @@ if TYPE_CHECKING:
10
import numpy.typing as npt
11
12
from vllm.lora.request import LoRARequest
13
- from vllm.multimodal import MultiModalKwargs
14
- from vllm.multimodal.base import PlaceholderRange
13
+ from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
14
from vllm.sampling_params import SamplingParams
15
from vllm.v1.request import Request
16
vllm/v1/worker/gpu_input_batch.py
+3
-6
@@ -2,13 +2,13 @@
2
# Datastructures defining an input batch
3
4
from dataclasses import dataclass
5
-from typing import TYPE_CHECKING, Optional, cast
5
+from typing import Optional, cast
6
7
import numpy as np
8
import torch
9
10
from vllm.lora.request import LoRARequest
11
-from vllm.multimodal import MultiModalKwargs
11
+from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
12
from vllm.sampling_params import SamplingParams, SamplingType
13
from vllm.utils import swap_dict_values
14
from vllm.v1.outputs import LogprobsTensors
@@ -18,9 +18,6 @@ from vllm.v1.worker.block_table import BlockTable
18
19
_SAMPLING_EPS = 1e-5
20
21
-if TYPE_CHECKING:
22
- from vllm.multimodal.inputs import PlaceholderRange
23
-
21
22
@dataclass
23
class CachedRequestState:
@@ -29,7 +26,7 @@ class CachedRequestState:
26
prompt_token_ids: list[int]
27
prompt: Optional[str]
28
mm_inputs: list[MultiModalKwargs]
32
- mm_positions: list["PlaceholderRange"]
29
+ mm_positions: list[PlaceholderRange]
30
sampling_params: SamplingParams
31
generator: Optional[torch.Generator]
32