@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3dd0692fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[BugFix] Propagate 'trust_remote_code' setting in internvl and minicpmv (#8250)

zifeitong committed Sep 24, 2024 at 22:53 UTC e3dd0692fa2c803cd6f59a88d2fdf8bca26d8d96
3 files changed +126 -41
vllm/model_executor/models/internvl.py
+9 -6
@@ -230,8 +230,9 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
230 else:
231 raise TypeError(f"Invalid image type: {type(image_data)}")
232
233 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
234 - trust_remote_code=True)
233 + tokenizer = cached_get_tokenizer(
234 + model_config.tokenizer,
235 + trust_remote_code=model_config.trust_remote_code)
236
237 prompt = llm_inputs.get("prompt")
238 prompt_token_ids = llm_inputs["prompt_token_ids"]
@@ -278,8 +279,9 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
279 use_thumbnail=use_thumbnail) for img in data
280 ]
281 model_config = ctx.model_config
281 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
282 - trust_remote_code=True)
282 + tokenizer = cached_get_tokenizer(
283 + model_config.tokenizer,
284 + trust_remote_code=model_config.trust_remote_code)
285 image_token_id = tokenizer.encode(IMG_CONTEXT,
286 add_special_tokens=False,
287 return_tensors="pt")[0]
@@ -298,8 +300,9 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int,
300 model_config = ctx.model_config
301 hf_config = ctx.get_hf_config()
302 vision_config = hf_config.vision_config
301 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
302 - trust_remote_code=True)
303 + tokenizer = cached_get_tokenizer(
304 + model_config.tokenizer,
305 + trust_remote_code=model_config.trust_remote_code)
306
307 seq_data = dummy_seq_data_for_clip(
308 vision_config,
vllm/model_executor/models/minicpmv.py
+108 -29
@@ -33,6 +33,7 @@ from PIL import Image
33 from torch import nn
34 from torch.nn.init import trunc_normal_
35 from transformers import PretrainedConfig
36 +from typing_extensions import NotRequired
37
38 from vllm.attention import AttentionMetadata
39 from vllm.config import CacheConfig, MultiModalConfig
@@ -52,6 +53,7 @@ from vllm.model_executor.models.minicpm import MiniCPMModel
53 from vllm.model_executor.models.qwen2 import Qwen2Model
54 from vllm.model_executor.sampling_metadata import SamplingMetadata
55 from vllm.multimodal import MULTIMODAL_REGISTRY
56 +from vllm.multimodal.base import MultiModalInputs
57 from vllm.multimodal.image import cached_get_image_processor
58 from vllm.multimodal.utils import cached_get_tokenizer
59 from vllm.sequence import IntermediateTensors, SequenceData
@@ -64,6 +66,17 @@ _KEYS_TO_MODIFY_MAPPING = {
66 }
67
68
69 +class MiniCPMVImageInput(TypedDict):
70 + """Input mapper input with auxiliary data for computing image bounds."""
71 + image: Image.Image
72 +
73 + # Image bounds token ids in 0-dim scaler tensor.
74 + im_start_id: torch.Tensor
75 + im_end_id: torch.Tensor
76 + slice_start_id: NotRequired[torch.Tensor]
77 + slice_end_id: NotRequired[torch.Tensor]
78 +
79 +
80 class MiniCPMVImagePixelInputs(TypedDict):
81 pixel_values: List[torch.Tensor]
82 """
@@ -88,8 +101,6 @@ class MiniCPMVImagePixelInputs(TypedDict):
101 """
102
103
91 -MiniCPMVImageInputs = MiniCPMVImagePixelInputs
92 -
104 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
105
106
@@ -234,6 +245,25 @@ class Resampler2_5(BaseResampler):
245 return x
246
247
248 +def _build_image_input(ctx: InputContext,
249 + image: Image.Image) -> MiniCPMVImageInput:
250 + tokenizer = cached_get_tokenizer(
251 + ctx.model_config.tokenizer,
252 + trust_remote_code=ctx.model_config.trust_remote_code)
253 + if hasattr(tokenizer, "slice_start_id"):
254 + return MiniCPMVImageInput(
255 + image=image,
256 + im_start_id=torch.tensor(tokenizer.im_start_id),
257 + im_end_id=torch.tensor(tokenizer.im_end_id),
258 + slice_start_id=torch.tensor(tokenizer.slice_start_id),
259 + slice_end_id=torch.tensor(tokenizer.slice_end_id))
260 + else:
261 + return MiniCPMVImageInput(image=image,
262 + im_start_id=torch.tensor(
263 + tokenizer.im_start_id),
264 + im_end_id=torch.tensor(tokenizer.im_end_id))
265 +
266 +
267 def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
268 version_float = getattr(config, "version", None)
269
@@ -257,10 +287,13 @@ def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
287 return SequenceData.from_token_counts((0, seq_len))
288
289
260 -def dummy_image_for_minicpmv(hf_config: PretrainedConfig, num_images: int):
290 +def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig,
291 + num_images: int):
292 width = height = hf_config.image_size
262 - image = Image.new("RGB", (width, height), color=0)
263 - return {"image": image if num_images == 1 else [image] * num_images}
293 + image = _build_image_input(ctx,
294 + image=Image.new("RGB", (width, height),
295 + color=0))
296 + return {"image": [image] if num_images == 1 else [image] * num_images}
297
298
299 def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
@@ -269,7 +302,7 @@ def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
302 num_images = mm_counts["image"]
303
304 seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
272 - mm_data = dummy_image_for_minicpmv(hf_config, num_images)
305 + mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images)
306
307 return seq_data, mm_data
308
@@ -280,8 +313,9 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
313 return llm_inputs
314 model_config = ctx.model_config
315 version = get_version_by_config(model_config.hf_config)
283 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
284 - trust_remote_code=True)
316 + tokenizer = cached_get_tokenizer(
317 + model_config.tokenizer,
318 + trust_remote_code=model_config.trust_remote_code)
319 image_processor = cached_get_image_processor(model_config.tokenizer)
320
321 def get_placeholder(image_size: Tuple[int, int], num_image: int):
@@ -317,6 +351,10 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
351 new_prompt = "".join(new_prompt_chunks)
352 new_token_ids = tokenizer.encode(new_prompt)
353
354 + multi_modal_data["image"] = [
355 + _build_image_input(ctx, image) for image in images
356 + ]
357 +
358 llm_inputs = LLMInputs(
359 prompt_token_ids=new_token_ids,
360 prompt=new_prompt,
@@ -325,6 +363,32 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
363 return llm_inputs
364
365
366 +def input_mapper_for_minicpmv(ctx: InputContext, data: object):
367 + model_config = ctx.model_config
368 +
369 + image_processor = cached_get_image_processor(
370 + model_config.model, trust_remote_code=model_config.trust_remote_code)
371 + if image_processor is None:
372 + raise RuntimeError("No HuggingFace processor is available "
373 + "to process the image object")
374 +
375 + if not isinstance(data, list):
376 + raise ValueError(
377 + "Image input must be list of MiniCPMVImageInput, got (%s)", data)
378 + batch_data = image_processor \
379 + .preprocess([img["image"] for img in data], return_tensors="pt") \
380 + .data
381 +
382 + if len(data) > 0:
383 + batch_data["im_start_id"] = data[0]["im_start_id"]
384 + batch_data["im_end_id"] = data[0]["im_end_id"]
385 + if "slice_start_id" in data[0]:
386 + batch_data["slice_start_id"] = data[0]["slice_start_id"]
387 + batch_data["slice_end_id"] = data[0]["slice_end_id"]
388 +
389 + return MultiModalInputs(batch_data)
390 +
391 +
392 class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
393 """
394 The abstract class of MiniCPMV can only be inherited, but cannot be
@@ -365,7 +429,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
429 def get_embedding(
430 self,
431 input_ids: torch.Tensor,
368 - image_inputs: Optional[MiniCPMVImageInputs],
432 + image_inputs: Optional[MiniCPMVImagePixelInputs],
433 ) -> Tuple[torch.Tensor, torch.Tensor]:
434 vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
435 if hasattr(self.config, "scale_emb"):
@@ -393,14 +457,20 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
457
458 return vlm_embedding, vision_hidden_states
459
396 - def _get_image_bounds(self, input_ids: torch.Tensor) -> torch.Tensor:
397 - tokenizer = cached_get_tokenizer(self.config._name_or_path,
398 - trust_remote_code=True)
399 - start_cond = input_ids == tokenizer.im_start_id
400 - end_cond = input_ids == tokenizer.im_end_id
401 - if hasattr(tokenizer, "slice_start_id"):
402 - start_cond |= (input_ids == tokenizer.slice_start_id)
403 - end_cond |= (input_ids == tokenizer.slice_end_id)
460 + def _get_image_bounds(
461 + self,
462 + input_ids: torch.Tensor,
463 + im_start_id: torch.Tensor,
464 + im_end_id: torch.Tensor,
465 + slice_start_id: Optional[torch.Tensor] = None,
466 + slice_end_id: Optional[torch.Tensor] = None) -> torch.Tensor:
467 + # All the images in the batch should share the same special image
468 + # bound token ids.
469 + start_cond = input_ids == im_start_id[0]
470 + end_cond = input_ids == im_end_id[0]
471 + if slice_start_id is not None:
472 + start_cond |= (input_ids == slice_start_id[0])
473 + end_cond |= (input_ids == slice_end_id[0])
474
475 image_start_tokens, = torch.where(start_cond)
476 image_start_tokens += 1
@@ -419,7 +489,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
489 self,
490 input_ids: torch.Tensor,
491 **kwargs: object,
422 - ) -> Optional[MiniCPMVImageInputs]:
492 + ) -> Optional[MiniCPMVImagePixelInputs]:
493 pixel_values = kwargs.pop("pixel_values", [])
494 tgt_sizes = kwargs.pop("tgt_sizes", [])
495
@@ -456,8 +526,17 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
526 if len(pixel_values_flat) == 0:
527 return None
528
459 - return MiniCPMVImageInputs(
460 - image_bounds=self._get_image_bounds(input_ids),
529 + im_start_id = kwargs.pop("im_start_id", None)
530 + im_end_id = kwargs.pop("im_end_id", None)
531 + slice_start_id = kwargs.pop("slice_start_id", None)
532 + slice_end_id = kwargs.pop("slice_end_id", None)
533 + if im_start_id is None:
534 + return None
535 +
536 + return MiniCPMVImagePixelInputs(
537 + image_bounds=self._get_image_bounds(input_ids, im_start_id,
538 + im_end_id, slice_start_id,
539 + slice_end_id),
540 pixel_values=pixel_values_flat,
541 tgt_sizes=torch.stack(tgt_sizes_flat),
542 )
@@ -564,8 +643,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
643 ) -> torch.Tensor:
644 raise NotImplementedError
645
567 - def get_vision_hidden_states(self,
568 - data: MiniCPMVImageInputs) -> torch.Tensor:
646 + def get_vision_hidden_states(
647 + self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
648 raise NotImplementedError
649
650 def is_default_weight_loading(self, name: str) -> bool:
@@ -654,8 +733,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
733 res.append(self.resampler(vision_embedding, tgt_size))
734 return torch.vstack(res)
735
657 - def get_vision_hidden_states(self,
658 - data: MiniCPMVImageInputs) -> torch.Tensor:
736 + def get_vision_hidden_states(
737 + self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
738 pixel_values = data["pixel_values"]
739
740 return self.get_vision_embedding(pixel_values)
@@ -713,8 +792,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel):
792 vision_embedding = self.resampler(vision_embedding, tgt_sizes)
793 return vision_embedding
794
716 - def get_vision_hidden_states(self,
717 - data: MiniCPMVImageInputs) -> torch.Tensor:
795 + def get_vision_hidden_states(
796 + self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
797 pixel_values = data["pixel_values"]
798 tgt_sizes = data["tgt_sizes"]
799
@@ -807,8 +886,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
886 ).last_hidden_state
887 return vision_embedding
888
810 - def get_vision_hidden_states(self,
811 - data: MiniCPMVImageInputs) -> torch.Tensor:
889 + def get_vision_hidden_states(
890 + self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
891 pixel_values = data["pixel_values"]
892 tgt_sizes = data["tgt_sizes"]
893
@@ -851,7 +930,7 @@ _SUPPORT_VERSION = {
930 }
931
932
854 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
933 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_minicpmv)
934 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
935 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
936 @INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
vllm/model_executor/models/qwen.py
+9 -6
@@ -674,8 +674,9 @@ def input_processor_for_qwen(ctx: InputContext,
674 prompt = llm_inputs.get("prompt")
675 prompt_token_ids = llm_inputs["prompt_token_ids"]
676 model_config = ctx.model_config
677 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
678 - trust_remote_code=True)
677 + tokenizer = cached_get_tokenizer(
678 + model_config.tokenizer,
679 + trust_remote_code=model_config.trust_remote_code)
680 image_data = multi_modal_data["image"]
681 if isinstance(image_data, torch.Tensor):
682 num_dims = len(image_data.shape)
@@ -735,8 +736,9 @@ def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
736 return MultiModalInputs()
737
738 model_config = ctx.model_config
738 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
739 - trust_remote_code=True)
739 + tokenizer = cached_get_tokenizer(
740 + model_config.tokenizer,
741 + trust_remote_code=model_config.trust_remote_code)
742
743 image_pair_tok = tokenizer.encode(IMG_START + IMG_END,
744 add_special_tokens=False,
@@ -824,8 +826,9 @@ def dummy_data_for_qwen(
826 # We have a visual component - use images to warm up
827 num_images = mm_counts["image"]
828 model_config = ctx.model_config
827 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
828 - trust_remote_code=True)
829 + tokenizer = cached_get_tokenizer(
830 + model_config.tokenizer,
831 + trust_remote_code=model_config.trust_remote_code)
832
833 # Build the image prompts with no imgpads; the tokenizer will add img pads
834 image_prompt = ''.join(