@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3132aac043
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix Idefics3 bug (#10778)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 29, 2024 at 21:56 UTC 3132aac04326286ae996bf0887e920096b2bb210
1 file changed +47 -45
vllm/model_executor/models/idefics3.py
+47 -45
@@ -267,54 +267,56 @@ def input_processor_for_idefics3(ctx: InputContext,
267 n_images_in_text = []
268
269 text = inputs.get("prompt")
270 - if text is not None:
271 - if isinstance(text, str):
272 - text = [text]
273 - elif not isinstance(text, list) and not isinstance(text[0], str):
274 - raise ValueError("Invalid input text. Please provide a string, "
275 - "or a list of strings")
276 -
277 - fake_image_token = processor.fake_image_token.content
278 - image_token = processor.image_token.content
279 - global_img_token = processor.global_image_tag
280 -
281 - prompt_strings = []
282 - for sample, sample_rows, sample_cols in zip(text, image_rows,
283 - image_cols):
284 - n_images_in_text.append(sample.count(image_token))
285 -
286 - # Replace the image token with fake tokens around the expanded
287 - # image token sequence of length `image_seq_len`
288 - image_prompt_strings = []
289 - for n_rows, n_cols in zip(sample_rows, sample_cols):
290 - image_prompt_string = _get_image_prompt_string(
291 - n_rows,
292 - n_cols,
293 - processor.image_seq_len,
294 - image_token=image_token,
295 - fake_token_around_image=fake_image_token,
296 - global_img_token=global_img_token,
297 - )
298 - image_prompt_strings.append(image_prompt_string)
299 -
300 - split_sample = sample.split(image_token)
301 - if len(split_sample) == 0:
302 - raise ValueError(
303 - "The image token should be present in the text.")
270 + if text is None:
271 + prompt_token_ids = inputs.get("prompt_token_ids", [])
272 + assert prompt_token_ids
273 + text = tokenizer.decode(prompt_token_ids)
274 +
275 + if isinstance(text, str):
276 + text = [text]
277 + elif not isinstance(text, list) and not isinstance(text[0], str):
278 + raise ValueError("Invalid input text. Please provide a string, "
279 + "or a list of strings")
280 +
281 + fake_image_token = processor.fake_image_token.content
282 + image_token = processor.image_token.content
283 + global_img_token = processor.global_image_tag
284 +
285 + prompt_strings = []
286 + for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
287 + n_images_in_text.append(sample.count(image_token))
288 +
289 + # Replace the image token with fake tokens around the expanded
290 + # image token sequence of length `image_seq_len`
291 + image_prompt_strings = []
292 + for n_rows, n_cols in zip(sample_rows, sample_cols):
293 + image_prompt_string = _get_image_prompt_string(
294 + n_rows,
295 + n_cols,
296 + processor.image_seq_len,
297 + image_token=image_token,
298 + fake_token_around_image=fake_image_token,
299 + global_img_token=global_img_token,
300 + )
301 + image_prompt_strings.append(image_prompt_string)
302
305 - # Place in the image prompt strings where the image tokens are
306 - sample = split_sample[0]
307 - for i, image_prompt_string in enumerate(image_prompt_strings):
308 - sample += image_prompt_string + split_sample[i + 1]
309 - prompt_strings.append(sample)
303 + split_sample = sample.split(image_token)
304 + if len(split_sample) == 0:
305 + raise ValueError("The image token should be present in the text.")
306
311 - prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
307 + # Place in the image prompt strings where the image tokens are
308 + sample = split_sample[0]
309 + for i, image_prompt_string in enumerate(image_prompt_strings):
310 + sample += image_prompt_string + split_sample[i + 1]
311 + prompt_strings.append(sample)
312
313 - return token_inputs(
314 - prompt_token_ids=prompt_token_ids,
315 - prompt=prompt_strings[0],
316 - multi_modal_data=multi_modal_data,
317 - )
313 + prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
314 +
315 + return token_inputs(
316 + prompt_token_ids=prompt_token_ids,
317 + prompt=prompt_strings[0],
318 + multi_modal_data=multi_modal_data,
319 + )
320
321
322 def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int: