A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix Idefics3 bug (#10778)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 29, 2024 at 21:56 UTC
3132aac04326286ae996bf0887e920096b2bb210
1 file changed
+47
-45
vllm/model_executor/models/idefics3.py
+47
-45
@@ -267,54 +267,56 @@ def input_processor_for_idefics3(ctx: InputContext,
267
n_images_in_text = []
268
269
text = inputs.get("prompt")
270
- if text is not None:
271
- if isinstance(text, str):
272
- text = [text]
273
- elif not isinstance(text, list) and not isinstance(text[0], str):
274
- raise ValueError("Invalid input text. Please provide a string, "
275
- "or a list of strings")
276
-
277
- fake_image_token = processor.fake_image_token.content
278
- image_token = processor.image_token.content
279
- global_img_token = processor.global_image_tag
280
-
281
- prompt_strings = []
282
- for sample, sample_rows, sample_cols in zip(text, image_rows,
283
- image_cols):
284
- n_images_in_text.append(sample.count(image_token))
285
-
286
- # Replace the image token with fake tokens around the expanded
287
- # image token sequence of length `image_seq_len`
288
- image_prompt_strings = []
289
- for n_rows, n_cols in zip(sample_rows, sample_cols):
290
- image_prompt_string = _get_image_prompt_string(
291
- n_rows,
292
- n_cols,
293
- processor.image_seq_len,
294
- image_token=image_token,
295
- fake_token_around_image=fake_image_token,
296
- global_img_token=global_img_token,
297
- )
298
- image_prompt_strings.append(image_prompt_string)
299
-
300
- split_sample = sample.split(image_token)
301
- if len(split_sample) == 0:
302
- raise ValueError(
303
- "The image token should be present in the text.")
270
+ if text is None:
271
+ prompt_token_ids = inputs.get("prompt_token_ids", [])
272
+ assert prompt_token_ids
273
+ text = tokenizer.decode(prompt_token_ids)
274
+
275
+ if isinstance(text, str):
276
+ text = [text]
277
+ elif not isinstance(text, list) and not isinstance(text[0], str):
278
+ raise ValueError("Invalid input text. Please provide a string, "
279
+ "or a list of strings")
280
+
281
+ fake_image_token = processor.fake_image_token.content
282
+ image_token = processor.image_token.content
283
+ global_img_token = processor.global_image_tag
284
+
285
+ prompt_strings = []
286
+ for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
287
+ n_images_in_text.append(sample.count(image_token))
288
+
289
+ # Replace the image token with fake tokens around the expanded
290
+ # image token sequence of length `image_seq_len`
291
+ image_prompt_strings = []
292
+ for n_rows, n_cols in zip(sample_rows, sample_cols):
293
+ image_prompt_string = _get_image_prompt_string(
294
+ n_rows,
295
+ n_cols,
296
+ processor.image_seq_len,
297
+ image_token=image_token,
298
+ fake_token_around_image=fake_image_token,
299
+ global_img_token=global_img_token,
300
+ )
301
+ image_prompt_strings.append(image_prompt_string)
302
305
- # Place in the image prompt strings where the image tokens are
306
- sample = split_sample[0]
307
- for i, image_prompt_string in enumerate(image_prompt_strings):
308
- sample += image_prompt_string + split_sample[i + 1]
309
- prompt_strings.append(sample)
303
+ split_sample = sample.split(image_token)
304
+ if len(split_sample) == 0:
305
+ raise ValueError("The image token should be present in the text.")
306
311
- prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
307
+ # Place in the image prompt strings where the image tokens are
308
+ sample = split_sample[0]
309
+ for i, image_prompt_string in enumerate(image_prompt_strings):
310
+ sample += image_prompt_string + split_sample[i + 1]
311
+ prompt_strings.append(sample)
312
313
- return token_inputs(
314
- prompt_token_ids=prompt_token_ids,
315
- prompt=prompt_strings[0],
316
- multi_modal_data=multi_modal_data,
317
- )
313
+ prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
314
+
315
+ return token_inputs(
316
+ prompt_token_ids=prompt_token_ids,
317
+ prompt=prompt_strings[0],
318
+ multi_modal_data=multi_modal_data,
319
+ )
320
321
322
def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int: