@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 43300bd98a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Properly propagate trust_remote_code settings (#10047)

Signed-off-by: Zifei Tong <zifeitong@gmail.com>

zifeitong committed Nov 5, 2024 at 16:34 UTC 43300bd98a54d48e97d9fb78c9db88eda3a88c64
2 files changed +16 -13
vllm/model_executor/models/chatglm.py
+4 -3
@@ -54,8 +54,9 @@ def mm_input_mapper_for_glmv(
54 data: MultiModalData[object],
55 ) -> Dict:
56 model_config = ctx.model_config
57 - tokenizer = cached_get_tokenizer(model_config.tokenizer,
58 - trust_remote_code=True)
57 + tokenizer = cached_get_tokenizer(
58 + model_config.tokenizer,
59 + trust_remote_code=model_config.trust_remote_code)
60 if tokenizer is None:
61 raise RuntimeError("No HuggingFace processor is available "
62 "to process the image object")
@@ -525,7 +526,7 @@ class ChatGLMModel(nn.Module):
526 elif isinstance(pixel_values, list):
527 return torch.concat(pixel_values)
528 else:
528 - raise TypeError("""pixel_values must be a torch.Tensor
529 + raise TypeError("""pixel_values must be a torch.Tensor
530 or a list of torch.Tensor
531 """)
532 return GLMImagePixelInputs(pixel_values=pixel_values)
vllm/model_executor/models/molmo.py
+12 -10
@@ -844,9 +844,10 @@ def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int,
844
845
846 def get_max_molmo_image_tokens(ctx: InputContext) -> int:
847 - processor = cached_get_processor(ctx.model_config.model,
848 - trust_remote_code=True,
849 - revision=ctx.model_config.code_revision)
847 + processor = cached_get_processor(
848 + ctx.model_config.model,
849 + trust_remote_code=ctx.model_config.trust_remote_code,
850 + revision=ctx.model_config.code_revision)
851 image_processor = processor.image_processor
852 max_llm_image_tokens = get_max_tokens(
853 image_processor.max_crops,
@@ -870,9 +871,10 @@ def image_input_mapper_for_molmo(
871
872 def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
873 mm_counts: Mapping[str, int]):
873 - processor = cached_get_processor(ctx.model_config.model,
874 - trust_remote_code=True,
875 - revision=ctx.model_config.code_revision)
874 + processor = cached_get_processor(
875 + ctx.model_config.model,
876 + trust_remote_code=ctx.model_config.trust_remote_code,
877 + revision=ctx.model_config.code_revision)
878 image_processor = processor.image_processor
879
880 base_image_input_d = image_processor.image_patch_size
@@ -935,11 +937,11 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
937 multi_modal_data = inputs.get("multi_modal_data")
938 image = None if multi_modal_data is None else multi_modal_data.get("image")
939
938 - processor = cached_get_processor(ctx.model_config.model,
939 - trust_remote_code=True,
940 - revision=ctx.model_config.code_revision)
941 -
940 model_config = ctx.model_config
941 + processor = cached_get_processor(
942 + ctx.model_config.model,
943 + trust_remote_code=model_config.trust_remote_code,
944 + revision=ctx.model_config.code_revision)
945 tokenizer = cached_get_tokenizer(
946 model_config.tokenizer,
947 trust_remote_code=model_config.trust_remote_code)