A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Properly propagate trust_remote_code settings (#10047)
Signed-off-by: Zifei Tong <zifeitong@gmail.com>
zifeitong committed
Nov 5, 2024 at 16:34 UTC
43300bd98a54d48e97d9fb78c9db88eda3a88c64
2 files changed
+16
-13
vllm/model_executor/models/chatglm.py
+4
-3
@@ -54,8 +54,9 @@ def mm_input_mapper_for_glmv(
54
data: MultiModalData[object],
55
) -> Dict:
56
model_config = ctx.model_config
57
- tokenizer = cached_get_tokenizer(model_config.tokenizer,
58
- trust_remote_code=True)
57
+ tokenizer = cached_get_tokenizer(
58
+ model_config.tokenizer,
59
+ trust_remote_code=model_config.trust_remote_code)
60
if tokenizer is None:
61
raise RuntimeError("No HuggingFace processor is available "
62
"to process the image object")
@@ -525,7 +526,7 @@ class ChatGLMModel(nn.Module):
526
elif isinstance(pixel_values, list):
527
return torch.concat(pixel_values)
528
else:
528
- raise TypeError("""pixel_values must be a torch.Tensor
529
+ raise TypeError("""pixel_values must be a torch.Tensor
530
or a list of torch.Tensor
531
""")
532
return GLMImagePixelInputs(pixel_values=pixel_values)
vllm/model_executor/models/molmo.py
+12
-10
@@ -844,9 +844,10 @@ def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int,
844
845
846
def get_max_molmo_image_tokens(ctx: InputContext) -> int:
847
- processor = cached_get_processor(ctx.model_config.model,
848
- trust_remote_code=True,
849
- revision=ctx.model_config.code_revision)
847
+ processor = cached_get_processor(
848
+ ctx.model_config.model,
849
+ trust_remote_code=ctx.model_config.trust_remote_code,
850
+ revision=ctx.model_config.code_revision)
851
image_processor = processor.image_processor
852
max_llm_image_tokens = get_max_tokens(
853
image_processor.max_crops,
@@ -870,9 +871,10 @@ def image_input_mapper_for_molmo(
871
872
def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
873
mm_counts: Mapping[str, int]):
873
- processor = cached_get_processor(ctx.model_config.model,
874
- trust_remote_code=True,
875
- revision=ctx.model_config.code_revision)
874
+ processor = cached_get_processor(
875
+ ctx.model_config.model,
876
+ trust_remote_code=ctx.model_config.trust_remote_code,
877
+ revision=ctx.model_config.code_revision)
878
image_processor = processor.image_processor
879
880
base_image_input_d = image_processor.image_patch_size
@@ -935,11 +937,11 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
937
multi_modal_data = inputs.get("multi_modal_data")
938
image = None if multi_modal_data is None else multi_modal_data.get("image")
939
938
- processor = cached_get_processor(ctx.model_config.model,
939
- trust_remote_code=True,
940
- revision=ctx.model_config.code_revision)
941
-
940
model_config = ctx.model_config
941
+ processor = cached_get_processor(
942
+ ctx.model_config.model,
943
+ trust_remote_code=model_config.trust_remote_code,
944
+ revision=ctx.model_config.code_revision)
945
tokenizer = cached_get_tokenizer(
946
model_config.tokenizer,
947
trust_remote_code=model_config.trust_remote_code)