@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c4d2ad5e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[VLM][Bugfix] internvl with num_scheduler_steps > 1 (#8614)

DefTruth committed Sep 26, 2024 at 00:35 UTC 0c4d2ad5e641de145682674066a84ffc632e714e
1 file changed +6 -1
vllm/model_executor/models/internvl.py
+6 -1
@@ -19,7 +19,7 @@ from vllm.config import CacheConfig, MultiModalConfig
19 from vllm.distributed import get_pp_group
20 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 -from vllm.model_executor.layers.sampler import SamplerOutput
22 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.models.intern_vit import InternVisionModel
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -379,6 +379,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
379 self.make_empty_intermediate_tensors = (
380 self.language_model.make_empty_intermediate_tensors)
381
382 + if hasattr(self.language_model, "sampler"):
383 + self.sampler = self.language_model.sampler
384 + else:
385 + self.sampler = Sampler()
386 +
387 def pixel_shuffle(self, x, scale_factor=0.5):
388 n, w, h, c = x.size()
389 # N, W, H, C --> N, W, H * scale, C // scale