A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[VLM][Bugfix] internvl with num_scheduler_steps > 1 (#8614)
DefTruth committed
Sep 26, 2024 at 00:35 UTC
0c4d2ad5e641de145682674066a84ffc632e714e
1 file changed
+6
-1
vllm/model_executor/models/internvl.py
+6
-1
@@ -19,7 +19,7 @@ from vllm.config import CacheConfig, MultiModalConfig
19
from vllm.distributed import get_pp_group
20
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
-from vllm.model_executor.layers.sampler import SamplerOutput
22
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.models.intern_vit import InternVisionModel
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -379,6 +379,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
379
self.make_empty_intermediate_tensors = (
380
self.language_model.make_empty_intermediate_tensors)
381
382
+ if hasattr(self.language_model, "sampler"):
383
+ self.sampler = self.language_model.sampler
384
+ else:
385
+ self.sampler = Sampler()
386
+
387
def pixel_shuffle(self, x, scale_factor=0.5):
388
n, w, h, c = x.size()
389
# N, W, H, C --> N, W, H * scale, C // scale