A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 20:15 UTC
Make voxtral compile friendly (#33959)
Signed-off-by: Tugsbayasgalan Manlaibaatar <tmanlaibaatar@fb.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
Tugsbayasgalan Manlaibaatar committed
Feb 24, 2026 at 16:33 UTC
f1c664545b954e30ac4887e32ce8f73b39310a9a
1 file changed
+16
-3
vllm/model_executor/models/voxtral_realtime.py
+16
-3
@@ -41,6 +41,7 @@ from vllm.multimodal.processing.processor import (
41
)
42
from vllm.sequence import IntermediateTensors
43
from vllm.tokenizers import cached_tokenizer_from_config
44
+from vllm.utils.torch_utils import is_torch_equal_or_newer
45
46
from .utils import (
47
_flatten_embeddings,
@@ -337,9 +338,21 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
338
assert input_ids is not None
339
340
pool_size = self.config.audio_config.block_pool_size
340
- inputs_embeds = inputs_embeds.view(
341
- inputs_embeds.shape[0] * pool_size, inputs_embeds.shape[1] // pool_size
342
- )
341
+ if is_torch_equal_or_newer("2.11"):
342
+ inputs_embeds = inputs_embeds.view(
343
+ inputs_embeds.shape[0] * pool_size, inputs_embeds.shape[1] // pool_size
344
+ )
345
+ else:
346
+ # TODO Use reshape + clone to break the view chain and avoid output
347
+ # aliasing input bug in torch.compile's AOT autograd cache.
348
+ # Without clone(), if any downstream operation returns a view that's
349
+ # connected to this view of inputs_embeds, the AOT autograd cache
350
+ # fails to pickle the ViewMetaSequence containing SymInt shapes.
351
+ # This will be fixed in pytorch 2.11 and beyond.
352
+ # issue: https://github.com/pytorch/pytorch/issues/174299
353
+ inputs_embeds = inputs_embeds.reshape(
354
+ inputs_embeds.shape[0] * pool_size, inputs_embeds.shape[1] // pool_size
355
+ ).clone()
356
357
whisper_positions = _expand_tensor(positions, pool_size)
358
audio_hidden_states = self.whisper_encoder.whisper_encoder(