A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 06:30 UTC
Fix phi4-mm and remove cuda binding (#35964)
Signed-off-by: Yan Ma <yan.ma@intel.com>
Yan Ma committed
Mar 5, 2026 at 01:08 UTC
58cfe0dc44b29ced86cf8a6db069e55faf5d4f7d
2 files changed
+22
-19
vllm/model_executor/models/phi4mm_audio.py
+5
-6
@@ -585,10 +585,9 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
585
enc_streaming_mask = self._streaming_mask(
586
seq_len, batch_size, self.chunk_size, self.left_chunk
587
)
588
-
589
- if xs_pad.is_cuda:
590
- enc_streaming_mask = enc_streaming_mask.cuda()
591
- xs_pad = xs_pad.cuda()
588
+ device = xs_pad.device
589
+ enc_streaming_mask = enc_streaming_mask.to(device)
590
+ xs_pad = xs_pad.to(device)
591
592
input_tensor = xs_pad
593
input_tensor, masks = self._forward_embeddings_core(input_tensor, masks)
@@ -605,8 +604,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
604
enc_streaming_mask_nc = self._streaming_mask(
605
seq_len, batch_size, chunk_size_nc, left_chunk_nc
606
)
608
- if xs_pad.is_cuda:
609
- enc_streaming_mask_nc = enc_streaming_mask_nc.cuda()
607
+ if device.type != "cpu":
608
+ enc_streaming_mask_nc = enc_streaming_mask_nc.to(device)
609
if masks is not None:
610
hs_mask_nc = masks & enc_streaming_mask_nc
611
else:
vllm/model_executor/models/phi4mm_utils.py
+17
-13
@@ -1309,16 +1309,15 @@ class NemoConvSubsampling(torch.nn.Module):
1309
raise ValueError(f"Not valid sub-sampling: {subsampling}!")
1310
1311
if subsampling in ["dw_striding", "striding"]:
1312
- in_length = torch.tensor(feat_in, dtype=torch.float)
1313
- out_length = calc_length(
1314
- lengths=in_length,
1312
+ out_length = calc_length_int(
1313
+ lengths=feat_in,
1314
all_paddings=self._left_padding + self._right_padding,
1315
kernel_size=self._kernel_size,
1316
stride=self._stride,
1317
ceil_mode=self._ceil_mode,
1318
repeat_num=self._sampling_num,
1319
)
1321
- self.out = torch.nn.Linear(conv_channels * int(out_length), feat_out)
1320
+ self.out = torch.nn.Linear(conv_channels * out_length, feat_out)
1321
self.conv2d_subsampling = True
1322
elif subsampling in ["striding_conv1d", "dw_striding_conv1d"]:
1323
self.out = None
@@ -1543,22 +1542,27 @@ class NemoConvSubsampling(torch.nn.Module):
1542
self.subsampling_conv_chunking_factor = subsampling_conv_chunking_factor
1543
1544
1546
-def calc_length(
1547
- lengths: Tensor,
1545
+def calc_length_int(
1546
+ lengths: int,
1547
all_paddings: int,
1548
kernel_size: int,
1549
stride: int,
1550
ceil_mode: bool,
1551
repeat_num: int = 1,
1553
-) -> Tensor:
1554
- """Calculates the output length of a Tensor passed through a convolution or
1555
- max pooling layer"""
1552
+) -> int:
1553
+ """Integer-only variant of calc_length for meta-safe shape computation.
1554
+
1555
+ Computes the output length of a 1D convolution / pooling stack using
1556
+ the same formula as calc_length, but operates purely on Python numbers
1557
+ so it can be safely used during meta tensor initialization.
1558
+ """
1559
add_pad: float = all_paddings - kernel_size
1560
one: float = 1.0
1558
- for i in range(repeat_num):
1559
- lengths = torch.div(lengths.to(dtype=torch.float) + add_pad, stride) + one
1560
- lengths = torch.ceil(lengths) if ceil_mode else torch.floor(lengths)
1561
- return lengths.to(dtype=torch.int)
1561
+ length_f: float = float(lengths)
1562
+ for _ in range(repeat_num):
1563
+ length_f = (length_f + add_pad) / stride + one
1564
+ length_f = math.ceil(length_f) if ceil_mode else math.floor(length_f)
1565
+ return int(length_f)
1566
1567
1568
#### multihead attention starts here