A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix][AMD][Deepseek] fix a dtype mismatch error for deepseek running on AMD (#23864)
Signed-off-by: Jinghui Zhang <jinghuizhang0804@gmail.com>
Jinghui Zhang committed
Aug 28, 2025 at 22:54 UTC
5264015d74f2e0213a1e7d51041a558d7ea580e8
1 file changed
+4
-4
vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py
+4
-4
@@ -279,7 +279,7 @@ def rocm_aiter_grouped_topk(
279
if e_score_correction_bias is not None:
280
torch.ops.vllm.rocm_aiter_biased_grouped_topk(
281
gating_output,
282
- e_score_correction_bias,
282
+ e_score_correction_bias.to(gating_output.dtype),
283
topk_weights,
284
topk_ids,
285
num_expert_group,
@@ -409,15 +409,15 @@ def shuffle_weights(
409
*tensors: torch.Tensor, layout: tuple[int, int] = (16, 16)
410
) -> tuple[torch.Tensor, ...]:
411
"""
412
- Applies shuffle_weight function from AITER to each
412
+ Applies shuffle_weight function from AITER to each
413
input tensor and returns them.
414
-
414
+
415
Rearranges (shuffles) the input tensor/s
416
into a specified block layout for optimized computation.
417
418
Args:
419
*tensors: Variable number of torch.Tensor objects.
420
- layout: A pair of integers specifying the
420
+ layout: A pair of integers specifying the
421
block sizes used to divide the tensors during shuffling.
422
Default is (16, 16).
423