A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[ROCm] [AITER] [Bugfix] Patch for AITER commit `648764942e552a8bb5fe16026703716a81f05374` (#18990)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
TJian committed
Jun 18, 2025 at 22:56 UTC
2de12be428a71bbdd83cc2cb79f328285aa6224f
2 files changed
+4
-3
docker/Dockerfile.rocm_base
+1
-1
@@ -12,7 +12,7 @@ ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git"
12
ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
13
ARG FA_BRANCH="1a7f4dfa"
14
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
15
-ARG AITER_BRANCH="c1debd8"
15
+ARG AITER_BRANCH="6487649"
16
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
17
18
FROM ${BASE_IMAGE} AS base
vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py
+3
-2
@@ -22,8 +22,9 @@ class QuantMethod(IntEnum):
22
NO = 0 # a16w16
23
PER_TENSOR = 1 # w8a8 (pre_Tensor)
24
PER_TOKEN = 2 # w8a8/w8a4 (per_Token)
25
- BLOCK_1X128 = 3 # block quantized w8a8 (per_1x128)
26
- BLOCK_128x128 = 4 # block quantized w8a8 (per_128x128)
25
+ BLOCK_1X32 = 3 # fp4x2
26
+ BLOCK_1X128 = 4 # block quantized w8a8 (per_1x128)
27
+ BLOCK_128x128 = 5 # block quantized w8a8 (per_128x128)
28
29
30
class ActivationMethod(IntEnum):