A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 20:45 UTC
[ROCm][CI] Fix AITER test flakiness by using explicit attention backend (#32346)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
Andreas Karatzas committed
Jan 21, 2026 at 23:55 UTC
eb1629da2496fe351251abe1905f88021fbc3509
3 files changed
+10
-6
.buildkite/test-amd.yaml
+1
-1
@@ -866,7 +866,7 @@ steps:
866
867
- label: Language Models Tests (Standard)
868
timeout_in_minutes: 25
869
- mirror_hardwares: [amdexperimental]
869
+ mirror_hardwares: [amdexperimental, amdproduction]
870
agent_pool: mi325_1
871
# grade: Blocking
872
torch_nightly: true
tests/models/language/generation/test_common.py
+5
-1
@@ -160,8 +160,12 @@ def test_models(
160
tokenizer_name=model_info.tokenizer or model,
161
tokenizer_mode=model_info.tokenizer_mode,
162
trust_remote_code=model_info.trust_remote_code,
163
- max_num_seqs=2,
163
+ # Remove the effects of batch variance on ROCm since batch invariance
164
+ # is not yet supported.
165
+ # See: https://github.com/vllm-project/vllm/issues/27433
166
+ max_num_seqs=1 if current_platform.is_rocm() else 2,
167
enable_prompt_embeds=use_prompt_embeds,
168
+ compilation_config={"cudagraph_capture_sizes": [1, 2]},
169
) as vllm_model:
170
vllm_outputs = vllm_model.generate_greedy_logprobs(
171
example_prompts, max_tokens, num_logprobs
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI325X.json
+4
-4
@@ -45,14 +45,14 @@
45
},
46
"16": {
47
"BLOCK_SIZE_M": 16,
48
- "BLOCK_SIZE_N": 16,
49
- "BLOCK_SIZE_K": 256,
48
+ "BLOCK_SIZE_N": 64,
49
+ "BLOCK_SIZE_K": 128,
50
"GROUP_SIZE_M": 1,
51
- "num_warps": 4,
51
+ "num_warps": 2,
52
"num_stages": 2,
53
"waves_per_eu": 0,
54
"matrix_instr_nonkdim": 16,
55
- "kpack": 2
55
+ "kpack": 1
56
},
57
"24": {
58
"BLOCK_SIZE_M": 16,