A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc/Testing] Use `torch.testing.assert_close` (#7324)
jon-chuang committed
Aug 15, 2024 at 21:24 UTC
50b8d08dbd4493327e344bc627a0613947deba8f
25 files changed
+197
-188
tests/distributed/test_comm_ops.py
+9
-9
@@ -34,7 +34,7 @@ def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
34
expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
35
t = all_tensors[rank % tp_size]
36
t = tensor_model_parallel_all_reduce(t)
37
- assert torch.allclose(t, expected)
37
+ torch.testing.assert_close(t, expected)
38
39
40
@ray.remote(num_gpus=1, max_calls=1)
@@ -62,7 +62,7 @@ def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
62
expected = torch.cat(all_tensors, dim=all_gather_dimension)
63
t = all_tensors[rank % tp_size]
64
t = tensor_model_parallel_all_gather(t, all_gather_dimension)
65
- assert torch.allclose(t, expected)
65
+ torch.testing.assert_close(t, expected)
66
67
68
@ray.remote(num_gpus=1, max_calls=1)
@@ -96,12 +96,12 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
96
else:
97
recv_dict = broadcast_tensor_dict(src=0)
98
assert len(recv_dict) == len(test_dict)
99
- assert torch.allclose(recv_dict["a"], test_dict["a"])
100
- assert torch.allclose(recv_dict["b"], test_dict["b"])
99
+ torch.testing.assert_close(recv_dict["a"], test_dict["a"])
100
+ torch.testing.assert_close(recv_dict["b"], test_dict["b"])
101
assert recv_dict["c"] == test_dict["c"]
102
assert recv_dict["d"] == test_dict["d"]
103
assert recv_dict["e"] == test_dict["e"]
104
- assert torch.allclose(recv_dict["f"], test_dict["f"])
104
+ torch.testing.assert_close(recv_dict["f"], test_dict["f"])
105
106
107
@ray.remote(num_gpus=1, max_calls=1)
@@ -136,12 +136,12 @@ def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
136
137
if not get_pp_group().is_first_rank:
138
assert len(recv_dict) == len(test_dict)
139
- assert torch.allclose(recv_dict["a"], test_dict["a"])
140
- assert torch.allclose(recv_dict["b"], test_dict["b"])
139
+ torch.testing.assert_close(recv_dict["a"], test_dict["a"])
140
+ torch.testing.assert_close(recv_dict["b"], test_dict["b"])
141
assert recv_dict["c"] == test_dict["c"]
142
assert recv_dict["d"] == test_dict["d"]
143
assert recv_dict["e"] == test_dict["e"]
144
- assert torch.allclose(recv_dict["f"], test_dict["f"])
144
+ torch.testing.assert_close(recv_dict["f"], test_dict["f"])
145
146
147
@ray.remote(num_gpus=1, max_calls=1)
@@ -163,7 +163,7 @@ def send_recv_test_worker(tp_size: int, pp_size: int, rank: int,
163
get_pp_group().send(test_tensor)
164
165
if not get_pp_group().is_first_rank:
166
- assert torch.allclose(test_tensor, recv_tensor)
166
+ torch.testing.assert_close(test_tensor, recv_tensor)
167
168
169
@pytest.mark.skipif(torch.cuda.device_count() < 2,
tests/distributed/test_custom_all_reduce.py
+4
-4
@@ -72,8 +72,8 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
72
out2 = tensor_model_parallel_all_reduce(inp2)
73
dist.all_reduce(inp2, group=group)
74
graph.replay()
75
- assert torch.allclose(out1, inp1)
76
- assert torch.allclose(out2, inp2)
75
+ torch.testing.assert_close(out1, inp1)
76
+ torch.testing.assert_close(out2, inp2)
77
78
79
@ray.remote(num_gpus=1, max_calls=1)
@@ -96,13 +96,13 @@ def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
96
out = inp
97
for _ in range(num_communication):
98
out = fa.all_reduce_unreg(out)
99
- assert torch.allclose(out, inp * (tp_size**num_communication))
99
+ torch.testing.assert_close(out, inp * (tp_size**num_communication))
100
101
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
102
out = inp
103
for _ in range(num_communication):
104
out = fa.all_reduce_unreg(out)
105
- assert torch.allclose(out, inp * (tp_size**num_communication))
105
+ torch.testing.assert_close(out, inp * (tp_size**num_communication))
106
107
108
@pytest.mark.parametrize("tp_size", [2])
tests/kernels/quant_utils.py
+1
-1
@@ -69,4 +69,4 @@ def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
69
ref_iscale = one / ref_scale
70
ref_out = (as_float32_tensor(x) * ref_iscale).clamp(
71
fp8_traits.min, fp8_traits.max).to(dtype=torch.float8_e4m3fn)
72
- return ref_out, ref_scale
72
+ return ref_out, ref_scale.view((1, ))
tests/kernels/test_activation.py
+5
-5
@@ -47,7 +47,7 @@ def test_act_and_mul(
47
ref_out = layer.forward_native(x)
48
# The SiLU and GELU implementations are equivalent to the native PyTorch
49
# implementations, so we can do exact comparison.
50
- assert torch.allclose(out, ref_out, atol=0.0, rtol=0.0)
50
+ torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
51
52
53
@pytest.mark.parametrize("activation", [FastGELU, NewGELU])
@@ -73,7 +73,7 @@ def test_activation(
73
layer = activation()
74
out = layer(x)
75
ref_out = layer.forward_native(x)
76
- assert torch.allclose(out,
77
- ref_out,
78
- atol=get_default_atol(out),
79
- rtol=get_default_rtol(out))
76
+ torch.testing.assert_close(out,
77
+ ref_out,
78
+ atol=get_default_atol(out),
79
+ rtol=get_default_rtol(out))
tests/kernels/test_attention.py
+2
-2
@@ -276,7 +276,7 @@ def test_paged_attention(
276
atol, rtol = 1e-3, 1e-5
277
if kv_cache_dtype == "fp8":
278
atol, rtol = 1e-2, 1e-5
279
- assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
279
+ torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
280
281
282
def ref_multi_query_kv_attention(
@@ -379,4 +379,4 @@ def test_multi_query_kv_attention(
379
)
380
atol = get_default_atol(output) if is_hip() else 1e-3
381
rtol = get_default_rtol(output) if is_hip() else 1e-5
382
- assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
382
+ torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
tests/kernels/test_blocksparse_attention.py
+2
-2
@@ -327,7 +327,7 @@ def test_paged_attention(
327
atol, rtol = 1e-3, 1e-5
328
if kv_cache_dtype == "fp8":
329
atol, rtol = 1e-2, 1e-5
330
- assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
330
+ torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
331
332
333
def ref_multi_query_kv_attention(
@@ -441,4 +441,4 @@ def test_varlen_blocksparse_attention_prefill(
441
scale,
442
dtype,
443
)
444
- assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2)
444
+ torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
tests/kernels/test_cache.py
+27
-27
@@ -98,10 +98,10 @@ def test_copy_blocks(
98
99
# Compare the results.
100
for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
101
- assert torch.allclose(key_cache, cloned_key_cache)
101
+ torch.testing.assert_close(key_cache, cloned_key_cache)
102
for value_cache, cloned_value_cache in zip(value_caches,
103
cloned_value_caches):
104
- assert torch.allclose(value_cache, cloned_value_cache)
104
+ torch.testing.assert_close(value_cache, cloned_value_cache)
105
106
107
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -184,17 +184,17 @@ def test_reshape_and_cache(
184
cloned_value_cache[block_idx, :, :, block_offset] = value[i]
185
186
if kv_cache_dtype == "fp8":
187
- assert torch.allclose(result_key_cache,
188
- cloned_key_cache,
189
- atol=0.001,
190
- rtol=0.1)
191
- assert torch.allclose(result_value_cache,
192
- cloned_value_cache,
193
- atol=0.001,
194
- rtol=0.1)
187
+ torch.testing.assert_close(result_key_cache,
188
+ cloned_key_cache,
189
+ atol=0.001,
190
+ rtol=0.1)
191
+ torch.testing.assert_close(result_value_cache,
192
+ cloned_value_cache,
193
+ atol=0.001,
194
+ rtol=0.1)
195
else:
196
- assert torch.allclose(key_cache, cloned_key_cache)
197
- assert torch.allclose(value_cache, cloned_value_cache)
196
+ torch.testing.assert_close(key_cache, cloned_key_cache)
197
+ torch.testing.assert_close(value_cache, cloned_value_cache)
198
199
200
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -290,17 +290,17 @@ def test_reshape_and_cache_flash(
290
cloned_value_cache[block_idx, block_offset, :, :] = value[i]
291
292
if kv_cache_dtype == "fp8":
293
- assert torch.allclose(result_key_cache,
294
- cloned_key_cache,
295
- atol=0.001,
296
- rtol=0.1)
297
- assert torch.allclose(result_value_cache,
298
- cloned_value_cache,
299
- atol=0.001,
300
- rtol=0.1)
293
+ torch.testing.assert_close(result_key_cache,
294
+ cloned_key_cache,
295
+ atol=0.001,
296
+ rtol=0.1)
297
+ torch.testing.assert_close(result_value_cache,
298
+ cloned_value_cache,
299
+ atol=0.001,
300
+ rtol=0.1)
301
else:
302
- assert torch.allclose(key_cache, cloned_key_cache)
303
- assert torch.allclose(value_cache, cloned_value_cache)
302
+ torch.testing.assert_close(key_cache, cloned_key_cache)
303
+ torch.testing.assert_close(value_cache, cloned_value_cache)
304
305
306
@pytest.mark.parametrize("direction", COPYING_DIRECTION)
@@ -372,10 +372,10 @@ def test_swap_blocks(
372
block_mapping_tensor)
373
374
for src, dst in block_mapping:
375
- assert torch.allclose(src_key_caches_clone[src].cpu(),
376
- dist_key_caches[0][dst].cpu())
377
- assert torch.allclose(src_value_caches_clone[src].cpu(),
378
- dist_value_caches[0][dst].cpu())
375
+ torch.testing.assert_close(src_key_caches_clone[src].cpu(),
376
+ dist_key_caches[0][dst].cpu())
377
+ torch.testing.assert_close(src_value_caches_clone[src].cpu(),
378
+ dist_value_caches[0][dst].cpu())
379
380
381
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -411,4 +411,4 @@ def test_fp8_e4m3_conversion(
411
converted_cache = torch.empty_like(cache)
412
ops.convert_fp8(converted_cache, cache_fp8)
413
414
- assert torch.allclose(cache, converted_cache, atol=0.001, rtol=0.1)
414
+ torch.testing.assert_close(cache, converted_cache, atol=0.001, rtol=0.1)
tests/kernels/test_cutlass.py
+13
-10
@@ -74,7 +74,7 @@ def cutlass_fp8_gemm_helper(m: int,
74
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
75
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
76
77
- assert torch.allclose(out, baseline, rtol=1e-2, atol=5e-2)
77
+ torch.testing.assert_close(out, baseline, rtol=1e-2, atol=5e-2)
78
79
80
def cutlass_int8_gemm_helper(m: int,
@@ -106,7 +106,7 @@ def cutlass_int8_gemm_helper(m: int,
106
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
107
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
108
109
- assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
109
+ torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
110
111
112
@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
@@ -252,7 +252,7 @@ def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int,
252
azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
253
254
a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32)
255
- assert torch.allclose(a_dq, scale_a * aq_f32 + azp_a)
255
+ torch.testing.assert_close(a_dq, scale_a * aq_f32 + azp_a)
256
257
baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype)
258
@@ -271,8 +271,8 @@ def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int,
271
scale_b,
272
out_dtype=out_dtype,
273
bias=azp_bias[0, :])
274
- assert torch.allclose(out, baseline_dq, rtol=1e-2, atol=1e0)
275
- assert torch.allclose(out, baseline_q, rtol=1e-2, atol=1e0)
274
+ torch.testing.assert_close(out, baseline_dq, rtol=1e-2, atol=1e0)
275
+ torch.testing.assert_close(out, baseline_q, rtol=1e-2, atol=1e0)
276
277
278
@pytest.mark.parametrize("m", [32, 64, 128])
@@ -302,7 +302,10 @@ def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
302
azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
303
304
a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32)
305
- assert torch.allclose(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3)
305
+ torch.testing.assert_close(a_dq,
306
+ scale_a * aq_f32 - azp_a,
307
+ rtol=1e-4,
308
+ atol=1e-3)
309
310
if use_bias:
311
bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5
@@ -335,8 +338,8 @@ def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
338
# float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05%
339
rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3
340
atol = 1e-3
338
- assert torch.allclose(out, baseline_dq, rtol=rtol, atol=atol)
339
- assert torch.allclose(out, baseline_q, rtol=rtol, atol=atol)
341
+ torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol)
342
+ torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol)
343
344
345
# Test working with a subset of A and B
@@ -363,7 +366,7 @@ def test_cutlass_subset():
366
scale_b,
367
out_dtype=torch.bfloat16)
368
366
- assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
369
+ torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
370
371
372
# Test to make sure cuda graphs work
@@ -411,4 +414,4 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
414
415
baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
416
scale_b * b.to(dtype=torch.float32)).to(torch.bfloat16)
414
- assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
417
+ torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
tests/kernels/test_flash_attn.py
+2
-2
@@ -126,7 +126,7 @@ def test_flash_attn_with_paged_kv(
126
scale=scale,
127
soft_cap=soft_cap,
128
)
129
- assert torch.allclose(output, ref_output, atol=2e-2, rtol=1e-2), \
129
+ torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), \
130
f"{torch.max(torch.abs(output - ref_output))}"
131
132
@@ -211,5 +211,5 @@ def test_varlen_with_paged_kv(
211
sliding_window=sliding_window,
212
soft_cap=soft_cap,
213
)
214
- assert torch.allclose(output, ref_output, atol=2e-2, rtol=1e-2), \
214
+ torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), \
215
f"{torch.max(torch.abs(output - ref_output))}"
tests/kernels/test_flashinfer.py
+2
-2
@@ -144,7 +144,7 @@ def test_flashinfer_decode_with_paged_kv(kv_lens: List[int],
144
block_tables=block_tables,
145
scale=scale,
146
soft_cap=soft_cap)
147
- assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
147
+ torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2), \
148
f"{torch.max(torch.abs(output - ref_output))}"
149
150
@@ -244,5 +244,5 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
244
block_tables=block_tables,
245
scale=scale,
246
soft_cap=soft_cap)
247
- assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
247
+ torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2), \
248
f"{torch.max(torch.abs(output - ref_output))}"
tests/kernels/test_fp8_quant.py
+7
-7
@@ -37,9 +37,9 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
37
scale_ub=scale_ub,
38
use_per_token_if_dynamic=True)
39
40
- assert torch.allclose(ref_scales, ops_scales)
41
- assert torch.allclose(ref_out.to(dtype=torch.float32),
42
- ops_out.to(dtype=torch.float32))
40
+ torch.testing.assert_close(ref_scales, ops_scales)
41
+ torch.testing.assert_close(ref_out.to(dtype=torch.float32),
42
+ ops_out.to(dtype=torch.float32))
43
44
45
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -57,9 +57,9 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
57
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
58
ops_out, ops_scale = ops.scaled_fp8_quant(x)
59
60
- assert torch.allclose(ref_scale, ops_scale)
61
- assert torch.allclose(ref_out.to(dtype=torch.float32),
62
- ops_out.to(dtype=torch.float32))
60
+ torch.testing.assert_close(ref_scale, ops_scale)
61
+ torch.testing.assert_close(ref_out.to(dtype=torch.float32),
62
+ ops_out.to(dtype=torch.float32))
63
64
65
# Regression test for a case with large activations where an int32 index cannot
@@ -84,4 +84,4 @@ def test_fp8_quant_large(seed: int) -> None:
84
ref_out = ref_out.to(dtype=dtype)
85
ops_out = ops_out.to(dtype=dtype)
86
87
- assert torch.allclose(ref_out, ops_out)
87
+ torch.testing.assert_close(ref_out, ops_out)
tests/kernels/test_int8_quant.py
+7
-5
@@ -29,9 +29,10 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
29
# kernel
30
ops_out, ops_scales = scaled_int8_quant(x)
31
32
- assert torch.allclose(ops_scales, ref_scales)
33
- assert torch.allclose(ops_out, ref_out,
34
- atol=1) # big atol to account for rounding errors
32
+ torch.testing.assert_close(ops_scales, ref_scales)
33
+ torch.testing.assert_close(
34
+ ops_out, ref_out, atol=1,
35
+ rtol=0.0) # big atol to account for rounding errors
36
37
38
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -54,5 +55,6 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
55
int8_traits.max).to(torch.int8)
56
out2, _ = scaled_int8_quant(x, scale)
57
57
- assert torch.allclose(out1, out2,
58
- atol=1) # big atol to account for rounding errors
58
+ torch.testing.assert_close(
59
+ out1, out2, atol=1,
60
+ rtol=0.0) # big atol to account for rounding errors
tests/kernels/test_layernorm.py
+3
-3
@@ -48,7 +48,7 @@ def test_rms_norm(
48
# numerical errors than other operators because they involve reductions.
49
# Therefore, we use a larger tolerance.
50
if add_residual:
51
- assert torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
52
- assert torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
51
+ torch.testing.assert_close(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
52
+ torch.testing.assert_close(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
53
else:
54
- assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)
54
+ torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
tests/kernels/test_marlin_gemm.py
+2
-2
@@ -122,7 +122,7 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
122
)
123
torch.cuda.synchronize()
124
125
- assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
125
+ torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
126
127
128
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
@@ -174,7 +174,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
174
)
175
torch.cuda.synchronize()
176
177
- assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
177
+ torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
178
179
180
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
tests/kernels/test_moe.py
+5
-5
@@ -50,7 +50,7 @@ def test_fused_moe(
50
score = torch.randn((m, e), device='cuda', dtype=dtype)
51
triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False)
52
torch_output = torch_moe(a, w1, w2, score, topk)
53
- assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
53
+ torch.testing.assert_close(triton_output, torch_output, atol=1e-2, rtol=0)
54
55
56
@pytest.mark.parametrize("dtype",
@@ -95,7 +95,7 @@ def test_mixtral_moe(dtype: torch.dtype):
95
torch.bfloat16: 1e-2,
96
}
97
98
- assert torch.allclose(hf_states.flatten(0, 1),
99
- vllm_states,
100
- rtol=mixtral_moe_tol[dtype],
101
- atol=mixtral_moe_tol[dtype])
98
+ torch.testing.assert_close(hf_states.flatten(0, 1),
99
+ vllm_states,
100
+ rtol=mixtral_moe_tol[dtype],
101
+ atol=mixtral_moe_tol[dtype])
tests/kernels/test_pos_encoding.py
+24
-24
@@ -67,14 +67,14 @@ def test_rotary_embedding(
67
ref_query, ref_key = rope.forward_native(positions, query, key)
68
out_query, out_key = rope.forward(positions, query, key)
69
# Compare the results.
70
- assert torch.allclose(out_query,
71
- ref_query,
72
- atol=get_default_atol(out_query),
73
- rtol=get_default_rtol(out_query))
74
- assert torch.allclose(out_key,
75
- ref_key,
76
- atol=get_default_atol(out_key),
77
- rtol=get_default_rtol(out_key))
70
+ torch.testing.assert_close(out_query,
71
+ ref_query,
72
+ atol=get_default_atol(out_query),
73
+ rtol=get_default_rtol(out_query))
74
+ torch.testing.assert_close(out_key,
75
+ ref_key,
76
+ atol=get_default_atol(out_key),
77
+ rtol=get_default_rtol(out_key))
78
79
80
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -129,14 +129,14 @@ def test_batched_rotary_embedding(
129
dtype=torch.long,
130
device=device))
131
# Compare the results.
132
- assert torch.allclose(out_query,
133
- ref_query,
134
- atol=get_default_atol(out_query),
135
- rtol=get_default_rtol(out_query))
136
- assert torch.allclose(out_key,
137
- ref_key,
138
- atol=get_default_atol(out_key),
139
- rtol=get_default_rtol(out_key))
132
+ torch.testing.assert_close(out_query,
133
+ ref_query,
134
+ atol=get_default_atol(out_query),
135
+ rtol=get_default_rtol(out_query))
136
+ torch.testing.assert_close(out_key,
137
+ ref_key,
138
+ atol=get_default_atol(out_key),
139
+ rtol=get_default_rtol(out_key))
140
141
142
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -200,14 +200,14 @@ def test_batched_rotary_embedding_multi_lora(
200
out_query, out_key = rope.forward(positions, query, key,
201
query_offsets.flatten())
202
# Compare the results.
203
- assert torch.allclose(out_query,
204
- ref_query,
205
- atol=get_default_atol(out_query),
206
- rtol=get_default_rtol(out_query))
207
- assert torch.allclose(out_key,
208
- ref_key,
209
- atol=get_default_atol(out_key),
210
- rtol=get_default_rtol(out_key))
203
+ torch.testing.assert_close(out_query,
204
+ ref_query,
205
+ atol=get_default_atol(out_query),
206
+ rtol=get_default_rtol(out_query))
207
+ torch.testing.assert_close(out_key,
208
+ ref_key,
209
+ atol=get_default_atol(out_key),
210
+ rtol=get_default_rtol(out_key))
211
212
213
@torch.inference_mode()
tests/kernels/test_sampler.py
+4
-4
@@ -100,11 +100,11 @@ def test_sample_decoding_only(random_sampling, max_best_of,
100
if modify_greedy_probs and not request_uses_random_sampling:
101
# If we are modifying greedy probs and the request is greedy,
102
# we want to make sure the probs tensor is modified in place
103
- assert torch.allclose(
103
+ torch.testing.assert_close(
104
probs[i][sampled_tokens[i]],
105
torch.full_like(probs[i][sampled_tokens[i]], 1.0))
106
assert torch.sum(probs[i]) == 1.0
107
- assert torch.allclose(
107
+ torch.testing.assert_close(
108
sampled_modified_probs[i][0],
109
torch.full_like(sampled_modified_probs[i][0], 1.0))
110
elif request_uses_random_sampling:
@@ -117,8 +117,8 @@ def test_sample_decoding_only(random_sampling, max_best_of,
117
# If the request is greedy and we are not modifying greedy probs,
118
# we want to make sure sampled_modified_probs tensor is the same as
119
# the probs tensor.
120
- assert torch.allclose(sampled_modified_probs[i][0],
121
- probs[i][sampled_tokens[i]])
120
+ torch.testing.assert_close(sampled_modified_probs[i],
121
+ probs[i][sampled_tokens[i]])
122
123
if save_logprobs:
124
assert sampled_logprobs.shape == (bs, max_best_of)
tests/kernels/utils.py
+2
-2
@@ -924,5 +924,5 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
924
* output_under_test: actually observed output value
925
'''
926
ideal_output = test_params.packed_qkvo.ideal_output
927
- assert torch.allclose(ideal_output,
928
- output_under_test.view_as(ideal_output))
927
+ torch.testing.assert_close(ideal_output,
928
+ output_under_test.view_as(ideal_output))
tests/lora/test_layers.py
+44
-44
@@ -247,10 +247,10 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
247
expected_result = torch.cat(expected_results)
248
249
rtol, atol = TOLERANCES[lora_result.dtype]
250
- assert torch.allclose(lora_result,
251
- expected_result,
252
- rtol=rtol,
253
- atol=atol)
250
+ torch.testing.assert_close(lora_result,
251
+ expected_result,
252
+ rtol=rtol,
253
+ atol=atol)
254
255
# Check that resetting the lora weights succeeds
256
@@ -274,10 +274,10 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
274
expected_result = embedding(torch.cat(inputs))
275
276
rtol, atol = TOLERANCES[lora_result.dtype]
277
- assert torch.allclose(lora_result,
278
- expected_result,
279
- rtol=rtol,
280
- atol=atol)
277
+ torch.testing.assert_close(lora_result,
278
+ expected_result,
279
+ rtol=rtol,
280
+ atol=atol)
281
282
283
@torch.inference_mode()
@@ -384,10 +384,10 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
384
expected_result = torch.cat(expected_results)
385
386
rtol, atol = TOLERANCES[lora_result.dtype]
387
- assert torch.allclose(lora_result,
388
- expected_result,
389
- rtol=rtol,
390
- atol=atol)
387
+ torch.testing.assert_close(lora_result,
388
+ expected_result,
389
+ rtol=rtol,
390
+ atol=atol)
391
392
# Check that resetting the lora weights succeeds
393
@@ -411,10 +411,10 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
411
expected_result = expanded_embedding(torch.cat(inputs))
412
413
rtol, atol = TOLERANCES[lora_result.dtype]
414
- assert torch.allclose(lora_result,
415
- expected_result,
416
- rtol=rtol,
417
- atol=atol)
414
+ torch.testing.assert_close(lora_result,
415
+ expected_result,
416
+ rtol=rtol,
417
+ atol=atol)
418
419
420
@torch.inference_mode()
@@ -541,10 +541,10 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
541
embedding_bias=None)
542
543
rtol, atol = TOLERANCES[lora_result.dtype]
544
- assert torch.allclose(lora_result,
545
- expected_result,
546
- rtol=rtol,
547
- atol=atol)
544
+ torch.testing.assert_close(lora_result,
545
+ expected_result,
546
+ rtol=rtol,
547
+ atol=atol)
548
549
550
@torch.inference_mode()
@@ -614,10 +614,10 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
614
expected_result = torch.cat(expected_results)
615
616
rtol, atol = TOLERANCES[lora_result.dtype]
617
- assert torch.allclose(lora_result,
618
- expected_result,
619
- rtol=rtol,
620
- atol=atol)
617
+ torch.testing.assert_close(lora_result,
618
+ expected_result,
619
+ rtol=rtol,
620
+ atol=atol)
621
622
# Check that resetting the lora weights succeeds
623
@@ -642,10 +642,10 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
642
expected_result = linear(torch.cat(inputs))[0]
643
644
rtol, atol = TOLERANCES[lora_result.dtype]
645
- assert torch.allclose(lora_result,
646
- expected_result,
647
- rtol=rtol,
648
- atol=atol)
645
+ torch.testing.assert_close(lora_result,
646
+ expected_result,
647
+ rtol=rtol,
648
+ atol=atol)
649
650
651
@torch.inference_mode()
@@ -728,10 +728,10 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
728
expected_result = torch.cat(expected_results)
729
730
rtol, atol = TOLERANCES[lora_result.dtype]
731
- assert torch.allclose(lora_result,
732
- expected_result,
733
- rtol=rtol,
734
- atol=atol)
731
+ torch.testing.assert_close(lora_result,
732
+ expected_result,
733
+ rtol=rtol,
734
+ atol=atol)
735
736
# Check that resetting the lora weights succeeds
737
@@ -756,10 +756,10 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
756
expected_result = linear(torch.cat(inputs))[0]
757
758
rtol, atol = TOLERANCES[lora_result.dtype]
759
- assert torch.allclose(lora_result,
760
- expected_result,
761
- rtol=rtol,
762
- atol=atol)
759
+ torch.testing.assert_close(lora_result,
760
+ expected_result,
761
+ rtol=rtol,
762
+ atol=atol)
763
764
765
@torch.inference_mode()
@@ -868,10 +868,10 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
868
expected_result = torch.cat(expected_results)
869
870
rtol, atol = TOLERANCES[lora_result.dtype]
871
- assert torch.allclose(lora_result,
872
- expected_result,
873
- rtol=rtol,
874
- atol=atol)
871
+ torch.testing.assert_close(lora_result,
872
+ expected_result,
873
+ rtol=rtol,
874
+ atol=atol)
875
876
for slot_idx in range(max_loras):
877
lora_linear.reset_lora(slot_idx)
@@ -900,10 +900,10 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
900
expected_result = linear(torch.cat(inputs))[0]
901
902
rtol, atol = TOLERANCES[lora_result.dtype]
903
- assert torch.allclose(lora_result,
904
- expected_result,
905
- rtol=rtol,
906
- atol=atol)
903
+ torch.testing.assert_close(lora_result,
904
+ expected_result,
905
+ rtol=rtol,
906
+ atol=atol)
907
908
909
@torch.inference_mode()
tests/lora/test_lora_manager.py
+12
-12
@@ -533,21 +533,21 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
533
packed_lora = model_lora.get_lora("gate_up_proj")
534
assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
535
536
- assert torch.allclose(packed_lora.lora_a[0],
537
- model_lora.get_lora("gate_proj").lora_a)
538
- assert torch.allclose(packed_lora.lora_b[0],
539
- model_lora.get_lora("gate_proj").lora_b)
540
- assert torch.allclose(packed_lora.lora_a[1],
541
- model_lora.get_lora("up_proj").lora_a)
542
- assert torch.allclose(packed_lora.lora_b[1],
543
- model_lora.get_lora("up_proj").lora_b)
536
+ torch.testing.assert_close(packed_lora.lora_a[0],
537
+ model_lora.get_lora("gate_proj").lora_a)
538
+ torch.testing.assert_close(packed_lora.lora_b[0],
539
+ model_lora.get_lora("gate_proj").lora_b)
540
+ torch.testing.assert_close(packed_lora.lora_a[1],
541
+ model_lora.get_lora("up_proj").lora_a)
542
+ torch.testing.assert_close(packed_lora.lora_b[1],
543
+ model_lora.get_lora("up_proj").lora_b)
544
545
packed_lora1 = model_lora1.get_lora("gate_up_proj")
546
assert packed_lora1 and isinstance(packed_lora1, PackedLoRALayerWeights)
547
548
assert packed_lora1.lora_a[0] is None
549
assert packed_lora1.lora_b[0] is None
550
- assert torch.allclose(packed_lora1.lora_a[1],
551
- model_lora1.get_lora("up_proj").lora_a)
552
- assert torch.allclose(packed_lora1.lora_b[1],
553
- model_lora1.get_lora("up_proj").lora_b)
550
+ torch.testing.assert_close(packed_lora1.lora_a[1],
551
+ model_lora1.get_lora("up_proj").lora_a)
552
+ torch.testing.assert_close(packed_lora1.lora_b[1],
553
+ model_lora1.get_lora("up_proj").lora_b)
tests/quantization/test_fp8.py
+5
-3
@@ -127,16 +127,18 @@ def test_scaled_fp8_quant(dtype) -> None:
127
128
# Reference dynamic quantizaton
129
y = quantize_ref(x, inv_scale)
130
- assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
130
+ torch.testing.assert_close(ref_y,
131
+ per_tensor_dequantize(y, inv_scale, dtype))
132
133
# Static quantization
134
y, _ = ops.scaled_fp8_quant(x, inv_scale)
134
- assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
135
+ torch.testing.assert_close(ref_y,
136
+ per_tensor_dequantize(y, inv_scale, dtype))
137
138
# Padding
139
y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
140
assert y.shape[0] == 17
139
- assert torch.allclose(
141
+ torch.testing.assert_close(
142
ref_y,
143
per_tensor_dequantize(torch.narrow(y, 0, 0, x.shape[0]), inv_scale,
144
dtype))
tests/samplers/test_sampler.py
+1
-1
@@ -632,7 +632,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
632
633
hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
634
hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
635
- assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
635
+ torch.testing.assert_close(hf_probs, sample_probs, rtol=0.0, atol=1e-5)
636
assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
637
638
tests/spec_decode/utils.py
+1
-1
@@ -161,7 +161,7 @@ def assert_logprobs_dict_allclose(
161
single_step_actual_logprobs[token_id].logprob)
162
expected = torch.tensor(
163
single_step_expected_logprobs[token_id].logprob)
164
- assert torch.allclose(actual, expected)
164
+ torch.testing.assert_close(actual, expected)
165
166
167
def create_sampler_output_list(
tests/test_logits_processor.py
+4
-2
@@ -90,5 +90,7 @@ def test_logits_processors(seed: int, device: str):
90
assert torch.isinf(logits_processor_output[:, 0]).all()
91
92
fake_logits *= logits_processor.scale
93
- assert torch.allclose(logits_processor_output[:, 1], fake_logits[:, 1],
94
- 1e-4)
93
+ torch.testing.assert_close(logits_processor_output[:, 1],
94
+ fake_logits[:, 1],
95
+ rtol=1e-4,
96
+ atol=0.0)
tests/worker/test_model_runner.py
+9
-9
@@ -77,7 +77,7 @@ def test_prepare_prompt(batch_size):
77
device = model_runner.device
78
assert attn_metadata.num_prefills > 0
79
assert attn_metadata.num_decode_tokens == 0
80
- assert torch.allclose(
80
+ torch.testing.assert_close(
81
attn_metadata.seq_lens_tensor,
82
torch.tensor(seq_lens, device=device, dtype=torch.int))
83
assert attn_metadata.seq_lens == seq_lens
@@ -90,7 +90,7 @@ def test_prepare_prompt(batch_size):
90
for seq_len in seq_lens:
91
start_idx += seq_len
92
start_loc.append(start_idx)
93
- assert torch.allclose(
93
+ torch.testing.assert_close(
94
attn_metadata.query_start_loc,
95
torch.tensor(start_loc, dtype=torch.int32, device=device))
96
@@ -102,10 +102,10 @@ def test_prepare_prompt(batch_size):
102
start_idx += seq_len
103
seq_start_loc.append(start_idx)
104
105
- assert torch.allclose(
105
+ torch.testing.assert_close(
106
attn_metadata.seq_start_loc,
107
torch.tensor(start_loc, dtype=torch.int32, device=device))
108
- assert torch.allclose(
108
+ torch.testing.assert_close(
109
attn_metadata.context_lens_tensor,
110
torch.zeros(attn_metadata.context_lens_tensor.shape[0],
111
dtype=torch.int,
@@ -114,7 +114,7 @@ def test_prepare_prompt(batch_size):
114
expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
115
dtype=torch.int32,
116
device=model_runner.device)
117
- assert torch.allclose(attn_metadata.block_tables, expected)
117
+ torch.testing.assert_close(attn_metadata.block_tables, expected)
118
# Cuda graph should not be used for prerill.
119
assert attn_metadata.use_cuda_graph is False
120
@@ -201,7 +201,7 @@ def test_prepare_decode_cuda_graph(batch_size):
201
# decode has only 1 token for query.
202
start_idx += 1
203
start_loc.append(start_idx)
204
- assert torch.allclose(
204
+ torch.testing.assert_close(
205
attn_metadata.query_start_loc,
206
torch.tensor(start_loc, dtype=torch.int32, device=device))
207
@@ -210,15 +210,15 @@ def test_prepare_decode_cuda_graph(batch_size):
210
for seq_len in seq_lens:
211
start_idx += seq_len
212
seq_start_loc.append(start_idx)
213
- assert torch.allclose(
213
+ torch.testing.assert_close(
214
attn_metadata.seq_start_loc,
215
torch.tensor(seq_start_loc, dtype=torch.int32, device=device))
216
217
- assert torch.allclose(
217
+ torch.testing.assert_close(
218
attn_metadata.context_lens_tensor,
219
torch.tensor(context_lens, dtype=torch.int, device=device))
220
assert attn_metadata.max_decode_seq_len == max(seq_lens)
221
- assert torch.allclose(
221
+ torch.testing.assert_close(
222
attn_metadata.seq_lens_tensor[:len(seq_lens)],
223
torch.tensor(seq_lens, dtype=torch.int, device=device))
224