@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 50b8d08dbd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc/Testing] Use `torch.testing.assert_close` (#7324)

jon-chuang committed Aug 15, 2024 at 21:24 UTC 50b8d08dbd4493327e344bc627a0613947deba8f
25 files changed +197 -188
tests/distributed/test_comm_ops.py
+9 -9
@@ -34,7 +34,7 @@ def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
34 expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
35 t = all_tensors[rank % tp_size]
36 t = tensor_model_parallel_all_reduce(t)
37 - assert torch.allclose(t, expected)
37 + torch.testing.assert_close(t, expected)
38
39
40 @ray.remote(num_gpus=1, max_calls=1)
@@ -62,7 +62,7 @@ def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
62 expected = torch.cat(all_tensors, dim=all_gather_dimension)
63 t = all_tensors[rank % tp_size]
64 t = tensor_model_parallel_all_gather(t, all_gather_dimension)
65 - assert torch.allclose(t, expected)
65 + torch.testing.assert_close(t, expected)
66
67
68 @ray.remote(num_gpus=1, max_calls=1)
@@ -96,12 +96,12 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
96 else:
97 recv_dict = broadcast_tensor_dict(src=0)
98 assert len(recv_dict) == len(test_dict)
99 - assert torch.allclose(recv_dict["a"], test_dict["a"])
100 - assert torch.allclose(recv_dict["b"], test_dict["b"])
99 + torch.testing.assert_close(recv_dict["a"], test_dict["a"])
100 + torch.testing.assert_close(recv_dict["b"], test_dict["b"])
101 assert recv_dict["c"] == test_dict["c"]
102 assert recv_dict["d"] == test_dict["d"]
103 assert recv_dict["e"] == test_dict["e"]
104 - assert torch.allclose(recv_dict["f"], test_dict["f"])
104 + torch.testing.assert_close(recv_dict["f"], test_dict["f"])
105
106
107 @ray.remote(num_gpus=1, max_calls=1)
@@ -136,12 +136,12 @@ def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
136
137 if not get_pp_group().is_first_rank:
138 assert len(recv_dict) == len(test_dict)
139 - assert torch.allclose(recv_dict["a"], test_dict["a"])
140 - assert torch.allclose(recv_dict["b"], test_dict["b"])
139 + torch.testing.assert_close(recv_dict["a"], test_dict["a"])
140 + torch.testing.assert_close(recv_dict["b"], test_dict["b"])
141 assert recv_dict["c"] == test_dict["c"]
142 assert recv_dict["d"] == test_dict["d"]
143 assert recv_dict["e"] == test_dict["e"]
144 - assert torch.allclose(recv_dict["f"], test_dict["f"])
144 + torch.testing.assert_close(recv_dict["f"], test_dict["f"])
145
146
147 @ray.remote(num_gpus=1, max_calls=1)
@@ -163,7 +163,7 @@ def send_recv_test_worker(tp_size: int, pp_size: int, rank: int,
163 get_pp_group().send(test_tensor)
164
165 if not get_pp_group().is_first_rank:
166 - assert torch.allclose(test_tensor, recv_tensor)
166 + torch.testing.assert_close(test_tensor, recv_tensor)
167
168
169 @pytest.mark.skipif(torch.cuda.device_count() < 2,
tests/distributed/test_custom_all_reduce.py
+4 -4
@@ -72,8 +72,8 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
72 out2 = tensor_model_parallel_all_reduce(inp2)
73 dist.all_reduce(inp2, group=group)
74 graph.replay()
75 - assert torch.allclose(out1, inp1)
76 - assert torch.allclose(out2, inp2)
75 + torch.testing.assert_close(out1, inp1)
76 + torch.testing.assert_close(out2, inp2)
77
78
79 @ray.remote(num_gpus=1, max_calls=1)
@@ -96,13 +96,13 @@ def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
96 out = inp
97 for _ in range(num_communication):
98 out = fa.all_reduce_unreg(out)
99 - assert torch.allclose(out, inp * (tp_size**num_communication))
99 + torch.testing.assert_close(out, inp * (tp_size**num_communication))
100
101 inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
102 out = inp
103 for _ in range(num_communication):
104 out = fa.all_reduce_unreg(out)
105 - assert torch.allclose(out, inp * (tp_size**num_communication))
105 + torch.testing.assert_close(out, inp * (tp_size**num_communication))
106
107
108 @pytest.mark.parametrize("tp_size", [2])
tests/kernels/quant_utils.py
+1 -1
@@ -69,4 +69,4 @@ def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
69 ref_iscale = one / ref_scale
70 ref_out = (as_float32_tensor(x) * ref_iscale).clamp(
71 fp8_traits.min, fp8_traits.max).to(dtype=torch.float8_e4m3fn)
72 - return ref_out, ref_scale
72 + return ref_out, ref_scale.view((1, ))
tests/kernels/test_activation.py
+5 -5
@@ -47,7 +47,7 @@ def test_act_and_mul(
47 ref_out = layer.forward_native(x)
48 # The SiLU and GELU implementations are equivalent to the native PyTorch
49 # implementations, so we can do exact comparison.
50 - assert torch.allclose(out, ref_out, atol=0.0, rtol=0.0)
50 + torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
51
52
53 @pytest.mark.parametrize("activation", [FastGELU, NewGELU])
@@ -73,7 +73,7 @@ def test_activation(
73 layer = activation()
74 out = layer(x)
75 ref_out = layer.forward_native(x)
76 - assert torch.allclose(out,
77 - ref_out,
78 - atol=get_default_atol(out),
79 - rtol=get_default_rtol(out))
76 + torch.testing.assert_close(out,
77 + ref_out,
78 + atol=get_default_atol(out),
79 + rtol=get_default_rtol(out))
tests/kernels/test_attention.py
+2 -2
@@ -276,7 +276,7 @@ def test_paged_attention(
276 atol, rtol = 1e-3, 1e-5
277 if kv_cache_dtype == "fp8":
278 atol, rtol = 1e-2, 1e-5
279 - assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
279 + torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
280
281
282 def ref_multi_query_kv_attention(
@@ -379,4 +379,4 @@ def test_multi_query_kv_attention(
379 )
380 atol = get_default_atol(output) if is_hip() else 1e-3
381 rtol = get_default_rtol(output) if is_hip() else 1e-5
382 - assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
382 + torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
tests/kernels/test_blocksparse_attention.py
+2 -2
@@ -327,7 +327,7 @@ def test_paged_attention(
327 atol, rtol = 1e-3, 1e-5
328 if kv_cache_dtype == "fp8":
329 atol, rtol = 1e-2, 1e-5
330 - assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
330 + torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
331
332
333 def ref_multi_query_kv_attention(
@@ -441,4 +441,4 @@ def test_varlen_blocksparse_attention_prefill(
441 scale,
442 dtype,
443 )
444 - assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2)
444 + torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
tests/kernels/test_cache.py
+27 -27
@@ -98,10 +98,10 @@ def test_copy_blocks(
98
99 # Compare the results.
100 for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
101 - assert torch.allclose(key_cache, cloned_key_cache)
101 + torch.testing.assert_close(key_cache, cloned_key_cache)
102 for value_cache, cloned_value_cache in zip(value_caches,
103 cloned_value_caches):
104 - assert torch.allclose(value_cache, cloned_value_cache)
104 + torch.testing.assert_close(value_cache, cloned_value_cache)
105
106
107 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -184,17 +184,17 @@ def test_reshape_and_cache(
184 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
185
186 if kv_cache_dtype == "fp8":
187 - assert torch.allclose(result_key_cache,
188 - cloned_key_cache,
189 - atol=0.001,
190 - rtol=0.1)
191 - assert torch.allclose(result_value_cache,
192 - cloned_value_cache,
193 - atol=0.001,
194 - rtol=0.1)
187 + torch.testing.assert_close(result_key_cache,
188 + cloned_key_cache,
189 + atol=0.001,
190 + rtol=0.1)
191 + torch.testing.assert_close(result_value_cache,
192 + cloned_value_cache,
193 + atol=0.001,
194 + rtol=0.1)
195 else:
196 - assert torch.allclose(key_cache, cloned_key_cache)
197 - assert torch.allclose(value_cache, cloned_value_cache)
196 + torch.testing.assert_close(key_cache, cloned_key_cache)
197 + torch.testing.assert_close(value_cache, cloned_value_cache)
198
199
200 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -290,17 +290,17 @@ def test_reshape_and_cache_flash(
290 cloned_value_cache[block_idx, block_offset, :, :] = value[i]
291
292 if kv_cache_dtype == "fp8":
293 - assert torch.allclose(result_key_cache,
294 - cloned_key_cache,
295 - atol=0.001,
296 - rtol=0.1)
297 - assert torch.allclose(result_value_cache,
298 - cloned_value_cache,
299 - atol=0.001,
300 - rtol=0.1)
293 + torch.testing.assert_close(result_key_cache,
294 + cloned_key_cache,
295 + atol=0.001,
296 + rtol=0.1)
297 + torch.testing.assert_close(result_value_cache,
298 + cloned_value_cache,
299 + atol=0.001,
300 + rtol=0.1)
301 else:
302 - assert torch.allclose(key_cache, cloned_key_cache)
303 - assert torch.allclose(value_cache, cloned_value_cache)
302 + torch.testing.assert_close(key_cache, cloned_key_cache)
303 + torch.testing.assert_close(value_cache, cloned_value_cache)
304
305
306 @pytest.mark.parametrize("direction", COPYING_DIRECTION)
@@ -372,10 +372,10 @@ def test_swap_blocks(
372 block_mapping_tensor)
373
374 for src, dst in block_mapping:
375 - assert torch.allclose(src_key_caches_clone[src].cpu(),
376 - dist_key_caches[0][dst].cpu())
377 - assert torch.allclose(src_value_caches_clone[src].cpu(),
378 - dist_value_caches[0][dst].cpu())
375 + torch.testing.assert_close(src_key_caches_clone[src].cpu(),
376 + dist_key_caches[0][dst].cpu())
377 + torch.testing.assert_close(src_value_caches_clone[src].cpu(),
378 + dist_value_caches[0][dst].cpu())
379
380
381 @pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -411,4 +411,4 @@ def test_fp8_e4m3_conversion(
411 converted_cache = torch.empty_like(cache)
412 ops.convert_fp8(converted_cache, cache_fp8)
413
414 - assert torch.allclose(cache, converted_cache, atol=0.001, rtol=0.1)
414 + torch.testing.assert_close(cache, converted_cache, atol=0.001, rtol=0.1)
tests/kernels/test_cutlass.py
+13 -10
@@ -74,7 +74,7 @@ def cutlass_fp8_gemm_helper(m: int,
74 out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
75 baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
76
77 - assert torch.allclose(out, baseline, rtol=1e-2, atol=5e-2)
77 + torch.testing.assert_close(out, baseline, rtol=1e-2, atol=5e-2)
78
79
80 def cutlass_int8_gemm_helper(m: int,
@@ -106,7 +106,7 @@ def cutlass_int8_gemm_helper(m: int,
106 out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
107 baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
108
109 - assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
109 + torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
110
111
112 @pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
@@ -252,7 +252,7 @@ def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int,
252 azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
253
254 a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32)
255 - assert torch.allclose(a_dq, scale_a * aq_f32 + azp_a)
255 + torch.testing.assert_close(a_dq, scale_a * aq_f32 + azp_a)
256
257 baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype)
258
@@ -271,8 +271,8 @@ def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int,
271 scale_b,
272 out_dtype=out_dtype,
273 bias=azp_bias[0, :])
274 - assert torch.allclose(out, baseline_dq, rtol=1e-2, atol=1e0)
275 - assert torch.allclose(out, baseline_q, rtol=1e-2, atol=1e0)
274 + torch.testing.assert_close(out, baseline_dq, rtol=1e-2, atol=1e0)
275 + torch.testing.assert_close(out, baseline_q, rtol=1e-2, atol=1e0)
276
277
278 @pytest.mark.parametrize("m", [32, 64, 128])
@@ -302,7 +302,10 @@ def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
302 azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
303
304 a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32)
305 - assert torch.allclose(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3)
305 + torch.testing.assert_close(a_dq,
306 + scale_a * aq_f32 - azp_a,
307 + rtol=1e-4,
308 + atol=1e-3)
309
310 if use_bias:
311 bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5
@@ -335,8 +338,8 @@ def test_cutlass_int8_azp(m: int, n: int, k: int, out_dtype: torch.dtype,
338 # float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05%
339 rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3
340 atol = 1e-3
338 - assert torch.allclose(out, baseline_dq, rtol=rtol, atol=atol)
339 - assert torch.allclose(out, baseline_q, rtol=rtol, atol=atol)
341 + torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol)
342 + torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol)
343
344
345 # Test working with a subset of A and B
@@ -363,7 +366,7 @@ def test_cutlass_subset():
366 scale_b,
367 out_dtype=torch.bfloat16)
368
366 - assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
369 + torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
370
371
372 # Test to make sure cuda graphs work
@@ -411,4 +414,4 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
414
415 baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
416 scale_b * b.to(dtype=torch.float32)).to(torch.bfloat16)
414 - assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
417 + torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
tests/kernels/test_flash_attn.py
+2 -2
@@ -126,7 +126,7 @@ def test_flash_attn_with_paged_kv(
126 scale=scale,
127 soft_cap=soft_cap,
128 )
129 - assert torch.allclose(output, ref_output, atol=2e-2, rtol=1e-2), \
129 + torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), \
130 f"{torch.max(torch.abs(output - ref_output))}"
131
132
@@ -211,5 +211,5 @@ def test_varlen_with_paged_kv(
211 sliding_window=sliding_window,
212 soft_cap=soft_cap,
213 )
214 - assert torch.allclose(output, ref_output, atol=2e-2, rtol=1e-2), \
214 + torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), \
215 f"{torch.max(torch.abs(output - ref_output))}"
tests/kernels/test_flashinfer.py
+2 -2
@@ -144,7 +144,7 @@ def test_flashinfer_decode_with_paged_kv(kv_lens: List[int],
144 block_tables=block_tables,
145 scale=scale,
146 soft_cap=soft_cap)
147 - assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
147 + torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2), \
148 f"{torch.max(torch.abs(output - ref_output))}"
149
150
@@ -244,5 +244,5 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
244 block_tables=block_tables,
245 scale=scale,
246 soft_cap=soft_cap)
247 - assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
247 + torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2), \
248 f"{torch.max(torch.abs(output - ref_output))}"
tests/kernels/test_fp8_quant.py
+7 -7
@@ -37,9 +37,9 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
37 scale_ub=scale_ub,
38 use_per_token_if_dynamic=True)
39
40 - assert torch.allclose(ref_scales, ops_scales)
41 - assert torch.allclose(ref_out.to(dtype=torch.float32),
42 - ops_out.to(dtype=torch.float32))
40 + torch.testing.assert_close(ref_scales, ops_scales)
41 + torch.testing.assert_close(ref_out.to(dtype=torch.float32),
42 + ops_out.to(dtype=torch.float32))
43
44
45 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -57,9 +57,9 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
57 ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
58 ops_out, ops_scale = ops.scaled_fp8_quant(x)
59
60 - assert torch.allclose(ref_scale, ops_scale)
61 - assert torch.allclose(ref_out.to(dtype=torch.float32),
62 - ops_out.to(dtype=torch.float32))
60 + torch.testing.assert_close(ref_scale, ops_scale)
61 + torch.testing.assert_close(ref_out.to(dtype=torch.float32),
62 + ops_out.to(dtype=torch.float32))
63
64
65 # Regression test for a case with large activations where an int32 index cannot
@@ -84,4 +84,4 @@ def test_fp8_quant_large(seed: int) -> None:
84 ref_out = ref_out.to(dtype=dtype)
85 ops_out = ops_out.to(dtype=dtype)
86
87 - assert torch.allclose(ref_out, ops_out)
87 + torch.testing.assert_close(ref_out, ops_out)
tests/kernels/test_int8_quant.py
+7 -5
@@ -29,9 +29,10 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
29 # kernel
30 ops_out, ops_scales = scaled_int8_quant(x)
31
32 - assert torch.allclose(ops_scales, ref_scales)
33 - assert torch.allclose(ops_out, ref_out,
34 - atol=1) # big atol to account for rounding errors
32 + torch.testing.assert_close(ops_scales, ref_scales)
33 + torch.testing.assert_close(
34 + ops_out, ref_out, atol=1,
35 + rtol=0.0) # big atol to account for rounding errors
36
37
38 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -54,5 +55,6 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
55 int8_traits.max).to(torch.int8)
56 out2, _ = scaled_int8_quant(x, scale)
57
57 - assert torch.allclose(out1, out2,
58 - atol=1) # big atol to account for rounding errors
58 + torch.testing.assert_close(
59 + out1, out2, atol=1,
60 + rtol=0.0) # big atol to account for rounding errors
tests/kernels/test_layernorm.py
+3 -3
@@ -48,7 +48,7 @@ def test_rms_norm(
48 # numerical errors than other operators because they involve reductions.
49 # Therefore, we use a larger tolerance.
50 if add_residual:
51 - assert torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
52 - assert torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
51 + torch.testing.assert_close(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
52 + torch.testing.assert_close(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
53 else:
54 - assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)
54 + torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
tests/kernels/test_marlin_gemm.py
+2 -2
@@ -122,7 +122,7 @@ def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
122 )
123 torch.cuda.synchronize()
124
125 - assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
125 + torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
126
127
128 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
@@ -174,7 +174,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, group_size,
174 )
175 torch.cuda.synchronize()
176
177 - assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
177 + torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
178
179
180 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
tests/kernels/test_moe.py
+5 -5
@@ -50,7 +50,7 @@ def test_fused_moe(
50 score = torch.randn((m, e), device='cuda', dtype=dtype)
51 triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False)
52 torch_output = torch_moe(a, w1, w2, score, topk)
53 - assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
53 + torch.testing.assert_close(triton_output, torch_output, atol=1e-2, rtol=0)
54
55
56 @pytest.mark.parametrize("dtype",
@@ -95,7 +95,7 @@ def test_mixtral_moe(dtype: torch.dtype):
95 torch.bfloat16: 1e-2,
96 }
97
98 - assert torch.allclose(hf_states.flatten(0, 1),
99 - vllm_states,
100 - rtol=mixtral_moe_tol[dtype],
101 - atol=mixtral_moe_tol[dtype])
98 + torch.testing.assert_close(hf_states.flatten(0, 1),
99 + vllm_states,
100 + rtol=mixtral_moe_tol[dtype],
101 + atol=mixtral_moe_tol[dtype])
tests/kernels/test_pos_encoding.py
+24 -24
@@ -67,14 +67,14 @@ def test_rotary_embedding(
67 ref_query, ref_key = rope.forward_native(positions, query, key)
68 out_query, out_key = rope.forward(positions, query, key)
69 # Compare the results.
70 - assert torch.allclose(out_query,
71 - ref_query,
72 - atol=get_default_atol(out_query),
73 - rtol=get_default_rtol(out_query))
74 - assert torch.allclose(out_key,
75 - ref_key,
76 - atol=get_default_atol(out_key),
77 - rtol=get_default_rtol(out_key))
70 + torch.testing.assert_close(out_query,
71 + ref_query,
72 + atol=get_default_atol(out_query),
73 + rtol=get_default_rtol(out_query))
74 + torch.testing.assert_close(out_key,
75 + ref_key,
76 + atol=get_default_atol(out_key),
77 + rtol=get_default_rtol(out_key))
78
79
80 @pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -129,14 +129,14 @@ def test_batched_rotary_embedding(
129 dtype=torch.long,
130 device=device))
131 # Compare the results.
132 - assert torch.allclose(out_query,
133 - ref_query,
134 - atol=get_default_atol(out_query),
135 - rtol=get_default_rtol(out_query))
136 - assert torch.allclose(out_key,
137 - ref_key,
138 - atol=get_default_atol(out_key),
139 - rtol=get_default_rtol(out_key))
132 + torch.testing.assert_close(out_query,
133 + ref_query,
134 + atol=get_default_atol(out_query),
135 + rtol=get_default_rtol(out_query))
136 + torch.testing.assert_close(out_key,
137 + ref_key,
138 + atol=get_default_atol(out_key),
139 + rtol=get_default_rtol(out_key))
140
141
142 @pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -200,14 +200,14 @@ def test_batched_rotary_embedding_multi_lora(
200 out_query, out_key = rope.forward(positions, query, key,
201 query_offsets.flatten())
202 # Compare the results.
203 - assert torch.allclose(out_query,
204 - ref_query,
205 - atol=get_default_atol(out_query),
206 - rtol=get_default_rtol(out_query))
207 - assert torch.allclose(out_key,
208 - ref_key,
209 - atol=get_default_atol(out_key),
210 - rtol=get_default_rtol(out_key))
203 + torch.testing.assert_close(out_query,
204 + ref_query,
205 + atol=get_default_atol(out_query),
206 + rtol=get_default_rtol(out_query))
207 + torch.testing.assert_close(out_key,
208 + ref_key,
209 + atol=get_default_atol(out_key),
210 + rtol=get_default_rtol(out_key))
211
212
213 @torch.inference_mode()
tests/kernels/test_sampler.py
+4 -4
@@ -100,11 +100,11 @@ def test_sample_decoding_only(random_sampling, max_best_of,
100 if modify_greedy_probs and not request_uses_random_sampling:
101 # If we are modifying greedy probs and the request is greedy,
102 # we want to make sure the probs tensor is modified in place
103 - assert torch.allclose(
103 + torch.testing.assert_close(
104 probs[i][sampled_tokens[i]],
105 torch.full_like(probs[i][sampled_tokens[i]], 1.0))
106 assert torch.sum(probs[i]) == 1.0
107 - assert torch.allclose(
107 + torch.testing.assert_close(
108 sampled_modified_probs[i][0],
109 torch.full_like(sampled_modified_probs[i][0], 1.0))
110 elif request_uses_random_sampling:
@@ -117,8 +117,8 @@ def test_sample_decoding_only(random_sampling, max_best_of,
117 # If the request is greedy and we are not modifying greedy probs,
118 # we want to make sure sampled_modified_probs tensor is the same as
119 # the probs tensor.
120 - assert torch.allclose(sampled_modified_probs[i][0],
121 - probs[i][sampled_tokens[i]])
120 + torch.testing.assert_close(sampled_modified_probs[i],
121 + probs[i][sampled_tokens[i]])
122
123 if save_logprobs:
124 assert sampled_logprobs.shape == (bs, max_best_of)
tests/kernels/utils.py
+2 -2
@@ -924,5 +924,5 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
924 * output_under_test: actually observed output value
925 '''
926 ideal_output = test_params.packed_qkvo.ideal_output
927 - assert torch.allclose(ideal_output,
928 - output_under_test.view_as(ideal_output))
927 + torch.testing.assert_close(ideal_output,
928 + output_under_test.view_as(ideal_output))
tests/lora/test_layers.py
+44 -44
@@ -247,10 +247,10 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
247 expected_result = torch.cat(expected_results)
248
249 rtol, atol = TOLERANCES[lora_result.dtype]
250 - assert torch.allclose(lora_result,
251 - expected_result,
252 - rtol=rtol,
253 - atol=atol)
250 + torch.testing.assert_close(lora_result,
251 + expected_result,
252 + rtol=rtol,
253 + atol=atol)
254
255 # Check that resetting the lora weights succeeds
256
@@ -274,10 +274,10 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
274 expected_result = embedding(torch.cat(inputs))
275
276 rtol, atol = TOLERANCES[lora_result.dtype]
277 - assert torch.allclose(lora_result,
278 - expected_result,
279 - rtol=rtol,
280 - atol=atol)
277 + torch.testing.assert_close(lora_result,
278 + expected_result,
279 + rtol=rtol,
280 + atol=atol)
281
282
283 @torch.inference_mode()
@@ -384,10 +384,10 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
384 expected_result = torch.cat(expected_results)
385
386 rtol, atol = TOLERANCES[lora_result.dtype]
387 - assert torch.allclose(lora_result,
388 - expected_result,
389 - rtol=rtol,
390 - atol=atol)
387 + torch.testing.assert_close(lora_result,
388 + expected_result,
389 + rtol=rtol,
390 + atol=atol)
391
392 # Check that resetting the lora weights succeeds
393
@@ -411,10 +411,10 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
411 expected_result = expanded_embedding(torch.cat(inputs))
412
413 rtol, atol = TOLERANCES[lora_result.dtype]
414 - assert torch.allclose(lora_result,
415 - expected_result,
416 - rtol=rtol,
417 - atol=atol)
414 + torch.testing.assert_close(lora_result,
415 + expected_result,
416 + rtol=rtol,
417 + atol=atol)
418
419
420 @torch.inference_mode()
@@ -541,10 +541,10 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
541 embedding_bias=None)
542
543 rtol, atol = TOLERANCES[lora_result.dtype]
544 - assert torch.allclose(lora_result,
545 - expected_result,
546 - rtol=rtol,
547 - atol=atol)
544 + torch.testing.assert_close(lora_result,
545 + expected_result,
546 + rtol=rtol,
547 + atol=atol)
548
549
550 @torch.inference_mode()
@@ -614,10 +614,10 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
614 expected_result = torch.cat(expected_results)
615
616 rtol, atol = TOLERANCES[lora_result.dtype]
617 - assert torch.allclose(lora_result,
618 - expected_result,
619 - rtol=rtol,
620 - atol=atol)
617 + torch.testing.assert_close(lora_result,
618 + expected_result,
619 + rtol=rtol,
620 + atol=atol)
621
622 # Check that resetting the lora weights succeeds
623
@@ -642,10 +642,10 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
642 expected_result = linear(torch.cat(inputs))[0]
643
644 rtol, atol = TOLERANCES[lora_result.dtype]
645 - assert torch.allclose(lora_result,
646 - expected_result,
647 - rtol=rtol,
648 - atol=atol)
645 + torch.testing.assert_close(lora_result,
646 + expected_result,
647 + rtol=rtol,
648 + atol=atol)
649
650
651 @torch.inference_mode()
@@ -728,10 +728,10 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
728 expected_result = torch.cat(expected_results)
729
730 rtol, atol = TOLERANCES[lora_result.dtype]
731 - assert torch.allclose(lora_result,
732 - expected_result,
733 - rtol=rtol,
734 - atol=atol)
731 + torch.testing.assert_close(lora_result,
732 + expected_result,
733 + rtol=rtol,
734 + atol=atol)
735
736 # Check that resetting the lora weights succeeds
737
@@ -756,10 +756,10 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
756 expected_result = linear(torch.cat(inputs))[0]
757
758 rtol, atol = TOLERANCES[lora_result.dtype]
759 - assert torch.allclose(lora_result,
760 - expected_result,
761 - rtol=rtol,
762 - atol=atol)
759 + torch.testing.assert_close(lora_result,
760 + expected_result,
761 + rtol=rtol,
762 + atol=atol)
763
764
765 @torch.inference_mode()
@@ -868,10 +868,10 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
868 expected_result = torch.cat(expected_results)
869
870 rtol, atol = TOLERANCES[lora_result.dtype]
871 - assert torch.allclose(lora_result,
872 - expected_result,
873 - rtol=rtol,
874 - atol=atol)
871 + torch.testing.assert_close(lora_result,
872 + expected_result,
873 + rtol=rtol,
874 + atol=atol)
875
876 for slot_idx in range(max_loras):
877 lora_linear.reset_lora(slot_idx)
@@ -900,10 +900,10 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
900 expected_result = linear(torch.cat(inputs))[0]
901
902 rtol, atol = TOLERANCES[lora_result.dtype]
903 - assert torch.allclose(lora_result,
904 - expected_result,
905 - rtol=rtol,
906 - atol=atol)
903 + torch.testing.assert_close(lora_result,
904 + expected_result,
905 + rtol=rtol,
906 + atol=atol)
907
908
909 @torch.inference_mode()
tests/lora/test_lora_manager.py
+12 -12
@@ -533,21 +533,21 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
533 packed_lora = model_lora.get_lora("gate_up_proj")
534 assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
535
536 - assert torch.allclose(packed_lora.lora_a[0],
537 - model_lora.get_lora("gate_proj").lora_a)
538 - assert torch.allclose(packed_lora.lora_b[0],
539 - model_lora.get_lora("gate_proj").lora_b)
540 - assert torch.allclose(packed_lora.lora_a[1],
541 - model_lora.get_lora("up_proj").lora_a)
542 - assert torch.allclose(packed_lora.lora_b[1],
543 - model_lora.get_lora("up_proj").lora_b)
536 + torch.testing.assert_close(packed_lora.lora_a[0],
537 + model_lora.get_lora("gate_proj").lora_a)
538 + torch.testing.assert_close(packed_lora.lora_b[0],
539 + model_lora.get_lora("gate_proj").lora_b)
540 + torch.testing.assert_close(packed_lora.lora_a[1],
541 + model_lora.get_lora("up_proj").lora_a)
542 + torch.testing.assert_close(packed_lora.lora_b[1],
543 + model_lora.get_lora("up_proj").lora_b)
544
545 packed_lora1 = model_lora1.get_lora("gate_up_proj")
546 assert packed_lora1 and isinstance(packed_lora1, PackedLoRALayerWeights)
547
548 assert packed_lora1.lora_a[0] is None
549 assert packed_lora1.lora_b[0] is None
550 - assert torch.allclose(packed_lora1.lora_a[1],
551 - model_lora1.get_lora("up_proj").lora_a)
552 - assert torch.allclose(packed_lora1.lora_b[1],
553 - model_lora1.get_lora("up_proj").lora_b)
550 + torch.testing.assert_close(packed_lora1.lora_a[1],
551 + model_lora1.get_lora("up_proj").lora_a)
552 + torch.testing.assert_close(packed_lora1.lora_b[1],
553 + model_lora1.get_lora("up_proj").lora_b)
tests/quantization/test_fp8.py
+5 -3
@@ -127,16 +127,18 @@ def test_scaled_fp8_quant(dtype) -> None:
127
128 # Reference dynamic quantizaton
129 y = quantize_ref(x, inv_scale)
130 - assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
130 + torch.testing.assert_close(ref_y,
131 + per_tensor_dequantize(y, inv_scale, dtype))
132
133 # Static quantization
134 y, _ = ops.scaled_fp8_quant(x, inv_scale)
134 - assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
135 + torch.testing.assert_close(ref_y,
136 + per_tensor_dequantize(y, inv_scale, dtype))
137
138 # Padding
139 y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
140 assert y.shape[0] == 17
139 - assert torch.allclose(
141 + torch.testing.assert_close(
142 ref_y,
143 per_tensor_dequantize(torch.narrow(y, 0, 0, x.shape[0]), inv_scale,
144 dtype))
tests/samplers/test_sampler.py
+1 -1
@@ -632,7 +632,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
632
633 hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
634 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
635 - assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
635 + torch.testing.assert_close(hf_probs, sample_probs, rtol=0.0, atol=1e-5)
636 assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
637
638
tests/spec_decode/utils.py
+1 -1
@@ -161,7 +161,7 @@ def assert_logprobs_dict_allclose(
161 single_step_actual_logprobs[token_id].logprob)
162 expected = torch.tensor(
163 single_step_expected_logprobs[token_id].logprob)
164 - assert torch.allclose(actual, expected)
164 + torch.testing.assert_close(actual, expected)
165
166
167 def create_sampler_output_list(
tests/test_logits_processor.py
+4 -2
@@ -90,5 +90,7 @@ def test_logits_processors(seed: int, device: str):
90 assert torch.isinf(logits_processor_output[:, 0]).all()
91
92 fake_logits *= logits_processor.scale
93 - assert torch.allclose(logits_processor_output[:, 1], fake_logits[:, 1],
94 - 1e-4)
93 + torch.testing.assert_close(logits_processor_output[:, 1],
94 + fake_logits[:, 1],
95 + rtol=1e-4,
96 + atol=0.0)
tests/worker/test_model_runner.py
+9 -9
@@ -77,7 +77,7 @@ def test_prepare_prompt(batch_size):
77 device = model_runner.device
78 assert attn_metadata.num_prefills > 0
79 assert attn_metadata.num_decode_tokens == 0
80 - assert torch.allclose(
80 + torch.testing.assert_close(
81 attn_metadata.seq_lens_tensor,
82 torch.tensor(seq_lens, device=device, dtype=torch.int))
83 assert attn_metadata.seq_lens == seq_lens
@@ -90,7 +90,7 @@ def test_prepare_prompt(batch_size):
90 for seq_len in seq_lens:
91 start_idx += seq_len
92 start_loc.append(start_idx)
93 - assert torch.allclose(
93 + torch.testing.assert_close(
94 attn_metadata.query_start_loc,
95 torch.tensor(start_loc, dtype=torch.int32, device=device))
96
@@ -102,10 +102,10 @@ def test_prepare_prompt(batch_size):
102 start_idx += seq_len
103 seq_start_loc.append(start_idx)
104
105 - assert torch.allclose(
105 + torch.testing.assert_close(
106 attn_metadata.seq_start_loc,
107 torch.tensor(start_loc, dtype=torch.int32, device=device))
108 - assert torch.allclose(
108 + torch.testing.assert_close(
109 attn_metadata.context_lens_tensor,
110 torch.zeros(attn_metadata.context_lens_tensor.shape[0],
111 dtype=torch.int,
@@ -114,7 +114,7 @@ def test_prepare_prompt(batch_size):
114 expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
115 dtype=torch.int32,
116 device=model_runner.device)
117 - assert torch.allclose(attn_metadata.block_tables, expected)
117 + torch.testing.assert_close(attn_metadata.block_tables, expected)
118 # Cuda graph should not be used for prerill.
119 assert attn_metadata.use_cuda_graph is False
120
@@ -201,7 +201,7 @@ def test_prepare_decode_cuda_graph(batch_size):
201 # decode has only 1 token for query.
202 start_idx += 1
203 start_loc.append(start_idx)
204 - assert torch.allclose(
204 + torch.testing.assert_close(
205 attn_metadata.query_start_loc,
206 torch.tensor(start_loc, dtype=torch.int32, device=device))
207
@@ -210,15 +210,15 @@ def test_prepare_decode_cuda_graph(batch_size):
210 for seq_len in seq_lens:
211 start_idx += seq_len
212 seq_start_loc.append(start_idx)
213 - assert torch.allclose(
213 + torch.testing.assert_close(
214 attn_metadata.seq_start_loc,
215 torch.tensor(seq_start_loc, dtype=torch.int32, device=device))
216
217 - assert torch.allclose(
217 + torch.testing.assert_close(
218 attn_metadata.context_lens_tensor,
219 torch.tensor(context_lens, dtype=torch.int, device=device))
220 assert attn_metadata.max_decode_seq_len == max(seq_lens)
221 - assert torch.allclose(
221 + torch.testing.assert_close(
222 attn_metadata.seq_lens_tensor[:len(seq_lens)],
223 torch.tensor(seq_lens, dtype=torch.int, device=device))
224