A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Distributed] Make it clear that % should not be in tensor dict keys. (#5927)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com>
xwjiang2010 committed
Jun 28, 2024 at 08:20 UTC
b90d8cd832669b9ad7c48cd9a431e80836778b56
2 files changed
+12
-1
tests/distributed/test_parallel_state.py
+9
-1
@@ -1,5 +1,6 @@
1
from typing import Any, Dict
2
3
+import pytest
4
import torch
5
6
from vllm.distributed.parallel_state import (_split_tensor_dict,
@@ -24,6 +25,14 @@ def test_split_tensor_dict():
25
assert torch.allclose(tensor_list[2], test_dict["key_c"]["key_2"])
26
27
28
+def test_split_tensor_dict_invalid_key():
29
+ test_dict = {
30
+ "a%b": "a",
31
+ }
32
+ with pytest.raises(AssertionError):
33
+ _split_tensor_dict(test_dict)
34
+
35
+
36
def test_update_nested_dict():
37
flattened_keys_values = [("key1%key2%key3", "value1"),
38
("key1%key2%key4", "value2"),
@@ -31,7 +40,6 @@ def test_update_nested_dict():
40
("key8", "value5")]
41
res: Dict[str, Any] = {}
42
34
- # Update the nested dictionary with each flattened key-value pair
43
for flat_key, value in flattened_keys_values:
44
_update_nested_dict(res, flat_key, value)
45
assert res == {
vllm/distributed/parallel_state.py
+3
@@ -58,6 +58,9 @@ def _split_tensor_dict(
58
metadata_list: List[Tuple[str, Any]] = []
59
tensor_list = []
60
for key, value in tensor_dict.items():
61
+ assert "%" not in key, (
62
+ "Avoid having '%' in key "
63
+ "as it is used as a separator for nested entries.")
64
if isinstance(value, torch.Tensor):
65
# Note: we cannot use `value.device` here,
66
# because it contains not only the device type but also the device