@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b90d8cd832
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Distributed] Make it clear that % should not be in tensor dict keys. (#5927)

Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com>

xwjiang2010 committed Jun 28, 2024 at 08:20 UTC b90d8cd832669b9ad7c48cd9a431e80836778b56
2 files changed +12 -1
tests/distributed/test_parallel_state.py
+9 -1
@@ -1,5 +1,6 @@
1 from typing import Any, Dict
2
3 +import pytest
4 import torch
5
6 from vllm.distributed.parallel_state import (_split_tensor_dict,
@@ -24,6 +25,14 @@ def test_split_tensor_dict():
25 assert torch.allclose(tensor_list[2], test_dict["key_c"]["key_2"])
26
27
28 +def test_split_tensor_dict_invalid_key():
29 + test_dict = {
30 + "a%b": "a",
31 + }
32 + with pytest.raises(AssertionError):
33 + _split_tensor_dict(test_dict)
34 +
35 +
36 def test_update_nested_dict():
37 flattened_keys_values = [("key1%key2%key3", "value1"),
38 ("key1%key2%key4", "value2"),
@@ -31,7 +40,6 @@ def test_update_nested_dict():
40 ("key8", "value5")]
41 res: Dict[str, Any] = {}
42
34 - # Update the nested dictionary with each flattened key-value pair
43 for flat_key, value in flattened_keys_values:
44 _update_nested_dict(res, flat_key, value)
45 assert res == {
vllm/distributed/parallel_state.py
+3
@@ -58,6 +58,9 @@ def _split_tensor_dict(
58 metadata_list: List[Tuple[str, Any]] = []
59 tensor_list = []
60 for key, value in tensor_dict.items():
61 + assert "%" not in key, (
62 + "Avoid having '%' in key "
63 + "as it is used as a separator for nested entries.")
64 if isinstance(value, torch.Tensor):
65 # Note: we cannot use `value.device` here,
66 # because it contains not only the device type but also the device