@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c593e117d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 04:45 UTC

Fix boolean nested params, add dict format support, and enhance plotting for vllm bench sweep (#29025)

Signed-off-by: Luka Govedič <luka.govedic@gmail.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ProExpertProg <11367180+ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <luka.govedic@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>

Copilot committed Dec 2, 2025 at 20:40 UTC 1c593e117d3a818815b5d07992a096d53b519a15
5 files changed +614 -22
tests/benchmarks/test_param_sweep.py new
+257
@@ -0,0 +1,257 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import json
4 +import tempfile
5 +from pathlib import Path
6 +
7 +import pytest
8 +
9 +from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
10 +
11 +
12 +class TestParameterSweepItem:
13 + """Test ParameterSweepItem functionality."""
14 +
15 + @pytest.mark.parametrize(
16 + "input_dict,expected",
17 + [
18 + (
19 + {"compilation_config.use_inductor_graph_partition": False},
20 + "--compilation-config.use_inductor_graph_partition=false",
21 + ),
22 + (
23 + {"compilation_config.use_inductor_graph_partition": True},
24 + "--compilation-config.use_inductor_graph_partition=true",
25 + ),
26 + (
27 + {"compilation_config.use_inductor": False},
28 + "--compilation-config.use_inductor=false",
29 + ),
30 + (
31 + {"compilation_config.use_inductor": True},
32 + "--compilation-config.use_inductor=true",
33 + ),
34 + ],
35 + )
36 + def test_nested_boolean_params(self, input_dict, expected):
37 + """Test that nested boolean params use =true/false syntax."""
38 + item = ParameterSweepItem.from_record(input_dict)
39 + cmd = item.apply_to_cmd(["vllm", "serve", "model"])
40 + assert expected in cmd
41 +
42 + @pytest.mark.parametrize(
43 + "input_dict,expected",
44 + [
45 + ({"enable_prefix_caching": False}, "--no-enable-prefix-caching"),
46 + ({"enable_prefix_caching": True}, "--enable-prefix-caching"),
47 + ({"disable_log_stats": False}, "--no-disable-log-stats"),
48 + ({"disable_log_stats": True}, "--disable-log-stats"),
49 + ],
50 + )
51 + def test_non_nested_boolean_params(self, input_dict, expected):
52 + """Test that non-nested boolean params use --no- prefix."""
53 + item = ParameterSweepItem.from_record(input_dict)
54 + cmd = item.apply_to_cmd(["vllm", "serve", "model"])
55 + assert expected in cmd
56 +
57 + @pytest.mark.parametrize(
58 + "compilation_config",
59 + [
60 + {"cudagraph_mode": "full", "mode": 2, "use_inductor_graph_partition": True},
61 + {
62 + "cudagraph_mode": "piecewise",
63 + "mode": 3,
64 + "use_inductor_graph_partition": False,
65 + },
66 + ],
67 + )
68 + def test_nested_dict_value(self, compilation_config):
69 + """Test that nested dict values are serialized as JSON."""
70 + item = ParameterSweepItem.from_record(
71 + {"compilation_config": compilation_config}
72 + )
73 + cmd = item.apply_to_cmd(["vllm", "serve", "model"])
74 + assert "--compilation-config" in cmd
75 + # The dict should be JSON serialized
76 + idx = cmd.index("--compilation-config")
77 + assert json.loads(cmd[idx + 1]) == compilation_config
78 +
79 + @pytest.mark.parametrize(
80 + "input_dict,expected_key,expected_value",
81 + [
82 + ({"model": "test-model"}, "--model", "test-model"),
83 + ({"max_tokens": 100}, "--max-tokens", "100"),
84 + ({"temperature": 0.7}, "--temperature", "0.7"),
85 + ],
86 + )
87 + def test_string_and_numeric_values(self, input_dict, expected_key, expected_value):
88 + """Test that string and numeric values are handled correctly."""
89 + item = ParameterSweepItem.from_record(input_dict)
90 + cmd = item.apply_to_cmd(["vllm", "serve"])
91 + assert expected_key in cmd
92 + assert expected_value in cmd
93 +
94 + @pytest.mark.parametrize(
95 + "input_dict,expected_key,key_idx_offset",
96 + [
97 + ({"max_tokens": 200}, "--max-tokens", 1),
98 + ({"enable_prefix_caching": False}, "--no-enable-prefix-caching", 0),
99 + ],
100 + )
101 + def test_replace_existing_parameter(self, input_dict, expected_key, key_idx_offset):
102 + """Test that existing parameters in cmd are replaced."""
103 + item = ParameterSweepItem.from_record(input_dict)
104 +
105 + if key_idx_offset == 1:
106 + # Key-value pair
107 + cmd = item.apply_to_cmd(["vllm", "serve", "--max-tokens", "100", "model"])
108 + assert expected_key in cmd
109 + idx = cmd.index(expected_key)
110 + assert cmd[idx + 1] == "200"
111 + assert "100" not in cmd
112 + else:
113 + # Boolean flag
114 + cmd = item.apply_to_cmd(
115 + ["vllm", "serve", "--enable-prefix-caching", "model"]
116 + )
117 + assert expected_key in cmd
118 + assert "--enable-prefix-caching" not in cmd
119 +
120 +
121 +class TestParameterSweep:
122 + """Test ParameterSweep functionality."""
123 +
124 + def test_from_records_list(self):
125 + """Test creating ParameterSweep from a list of records."""
126 + records = [
127 + {"max_tokens": 100, "temperature": 0.7},
128 + {"max_tokens": 200, "temperature": 0.9},
129 + ]
130 + sweep = ParameterSweep.from_records(records)
131 + assert len(sweep) == 2
132 + assert sweep[0]["max_tokens"] == 100
133 + assert sweep[1]["max_tokens"] == 200
134 +
135 + def test_read_from_dict(self):
136 + """Test creating ParameterSweep from a dict format."""
137 + data = {
138 + "experiment1": {"max_tokens": 100, "temperature": 0.7},
139 + "experiment2": {"max_tokens": 200, "temperature": 0.9},
140 + }
141 + sweep = ParameterSweep.read_from_dict(data)
142 + assert len(sweep) == 2
143 +
144 + # Check that items have the _benchmark_name field
145 + names = {item["_benchmark_name"] for item in sweep}
146 + assert names == {"experiment1", "experiment2"}
147 +
148 + # Check that parameters are preserved
149 + for item in sweep:
150 + if item["_benchmark_name"] == "experiment1":
151 + assert item["max_tokens"] == 100
152 + assert item["temperature"] == 0.7
153 + elif item["_benchmark_name"] == "experiment2":
154 + assert item["max_tokens"] == 200
155 + assert item["temperature"] == 0.9
156 +
157 + def test_read_json_list_format(self):
158 + """Test reading JSON file with list format."""
159 + records = [
160 + {"max_tokens": 100, "temperature": 0.7},
161 + {"max_tokens": 200, "temperature": 0.9},
162 + ]
163 +
164 + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
165 + json.dump(records, f)
166 + temp_path = Path(f.name)
167 +
168 + try:
169 + sweep = ParameterSweep.read_json(temp_path)
170 + assert len(sweep) == 2
171 + assert sweep[0]["max_tokens"] == 100
172 + assert sweep[1]["max_tokens"] == 200
173 + finally:
174 + temp_path.unlink()
175 +
176 + def test_read_json_dict_format(self):
177 + """Test reading JSON file with dict format."""
178 + data = {
179 + "experiment1": {"max_tokens": 100, "temperature": 0.7},
180 + "experiment2": {"max_tokens": 200, "temperature": 0.9},
181 + }
182 +
183 + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
184 + json.dump(data, f)
185 + temp_path = Path(f.name)
186 +
187 + try:
188 + sweep = ParameterSweep.read_json(temp_path)
189 + assert len(sweep) == 2
190 +
191 + # Check that items have the _benchmark_name field
192 + names = {item["_benchmark_name"] for item in sweep}
193 + assert names == {"experiment1", "experiment2"}
194 + finally:
195 + temp_path.unlink()
196 +
197 + def test_unique_benchmark_names_validation(self):
198 + """Test that duplicate _benchmark_name values raise an error."""
199 + # Test with duplicate names in list format
200 + records = [
201 + {"_benchmark_name": "exp1", "max_tokens": 100},
202 + {"_benchmark_name": "exp1", "max_tokens": 200},
203 + ]
204 +
205 + with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
206 + ParameterSweep.from_records(records)
207 +
208 + def test_unique_benchmark_names_multiple_duplicates(self):
209 + """Test validation with multiple duplicate names."""
210 + records = [
211 + {"_benchmark_name": "exp1", "max_tokens": 100},
212 + {"_benchmark_name": "exp1", "max_tokens": 200},
213 + {"_benchmark_name": "exp2", "max_tokens": 300},
214 + {"_benchmark_name": "exp2", "max_tokens": 400},
215 + ]
216 +
217 + with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
218 + ParameterSweep.from_records(records)
219 +
220 + def test_no_benchmark_names_allowed(self):
221 + """Test that records without _benchmark_name are allowed."""
222 + records = [
223 + {"max_tokens": 100, "temperature": 0.7},
224 + {"max_tokens": 200, "temperature": 0.9},
225 + ]
226 + sweep = ParameterSweep.from_records(records)
227 + assert len(sweep) == 2
228 +
229 + def test_mixed_benchmark_names_allowed(self):
230 + """Test that mixing records with and without _benchmark_name is allowed."""
231 + records = [
232 + {"_benchmark_name": "exp1", "max_tokens": 100},
233 + {"max_tokens": 200, "temperature": 0.9},
234 + ]
235 + sweep = ParameterSweep.from_records(records)
236 + assert len(sweep) == 2
237 +
238 +
239 +class TestParameterSweepItemKeyNormalization:
240 + """Test key normalization in ParameterSweepItem."""
241 +
242 + def test_underscore_to_hyphen_conversion(self):
243 + """Test that underscores are converted to hyphens in CLI."""
244 + item = ParameterSweepItem.from_record({"max_tokens": 100})
245 + cmd = item.apply_to_cmd(["vllm", "serve"])
246 + assert "--max-tokens" in cmd
247 +
248 + def test_nested_key_preserves_suffix(self):
249 + """Test that nested keys preserve the suffix format."""
250 + # The suffix after the dot should preserve underscores
251 + item = ParameterSweepItem.from_record(
252 + {"compilation_config.some_nested_param": "value"}
253 + )
254 + cmd = item.apply_to_cmd(["vllm", "serve"])
255 + # The prefix (compilation_config) gets converted to hyphens,
256 + # but the suffix (some_nested_param) is preserved
257 + assert any("compilation-config.some_nested_param" in arg for arg in cmd)
tests/benchmarks/test_plot_filters.py new
+171
@@ -0,0 +1,171 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import pandas as pd
4 +import pytest
5 +
6 +from vllm.benchmarks.sweep.plot import (
7 + PlotEqualTo,
8 + PlotFilterBase,
9 + PlotFilters,
10 + PlotGreaterThan,
11 + PlotGreaterThanOrEqualTo,
12 + PlotLessThan,
13 + PlotLessThanOrEqualTo,
14 + PlotNotEqualTo,
15 +)
16 +
17 +
18 +class TestPlotFilters:
19 + """Test PlotFilter functionality including 'inf' edge case."""
20 +
21 + def setup_method(self):
22 + """Create sample DataFrames for testing."""
23 + # DataFrame with numeric values
24 + self.df_numeric = pd.DataFrame(
25 + {
26 + "request_rate": [1.0, 5.0, 10.0, 50.0, 100.0],
27 + "value": [10, 20, 30, 40, 50],
28 + }
29 + )
30 +
31 + # DataFrame with float('inf') - note: string "inf" values are coerced
32 + # to float when loading data, so we only test with float('inf')
33 + self.df_inf_float = pd.DataFrame(
34 + {
35 + "request_rate": [1.0, 5.0, 10.0, float("inf"), float("inf")],
36 + "value": [10, 20, 30, 40, 50],
37 + }
38 + )
39 +
40 + @pytest.mark.parametrize(
41 + "target,expected_count",
42 + [
43 + ("5.0", 1),
44 + ("10.0", 1),
45 + ("1.0", 1),
46 + ],
47 + )
48 + def test_equal_to_numeric(self, target, expected_count):
49 + """Test PlotEqualTo with numeric values."""
50 + filter_obj = PlotEqualTo("request_rate", target)
51 + result = filter_obj.apply(self.df_numeric)
52 + assert len(result) == expected_count
53 +
54 + def test_equal_to_inf_float(self):
55 + """Test PlotEqualTo with float('inf')."""
56 + filter_obj = PlotEqualTo("request_rate", "inf")
57 + result = filter_obj.apply(self.df_inf_float)
58 + # Should match both float('inf') entries because float('inf') == float('inf')
59 + assert len(result) == 2
60 +
61 + @pytest.mark.parametrize(
62 + "target,expected_count",
63 + [
64 + ("5.0", 4), # All except 5.0
65 + ("1.0", 4), # All except 1.0
66 + ],
67 + )
68 + def test_not_equal_to_numeric(self, target, expected_count):
69 + """Test PlotNotEqualTo with numeric values."""
70 + filter_obj = PlotNotEqualTo("request_rate", target)
71 + result = filter_obj.apply(self.df_numeric)
72 + assert len(result) == expected_count
73 +
74 + def test_not_equal_to_inf_float(self):
75 + """Test PlotNotEqualTo with float('inf')."""
76 + filter_obj = PlotNotEqualTo("request_rate", "inf")
77 + result = filter_obj.apply(self.df_inf_float)
78 + # Should exclude float('inf') entries
79 + assert len(result) == 3
80 +
81 + @pytest.mark.parametrize(
82 + "target,expected_count",
83 + [
84 + ("10.0", 2), # 1.0, 5.0
85 + ("50.0", 3), # 1.0, 5.0, 10.0
86 + ("5.0", 1), # 1.0
87 + ],
88 + )
89 + def test_less_than(self, target, expected_count):
90 + """Test PlotLessThan with numeric values."""
91 + filter_obj = PlotLessThan("request_rate", target)
92 + result = filter_obj.apply(self.df_numeric)
93 + assert len(result) == expected_count
94 +
95 + @pytest.mark.parametrize(
96 + "target,expected_count",
97 + [
98 + ("10.0", 3), # 1.0, 5.0, 10.0
99 + ("5.0", 2), # 1.0, 5.0
100 + ],
101 + )
102 + def test_less_than_or_equal_to(self, target, expected_count):
103 + """Test PlotLessThanOrEqualTo with numeric values."""
104 + filter_obj = PlotLessThanOrEqualTo("request_rate", target)
105 + result = filter_obj.apply(self.df_numeric)
106 + assert len(result) == expected_count
107 +
108 + @pytest.mark.parametrize(
109 + "target,expected_count",
110 + [
111 + ("10.0", 2), # 50.0, 100.0
112 + ("5.0", 3), # 10.0, 50.0, 100.0
113 + ],
114 + )
115 + def test_greater_than(self, target, expected_count):
116 + """Test PlotGreaterThan with numeric values."""
117 + filter_obj = PlotGreaterThan("request_rate", target)
118 + result = filter_obj.apply(self.df_numeric)
119 + assert len(result) == expected_count
120 +
121 + @pytest.mark.parametrize(
122 + "target,expected_count",
123 + [
124 + ("10.0", 3), # 10.0, 50.0, 100.0
125 + ("5.0", 4), # 5.0, 10.0, 50.0, 100.0
126 + ],
127 + )
128 + def test_greater_than_or_equal_to(self, target, expected_count):
129 + """Test PlotGreaterThanOrEqualTo with numeric values."""
130 + filter_obj = PlotGreaterThanOrEqualTo("request_rate", target)
131 + result = filter_obj.apply(self.df_numeric)
132 + assert len(result) == expected_count
133 +
134 + @pytest.mark.parametrize(
135 + "filter_str,expected_var,expected_target,expected_type",
136 + [
137 + ("request_rate==5.0", "request_rate", "5.0", PlotEqualTo),
138 + ("request_rate!=10.0", "request_rate", "10.0", PlotNotEqualTo),
139 + ("request_rate<50.0", "request_rate", "50.0", PlotLessThan),
140 + ("request_rate<=50.0", "request_rate", "50.0", PlotLessThanOrEqualTo),
141 + ("request_rate>10.0", "request_rate", "10.0", PlotGreaterThan),
142 + ("request_rate>=10.0", "request_rate", "10.0", PlotGreaterThanOrEqualTo),
143 + ("request_rate==inf", "request_rate", "inf", PlotEqualTo),
144 + ("request_rate!='inf'", "request_rate", "inf", PlotNotEqualTo),
145 + ],
146 + )
147 + def test_parse_str(self, filter_str, expected_var, expected_target, expected_type):
148 + """Test parsing filter strings."""
149 + filter_obj = PlotFilterBase.parse_str(filter_str)
150 + assert isinstance(filter_obj, expected_type)
151 + assert filter_obj.var == expected_var
152 + assert filter_obj.target == expected_target
153 +
154 + def test_parse_str_inf_edge_case(self):
155 + """Test parsing 'inf' string in filter."""
156 + filter_obj = PlotFilterBase.parse_str("request_rate==inf")
157 + assert isinstance(filter_obj, PlotEqualTo)
158 + assert filter_obj.var == "request_rate"
159 + assert filter_obj.target == "inf"
160 +
161 + def test_parse_multiple_filters(self):
162 + """Test parsing multiple filters."""
163 + filters = PlotFilters.parse_str("request_rate>5.0,value<=40")
164 + assert len(filters) == 2
165 + assert isinstance(filters[0], PlotGreaterThan)
166 + assert isinstance(filters[1], PlotLessThanOrEqualTo)
167 +
168 + def test_parse_empty_filter(self):
169 + """Test parsing empty filter string."""
170 + filters = PlotFilters.parse_str("")
171 + assert len(filters) == 0
vllm/benchmarks/sweep/param_sweep.py
+76 -9
@@ -9,8 +9,26 @@ class ParameterSweep(list["ParameterSweepItem"]):
9 @classmethod
10 def read_json(cls, filepath: os.PathLike):
11 with open(filepath, "rb") as f:
12 - records = json.load(f)
12 + data = json.load(f)
13
14 + # Support both list and dict formats
15 + if isinstance(data, dict):
16 + return cls.read_from_dict(data)
17 +
18 + return cls.from_records(data)
19 +
20 + @classmethod
21 + def read_from_dict(cls, data: dict[str, dict[str, object]]):
22 + """
23 + Read parameter sweep from a dict format where keys are names.
24 +
25 + Example:
26 + {
27 + "experiment1": {"max_tokens": 100, "temperature": 0.7},
28 + "experiment2": {"max_tokens": 200, "temperature": 0.9}
29 + }
30 + """
31 + records = [{"_benchmark_name": name, **params} for name, params in data.items()]
32 return cls.from_records(records)
33
34 @classmethod
@@ -21,6 +39,15 @@ class ParameterSweep(list["ParameterSweepItem"]):
39 f"but found type: {type(records)}"
40 )
41
42 + # Validate that all _benchmark_name values are unique if provided
43 + names = [r["_benchmark_name"] for r in records if "_benchmark_name" in r]
44 + if names and len(names) != len(set(names)):
45 + duplicates = [name for name in names if names.count(name) > 1]
46 + raise ValueError(
47 + f"Duplicate _benchmark_name values found: {set(duplicates)}. "
48 + f"All _benchmark_name values must be unique."
49 + )
50 +
51 return cls(ParameterSweepItem.from_record(record) for record in records)
52
53
@@ -38,6 +65,18 @@ class ParameterSweepItem(dict[str, object]):
65 def __or__(self, other: dict[str, Any]):
66 return type(self)(super().__or__(other))
67
68 + @property
69 + def name(self) -> str:
70 + """
71 + Get the name for this parameter sweep item.
72 +
73 + Returns the '_benchmark_name' field if present, otherwise returns a text
74 + representation of all parameters.
75 + """
76 + if "_benchmark_name" in self:
77 + return self["_benchmark_name"]
78 + return self.as_text(sep="-")
79 +
80 # In JSON, we prefer "_"
81 def _iter_param_key_candidates(self, param_key: str):
82 # Inner config arguments are not converted by the CLI
@@ -63,29 +102,57 @@ class ParameterSweepItem(dict[str, object]):
102 def has_param(self, param_key: str) -> bool:
103 return any(k in self for k in self._iter_param_key_candidates(param_key))
104
105 + def _normalize_cmd_kv_pair(self, k: str, v: object) -> list[str]:
106 + """
107 + Normalize a key-value pair into command-line arguments.
108 +
109 + Returns a list containing either:
110 + - A single element for boolean flags (e.g., ['--flag'] or ['--flag=true'])
111 + - Two elements for key-value pairs (e.g., ['--key', 'value'])
112 + """
113 + if isinstance(v, bool):
114 + # For nested params (containing "."), use =true/false syntax
115 + if "." in k:
116 + return [f"{self._normalize_cmd_key(k)}={'true' if v else 'false'}"]
117 + else:
118 + return [self._normalize_cmd_key(k if v else "no-" + k)]
119 + else:
120 + return [self._normalize_cmd_key(k), str(v)]
121 +
122 def apply_to_cmd(self, cmd: list[str]) -> list[str]:
123 cmd = list(cmd)
124
125 for k, v in self.items():
126 + # Skip the '_benchmark_name' field, not a parameter
127 + if k == "_benchmark_name":
128 + continue
129 +
130 + # Serialize dict values as JSON
131 + if isinstance(v, dict):
132 + v = json.dumps(v)
133 +
134 for k_candidate in self._iter_cmd_key_candidates(k):
135 try:
136 k_idx = cmd.index(k_candidate)
137
74 - if isinstance(v, bool):
75 - cmd[k_idx] = self._normalize_cmd_key(k if v else "no-" + k)
138 + # Replace existing parameter
139 + normalized = self._normalize_cmd_kv_pair(k, v)
140 + if len(normalized) == 1:
141 + # Boolean flag
142 + cmd[k_idx] = normalized[0]
143 else:
77 - cmd[k_idx + 1] = str(v)
144 + # Key-value pair
145 + cmd[k_idx] = normalized[0]
146 + cmd[k_idx + 1] = normalized[1]
147
148 break
149 except ValueError:
150 continue
151 else:
83 - if isinstance(v, bool):
84 - cmd.append(self._normalize_cmd_key(k if v else "no-" + k))
85 - else:
86 - cmd.extend([self._normalize_cmd_key(k), str(v)])
152 + # Add new parameter
153 + cmd.extend(self._normalize_cmd_kv_pair(k, v))
154
155 return cmd
156
157 def as_text(self, sep: str = ", ") -> str:
91 - return sep.join(f"{k}={v}" for k, v in self.items())
158 + return sep.join(f"{k}={v}" for k, v in self.items() if k != "_benchmark_name")
vllm/benchmarks/sweep/plot.py
+102 -7
@@ -65,6 +65,18 @@ class PlotEqualTo(PlotFilterBase):
65 return df[df[self.var] == target]
66
67
68 +@dataclass
69 +class PlotNotEqualTo(PlotFilterBase):
70 + @override
71 + def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
72 + try:
73 + target = float(self.target)
74 + except ValueError:
75 + target = self.target
76 +
77 + return df[df[self.var] != target]
78 +
79 +
80 @dataclass
81 class PlotLessThan(PlotFilterBase):
82 @override
@@ -96,6 +108,7 @@ class PlotGreaterThanOrEqualTo(PlotFilterBase):
108 # NOTE: The ordering is important! Match longer op_keys first
109 PLOT_FILTERS: dict[str, type[PlotFilterBase]] = {
110 "==": PlotEqualTo,
111 + "!=": PlotNotEqualTo,
112 "<=": PlotLessThanOrEqualTo,
113 ">=": PlotGreaterThanOrEqualTo,
114 "<": PlotLessThan,
@@ -167,6 +180,27 @@ def _json_load_bytes(path: Path) -> list[dict[str, object]]:
180 return json.load(f)
181
182
183 +def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, object]]:
184 + """
185 + Convert string values "inf", "-inf", and "nan" to their float equivalents.
186 +
187 + This handles the case where JSON serialization represents inf/nan as strings.
188 + """
189 + converted_data = []
190 + for record in data:
191 + converted_record = {}
192 + for key, value in record.items():
193 + if isinstance(value, str):
194 + if value in ["inf", "-inf", "nan"]:
195 + converted_record[key] = float(value)
196 + else:
197 + converted_record[key] = value
198 + else:
199 + converted_record[key] = value
200 + converted_data.append(converted_record)
201 + return converted_data
202 +
203 +
204 def _get_metric(run_data: dict[str, object], metric_key: str):
205 try:
206 return run_data[metric_key]
@@ -178,12 +212,15 @@ def _get_group(run_data: dict[str, object], group_keys: list[str]):
212 return tuple((k, str(_get_metric(run_data, k))) for k in group_keys)
213
214
181 -def _get_fig_path(fig_dir: Path, group: tuple[tuple[str, str], ...]):
215 +def _get_fig_path(fig_dir: Path, group: tuple[tuple[str, str], ...], fig_name: str):
216 parts = list[str]()
217 +
218 + # Start with figure name (always provided, defaults to "FIGURE")
219 + parts.append(fig_name)
220 +
221 + # Always append group data if present
222 if group:
184 - parts.extend(("FIGURE-", *(f"{k}={v}" for k, v in group)))
185 - else:
186 - parts.append("figure")
223 + parts.extend(f"{k}={v}" for k, v in group)
224
225 return fig_dir / sanitize_filename("-".join(parts) + ".png")
226
@@ -217,6 +254,10 @@ def _plot_fig(
254 scale_x: str | None,
255 scale_y: str | None,
256 dry_run: bool,
257 + fig_name: str,
258 + error_bars: bool,
259 + fig_height: float,
260 + fig_dpi: int,
261 ):
262 fig_group, fig_data = fig_group_data
263
@@ -230,7 +271,7 @@ def _plot_fig(
271 for _, row_data in row_groups
272 )
273
233 - fig_path = _get_fig_path(fig_dir, fig_group)
274 + fig_path = _get_fig_path(fig_dir, fig_group, fig_name)
275
276 print("[BEGIN FIGURE]")
277 print(f"Group: {dict(fig_group)}")
@@ -241,6 +282,8 @@ def _plot_fig(
282 print("[END FIGURE]")
283 return
284
285 + # Convert string "inf", "-inf", and "nan" to their float equivalents
286 + fig_data = _convert_inf_nan_strings(fig_data)
287 df = pd.DataFrame.from_records(fig_data)
288
289 if var_x not in df.columns:
@@ -275,6 +318,10 @@ def _plot_fig(
318 df = filter_by.apply(df)
319 df = bin_by.apply(df)
320
321 + # Sort by curve_by columns alphabetically for consistent legend ordering
322 + if curve_by:
323 + df = df.sort_values(by=curve_by)
324 +
325 df["row_group"] = (
326 pd.concat(
327 [k + "=" + df[k].astype(str) for k in row_by],
@@ -293,7 +340,7 @@ def _plot_fig(
340 else "(All)"
341 )
342
296 - g = sns.FacetGrid(df, row="row_group", col="col_group")
343 + g = sns.FacetGrid(df, row="row_group", col="col_group", height=fig_height)
344
345 if row_by and col_by:
346 g.set_titles("{row_name}\n{col_name}")
@@ -320,6 +367,7 @@ def _plot_fig(
367 style=style,
368 size=size,
369 markers=True,
370 + errorbar="sd" if error_bars else None,
371 )
372
373 g.add_legend(title=hue)
@@ -339,11 +387,12 @@ def _plot_fig(
387 y=var_y,
388 hue="curve_group",
389 markers=True,
390 + errorbar="sd" if error_bars else None,
391 )
392
393 g.add_legend()
394
346 - g.savefig(fig_path)
395 + g.savefig(fig_path, dpi=fig_dpi)
396 plt.close(g.figure)
397
398 print("[END FIGURE]")
@@ -364,6 +413,10 @@ def plot(
413 scale_x: str | None,
414 scale_y: str | None,
415 dry_run: bool,
416 + fig_name: str = "FIGURE",
417 + error_bars: bool = True,
418 + fig_height: float = 6.4,
419 + fig_dpi: int = 300,
420 ):
421 all_data = [
422 run_data
@@ -398,6 +451,10 @@ def plot(
451 scale_x=scale_x,
452 scale_y=scale_y,
453 dry_run=dry_run,
454 + fig_name=fig_name,
455 + error_bars=error_bars,
456 + fig_height=fig_height,
457 + fig_dpi=fig_dpi,
458 ),
459 fig_groups,
460 )
@@ -419,6 +476,10 @@ class SweepPlotArgs:
476 scale_x: str | None
477 scale_y: str | None
478 dry_run: bool
479 + fig_name: str = "FIGURE"
480 + error_bars: bool = True
481 + fig_height: float = 6.4
482 + fig_dpi: int = 300
483
484 parser_name: ClassVar[str] = "plot"
485 parser_help: ClassVar[str] = "Plot performance curves from parameter sweep results."
@@ -448,6 +509,10 @@ class SweepPlotArgs:
509 scale_x=args.scale_x,
510 scale_y=args.scale_y,
511 dry_run=args.dry_run,
512 + fig_name=args.fig_name,
513 + error_bars=not args.no_error_bars,
514 + fig_height=args.fig_height,
515 + fig_dpi=args.fig_dpi,
516 )
517
518 @classmethod
@@ -541,6 +606,32 @@ class SweepPlotArgs:
606 "Currently only accepts string values such as 'log' and 'sqrt'. "
607 "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
608 )
609 + parser.add_argument(
610 + "--fig-name",
611 + type=str,
612 + default="FIGURE",
613 + help="Name prefix for the output figure file. "
614 + "Group data is always appended when present. "
615 + "Default: 'FIGURE'. Example: --fig-name my_performance_plot",
616 + )
617 + parser.add_argument(
618 + "--no-error-bars",
619 + action="store_true",
620 + help="If set, disables error bars on the plot. "
621 + "By default, error bars are shown.",
622 + )
623 + parser.add_argument(
624 + "--fig-height",
625 + type=float,
626 + default=6.4,
627 + help="Height of each subplot in inches. Default: 6.4",
628 + )
629 + parser.add_argument(
630 + "--fig-dpi",
631 + type=int,
632 + default=300,
633 + help="Resolution of the output figure in dots per inch. Default: 300",
634 + )
635 parser.add_argument(
636 "--dry-run",
637 action="store_true",
@@ -566,6 +657,10 @@ def run_main(args: SweepPlotArgs):
657 scale_x=args.scale_x,
658 scale_y=args.scale_y,
659 dry_run=args.dry_run,
660 + fig_name=args.fig_name,
661 + error_bars=args.error_bars,
662 + fig_height=args.fig_height,
663 + fig_dpi=args.fig_dpi,
664 )
665
666
vllm/benchmarks/sweep/serve.py
+8 -6
@@ -138,9 +138,9 @@ def _get_comb_base_path(
138 ):
139 parts = list[str]()
140 if serve_comb:
141 - parts.extend(("SERVE-", serve_comb.as_text(sep="-")))
141 + parts.extend(("SERVE-", serve_comb.name))
142 if bench_comb:
143 - parts.extend(("BENCH-", bench_comb.as_text(sep="-")))
143 + parts.extend(("BENCH-", bench_comb.name))
144
145 return output_dir / sanitize_filename("-".join(parts))
146
@@ -345,8 +345,9 @@ class SweepServeArgs:
345 "--serve-params",
346 type=str,
347 default=None,
348 - help="Path to JSON file containing a list of parameter combinations "
349 - "for the `vllm serve` command. "
348 + help="Path to JSON file containing parameter combinations "
349 + "for the `vllm serve` command. Can be either a list of dicts or a dict "
350 + "where keys are benchmark names. "
351 "If both `serve_params` and `bench_params` are given, "
352 "this script will iterate over their Cartesian product.",
353 )
@@ -354,8 +355,9 @@ class SweepServeArgs:
355 "--bench-params",
356 type=str,
357 default=None,
357 - help="Path to JSON file containing a list of parameter combinations "
358 - "for the `vllm bench serve` command. "
358 + help="Path to JSON file containing parameter combinations "
359 + "for the `vllm bench serve` command. Can be either a list of dicts or "
360 + "a dict where keys are benchmark names. "
361 "If both `serve_params` and `bench_params` are given, "
362 "this script will iterate over their Cartesian product.",
363 )