A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Add more documentation and improve usability of lognormal dist (benchmark_serving_multi_turn) (#23255)
Signed-off-by: daniels <daniels@pliops.com>
Daniel Serebrenik committed
Sep 17, 2025 at 08:53 UTC
43a62c51be1254a9b2923c7f477af0cfc52d1937
3 files changed
+203
-8
benchmarks/multi_turn/README.md
+101
@@ -55,6 +55,107 @@ output_num_chunks 166.0 99.01 11.80 79.00 90.00 98.00 108.75
55
----------------------------------------------------------------------------------------------------
56
```
57
58
+### JSON configuration file for synthetic conversations generation
59
+
60
+The input flag `--input-file` is used to determine the input conversations for the benchmark.<br/>
61
+When the input is a JSON file with the field `"filetype": "generate_conversations"` the tool will generate synthetic multi-turn (questions and answers) conversations.
62
+
63
+The file `generate_multi_turn.json` is an example file.
64
+
65
+The file must contain the sections `prompt_input` and `prompt_output`.
66
+
67
+The `prompt_input` section must contain `num_turns`, `prefix_num_tokens` and `num_tokens`:
68
+
69
+* `num_turns` - Number of total turns in the conversation (both user & assistant).<br/>
70
+The final value will always be rounded to an even number so each user turn has a reply.
71
+* `prefix_num_tokens` - Tokens added at the start of only the **first user turn** in a conversation (unique per conversation).
72
+* `num_tokens` - Total token length of each **user** message (one turn).
73
+
74
+The `prompt_output` section must contain `num_tokens`:
75
+
76
+* `num_tokens` - Total token length of each **assistant** message (one turn).
77
+
78
+### Random distributions for synthetic conversations generation
79
+
80
+When creating an input JSON file (such as `generate_multi_turn.json`),<br/>
81
+every numeric field (such as `num_turns` or `num_tokens`) requires a distribution.<br/>
82
+The distribution determines how to randomly sample values for the field.
83
+
84
+The available distributions are listed below.
85
+
86
+**Note:** The optional `max` field (for lognormal, zipf, and poisson) can be used to cap sampled values at an upper bound.</br>
87
+Can be used to make sure that the total number of tokens in every request does not exceed `--max-model-len`.
88
+
89
+#### constant
90
+
91
+```json
92
+{
93
+ "distribution": "constant",
94
+ "value": 500
95
+}
96
+```
97
+
98
+* `value` - the fixed integer value (always returns the same number).
99
+
100
+#### uniform
101
+
102
+```json
103
+{
104
+ "distribution": "uniform",
105
+ "min": 12,
106
+ "max": 18
107
+}
108
+```
109
+
110
+* `min` - minimum value (inclusive).
111
+* `max` - maximum value (inclusive), should be equal or larger than min.
112
+
113
+#### lognormal
114
+
115
+```json
116
+{
117
+ "distribution": "lognormal",
118
+ "average": 1000,
119
+ "max": 5000
120
+}
121
+```
122
+
123
+You can parameterize the lognormal distribution in one of two ways:
124
+
125
+Using the average and optional median ratio:
126
+
127
+* `average` - target average value of the distribution.
128
+* `median_ratio` - the ratio of the median to the average; controls the skewness. Must be in the range (0, 1).
129
+
130
+Using the parameters of the underlying normal distribution:
131
+
132
+* `mean` - mean of the underlying normal distribution.
133
+* `sigma` - standard deviation of the underlying normal distribution.
134
+
135
+#### zipf
136
+
137
+```json
138
+{
139
+ "distribution": "zipf",
140
+ "alpha": 1.2,
141
+ "max": 100
142
+}
143
+```
144
+
145
+* `alpha` - skew parameter (> 1). Larger values produce stronger skew toward smaller integers.
146
+
147
+#### poisson
148
+
149
+```json
150
+{
151
+ "distribution": "poisson",
152
+ "alpha": 10,
153
+ "max": 50
154
+}
155
+```
156
+
157
+* `alpha` - expected value (λ). Also the variance of the distribution.
158
+
159
## ShareGPT Conversations
160
161
To run with the ShareGPT data, download the following ShareGPT dataset:
benchmarks/multi_turn/bench_dataset.py
+100
-5
@@ -99,21 +99,105 @@ class PoissonDistribution(Distribution):
99
100
class LognormalDistribution(Distribution):
101
def __init__(
102
- self, mean: float, sigma: float, max_val: Optional[int] = None
102
+ self,
103
+ mean: Optional[float] = None,
104
+ sigma: Optional[float] = None,
105
+ average: Optional[int] = None,
106
+ median_ratio: Optional[float] = None,
107
+ max_val: Optional[int] = None,
108
) -> None:
109
+ self.average = average
110
+ self.median_ratio = median_ratio
111
+ self.max_val = max_val
112
+
113
+ if average is not None:
114
+ if average < 1:
115
+ raise ValueError("Lognormal average must be positive")
116
+
117
+ if mean or sigma:
118
+ raise ValueError(
119
+ "When using lognormal average, you can't provide mean/sigma"
120
+ )
121
+
122
+ if self.median_ratio is None:
123
+ # Default value that provides relatively wide range of values
124
+ self.median_ratio = 0.85
125
+
126
+ # Calculate mean/sigma of np.random.lognormal based on the average
127
+ mean, sigma = self._generate_lognormal_by_median(
128
+ target_average=self.average, median_ratio=self.median_ratio
129
+ )
130
+ else:
131
+ if mean is None or sigma is None:
132
+ raise ValueError(
133
+ "Must provide both mean and sigma if average is not used"
134
+ )
135
+
136
+ if mean <= 0 or sigma < 0:
137
+ raise ValueError(
138
+ "Lognormal mean must be positive and sigma must be non-negative"
139
+ )
140
+
141
+ # Mean and standard deviation of the underlying normal distribution
142
+ # Based on numpy.random.lognormal
143
self.mean = mean
144
self.sigma = sigma
106
- self.max_val = max_val
145
+
146
+ @staticmethod
147
+ def _generate_lognormal_by_median(
148
+ target_average: int, median_ratio: float
149
+ ) -> tuple[float, float]:
150
+ """
151
+ Compute (mu, sigma) for a lognormal distribution given:
152
+ - a target average (mean of the distribution)
153
+ - a ratio of median / mean (controls skewness), assume mean > median
154
+
155
+ Background:
156
+ If Z ~ Normal(mu, sigma^2), then X = exp(Z) ~ LogNormal(mu, sigma).
157
+ * mean(X) = exp(mu + sigma^2 / 2)
158
+ * median(X) = exp(mu)
159
+
160
+ So:
161
+ median / mean = exp(mu) / exp(mu + sigma^2 / 2)
162
+ = exp(-sigma^2 / 2)
163
+
164
+ Rearranging:
165
+ sigma^2 = 2 * ln(mean / median)
166
+ mu = ln(median)
167
+
168
+ This gives a unique (mu, sigma) for any valid mean and median.
169
+ """
170
+ # Check input validity: median must be smaller than mean
171
+ if median_ratio <= 0 or median_ratio >= 1:
172
+ raise ValueError("median_ratio must be in range (0, 1)")
173
+
174
+ target_median = target_average * median_ratio
175
+
176
+ # Solve sigma^2 = 2 * ln(mean / median)
177
+ sigma = np.sqrt(2 * np.log(target_average / target_median))
178
+ mu = np.log(target_median)
179
+
180
+ return mu, sigma
181
182
def sample(self, size: int = 1) -> np.ndarray:
183
samples = np.random.lognormal(mean=self.mean, sigma=self.sigma, size=size)
184
+
185
+ if self.average is not None:
186
+ # Scale to average
187
+ samples *= self.average / samples.mean()
188
+
189
if self.max_val:
190
samples = np.minimum(samples, self.max_val)
191
192
return np.round(samples).astype(int)
193
194
def __repr__(self) -> str:
116
- return f"LognormalDistribution[{self.mean}, {self.sigma}]"
195
+ if self.average:
196
+ return (
197
+ f"LognormalDistribution[{self.average}, "
198
+ f"{self.median_ratio}, {self.max_val}]"
199
+ )
200
+ return f"LognormalDistribution[{self.mean}, {self.sigma}, {self.max_val}]"
201
202
203
class GenConvArgs(NamedTuple):
@@ -173,10 +257,21 @@ def get_random_distribution(
257
return PoissonDistribution(conf["alpha"], max_val=max_val)
258
259
elif distribution == "lognormal":
260
+ max_val = conf.get("max", None)
261
+
262
+ if "average" in conf:
263
+ # Infer lognormal mean/sigma (numpy) from input average
264
+ median_ratio = conf.get("median_ratio", None)
265
+ return LognormalDistribution(
266
+ average=conf["average"], median_ratio=median_ratio, max_val=max_val
267
+ )
268
+
269
+ # Use mean/sigma directly (for full control over the distribution)
270
verify_field_exists(conf, "mean", section, subsection)
271
verify_field_exists(conf, "sigma", section, subsection)
178
- max_val = conf.get("max", None)
179
- return LognormalDistribution(conf["mean"], conf["sigma"], max_val=max_val)
272
+ return LognormalDistribution(
273
+ mean=conf["mean"], sigma=conf["sigma"], max_val=max_val
274
+ )
275
276
elif distribution == "uniform":
277
verify_field_exists(conf, "min", section, subsection)
benchmarks/multi_turn/generate_multi_turn.json
+2
-3
@@ -15,9 +15,8 @@
15
},
16
"prefix_num_tokens": {
17
"distribution": "lognormal",
18
- "mean": 6,
19
- "sigma": 4,
20
- "max": 1500
18
+ "average": 1000,
19
+ "max": 5000
20
},
21
"num_tokens": {
22
"distribution": "uniform",