A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
Fix range_ratio Bug in RandomDataset (#16126)
Signed-off-by: jadewang21 <jadewangcn@outlook.com>
WWW committed
Apr 11, 2025 at 06:31 UTC
268c3250787729198a0d86378a36e81d4606a2af
3 files changed
+42
-20
benchmarks/benchmark_dataset.py
+18
-5
@@ -288,7 +288,7 @@ def process_image(image: Any) -> Mapping[str, Any]:
288
class RandomDataset(BenchmarkDataset):
289
# Default values copied from benchmark_serving.py for the random dataset.
290
DEFAULT_PREFIX_LEN = 0
291
- DEFAULT_RANGE_RATIO = 1.0
291
+ DEFAULT_RANGE_RATIO = 0.0
292
DEFAULT_INPUT_LEN = 1024
293
DEFAULT_OUTPUT_LEN = 128
294
@@ -308,19 +308,32 @@ class RandomDataset(BenchmarkDataset):
308
output_len: int = DEFAULT_OUTPUT_LEN,
309
**kwargs,
310
) -> list[SampleRequest]:
311
+ # Enforce range_ratio < 1
312
+ assert range_ratio < 1.0, (
313
+ "random_range_ratio must be < 1.0 to ensure a valid sampling range"
314
+ )
315
+
316
vocab_size = tokenizer.vocab_size
317
318
prefix_token_ids = (np.random.randint(
319
0, vocab_size, size=prefix_len).tolist() if prefix_len > 0 else [])
320
316
- input_low = int(input_len * range_ratio)
317
- output_low = int(output_len * range_ratio)
321
+ # New sampling logic: [X * (1 - b), X * (1 + b)]
322
+ input_low = int(input_len * (1 - range_ratio))
323
+ input_high = int(input_len * (1 + range_ratio))
324
+ output_low = int(output_len * (1 - range_ratio))
325
+ output_high = int(output_len * (1 + range_ratio))
326
+
327
+ # Add logging for debugging
328
+ logger.info("Sampling input_len from [%s, %s]", input_low, input_high)
329
+ logger.info("Sampling output_len from [%s, %s]", output_low,
330
+ output_high)
331
332
input_lens = np.random.randint(input_low,
320
- input_len + 1,
333
+ input_high + 1,
334
size=num_requests)
335
output_lens = np.random.randint(output_low,
323
- output_len + 1,
336
+ output_high + 1,
337
size=num_requests)
338
offsets = np.random.randint(0, vocab_size, size=num_requests)
339
benchmarks/benchmark_serving.py
+12
-7
@@ -996,18 +996,23 @@ if __name__ == "__main__":
996
random_group.add_argument(
997
"--random-range-ratio",
998
type=float,
999
- default=1.0,
1000
- help="Range of sampled ratio of input/output length, "
1001
- "used only for random sampling.",
999
+ default=0.0,
1000
+ help="Range ratio for sampling input/output length, "
1001
+ "used only for random sampling. Must be in the range [0, 1) to define "
1002
+ "a symmetric sampling range"
1003
+ "[length * (1 - range_ratio), length * (1 + range_ratio)].",
1004
)
1005
random_group.add_argument(
1006
"--random-prefix-len",
1007
type=int,
1008
default=0,
1007
- help="Number of fixed prefix tokens before random "
1008
- " context. The length range of context in a random "
1009
- " request is [random-prefix-len, "
1010
- " random-prefix-len + random-prefix-len * random-range-ratio).")
1009
+ help=("Number of fixed prefix tokens before the random context "
1010
+ "in a request. "
1011
+ "The total input length is the sum of `random-prefix-len` and "
1012
+ "a random "
1013
+ "context length sampled from [input_len * (1 - range_ratio), "
1014
+ "input_len * (1 + range_ratio)]."),
1015
+ )
1016
1017
hf_group = parser.add_argument_group("hf dataset options")
1018
hf_group.add_argument("--hf-subset",
benchmarks/benchmark_throughput.py
+12
-8
@@ -594,18 +594,22 @@ if __name__ == "__main__":
594
default=None,
595
help="Path to the lora adapters to use. This can be an absolute path, "
596
"a relative path, or a Hugging Face model identifier.")
597
- parser.add_argument("--prefix-len",
598
- type=int,
599
- default=None,
600
- help="Number of prefix tokens per request."
601
- "This is for the RandomDataset and SonnetDataset")
597
+ parser.add_argument(
598
+ "--prefix-len",
599
+ type=int,
600
+ default=0,
601
+ help="Number of fixed prefix tokens before the random "
602
+ "context in a request (default: 0).",
603
+ )
604
# random dataset
605
parser.add_argument(
606
"--random-range-ratio",
607
type=float,
606
- default=None,
607
- help="Range of sampled ratio of input/output length, "
608
- "used only for RandomDataSet.",
608
+ default=0.0,
609
+ help="Range ratio for sampling input/output length, "
610
+ "used only for RandomDataset. Must be in the range [0, 1) to define "
611
+ "a symmetric sampling range "
612
+ "[length * (1 - range_ratio), length * (1 + range_ratio)].",
613
)
614
615
# hf dtaset