A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Benchmark][Bug] Fix multiple bugs in bench and add args to spec_decode offline (#20083)
Ekagra Ranjan committed
Jun 26, 2025 at 01:06 UTC
9502c38138a03669c4d54225336553db70ad799d
4 files changed
+28
-11
benchmarks/benchmark_dataset.py
+2
-1
@@ -349,8 +349,9 @@ class RandomDataset(BenchmarkDataset):
349
# [1650, 939, 486] -> ['Ġcall', 'sh', 'ere']
350
# To avoid uncontrolled change of the prompt length,
351
# the encoded sequence is truncated before being decode again.
352
+ total_input_len = prefix_len + int(input_lens[i])
353
re_encoded_sequence = tokenizer.encode(prompt, add_special_tokens=False)[
353
- : input_lens[i]
354
+ :total_input_len
355
]
356
prompt = tokenizer.decode(re_encoded_sequence)
357
total_input_len = len(re_encoded_sequence)
examples/offline_inference/spec_decode.py
+13
-7
@@ -39,6 +39,9 @@ def parse_args():
39
parser.add_argument("--top-k", type=int, default=-1)
40
parser.add_argument("--print-output", action="store_true")
41
parser.add_argument("--output-len", type=int, default=256)
42
+ parser.add_argument("--model-dir", type=str, default=None)
43
+ parser.add_argument("--eagle-dir", type=str, default=None)
44
+ parser.add_argument("--max-model-len", type=int, default=2048)
45
return parser.parse_args()
46
47
@@ -46,9 +49,10 @@ def main():
49
args = parse_args()
50
args.endpoint_type = "openai-chat"
51
49
- model_dir = "meta-llama/Llama-3.1-8B-Instruct"
52
+ model_dir = args.model_dir
53
+ if args.model_dir is None:
54
+ model_dir = "meta-llama/Llama-3.1-8B-Instruct"
55
tokenizer = AutoTokenizer.from_pretrained(model_dir)
51
- max_model_len = 2048
56
57
prompts = get_samples(args, tokenizer)
58
# add_special_tokens is False to avoid adding bos twice when using chat templates
@@ -57,16 +61,18 @@ def main():
61
]
62
63
if args.method == "eagle" or args.method == "eagle3":
60
- if args.method == "eagle":
64
+ eagle_dir = args.eagle_dir
65
+ if args.method == "eagle" and eagle_dir is None:
66
eagle_dir = "yuhuili/EAGLE-LLaMA3.1-Instruct-8B"
62
- elif args.method == "eagle3":
67
+
68
+ elif args.method == "eagle3" and eagle_dir is None:
69
eagle_dir = "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B"
70
speculative_config = {
71
"method": args.method,
72
"model": eagle_dir,
73
"num_speculative_tokens": args.num_spec_tokens,
74
"draft_tensor_parallel_size": args.draft_tp,
69
- "max_model_len": max_model_len,
75
+ "max_model_len": args.max_model_len,
76
}
77
elif args.method == "ngram":
78
speculative_config = {
@@ -74,7 +80,7 @@ def main():
80
"num_speculative_tokens": args.num_spec_tokens,
81
"prompt_lookup_max": args.prompt_lookup_max,
82
"prompt_lookup_min": args.prompt_lookup_min,
77
- "max_model_len": max_model_len,
83
+ "max_model_len": args.max_model_len,
84
}
85
else:
86
raise ValueError(f"unknown method: {args.method}")
@@ -86,7 +92,7 @@ def main():
92
enable_chunked_prefill=args.enable_chunked_prefill,
93
max_num_batched_tokens=args.max_num_batched_tokens,
94
enforce_eager=args.enforce_eager,
89
- max_model_len=max_model_len,
95
+ max_model_len=args.max_model_len,
96
max_num_seqs=args.max_num_seqs,
97
gpu_memory_utilization=0.8,
98
speculative_config=speculative_config,
vllm/benchmarks/datasets.py
+7
-3
@@ -320,6 +320,8 @@ class RandomDataset(BenchmarkDataset):
320
**kwargs,
321
) -> None:
322
super().__init__(**kwargs)
323
+ random.seed(self.random_seed)
324
+ np.random.seed(self.random_seed)
325
326
def sample(
327
self,
@@ -376,10 +378,11 @@ class RandomDataset(BenchmarkDataset):
378
# [1650, 939, 486] -> ['Ġcall', 'sh', 'ere']
379
# To avoid uncontrolled change of the prompt length,
380
# the encoded sequence is truncated before being decode again.
381
+ total_input_len = prefix_len + int(input_lens[i])
382
re_encoded_sequence = tokenizer.encode(
380
- prompt, add_special_tokens=False)[:input_lens[i]]
383
+ prompt, add_special_tokens=False)[:total_input_len]
384
prompt = tokenizer.decode(re_encoded_sequence)
382
- total_input_len = prefix_len + int(input_lens[i])
385
+ total_input_len = len(re_encoded_sequence)
386
requests.append(
387
SampleRequest(
388
prompt=prompt,
@@ -692,7 +695,8 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
695
dataset_path=args.dataset_path).
696
sample(tokenizer=tokenizer, num_requests=args.num_prompts),
697
"random":
695
- lambda: RandomDataset(dataset_path=args.dataset_path).sample(
698
+ lambda: RandomDataset(random_seed=args.seed,
699
+ dataset_path=args.dataset_path).sample(
700
tokenizer=tokenizer,
701
num_requests=args.num_prompts,
702
prefix_len=args.random_prefix_len,
vllm/benchmarks/serve.py
+6
@@ -631,6 +631,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
631
help="The label (prefix) of the benchmark results. If not specified, "
632
"the endpoint type will be used as the label.",
633
)
634
+ parser.add_argument(
635
+ "--backend",
636
+ type=str,
637
+ default="vllm",
638
+ choices=list(ASYNC_REQUEST_FUNCS.keys()),
639
+ )
640
parser.add_argument(
641
"--base-url",
642
type=str,