A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[V1][Spec Decode][Ngram] 1.35x gain -> 1.95x gain on InstructCoder with prompt fix (#18971)
Ekagra Ranjan committed
Jun 3, 2025 at 18:26 UTC
135cf55cd1d83cd4e18266e343a59e6d9f87856f
2 files changed
+22
-2
benchmarks/benchmark_dataset.py
+9
-1
@@ -865,7 +865,15 @@ class InstructCoderDataset(HuggingFaceDataset):
865
for item in self.data:
866
if len(sampled_requests) >= num_requests:
867
break
868
- prompt = f"{item['instruction']}:\n{item['input']}"
868
+ prompt = f"{item['input']}\n\n{item['instruction']} Just output \
869
+ the code, do not include any explanation."
870
+
871
+ # apply template
872
+ prompt = tokenizer.apply_chat_template(
873
+ [{"role": "user", "content": prompt}],
874
+ add_generation_prompt=True,
875
+ tokenize=False,
876
+ )
877
prompt_len = len(tokenizer(prompt).input_ids)
878
sampled_requests.append(
879
SampleRequest(
vllm/benchmarks/datasets.py
+13
-1
@@ -880,7 +880,19 @@ class InstructCoderDataset(HuggingFaceDataset):
880
for item in self.data:
881
if len(sampled_requests) >= num_requests:
882
break
883
- prompt = f"{item['instruction']}:\n{item['input']}"
883
+ prompt = f"{item['input']}\n\n{item['instruction']} Just output \
884
+ the code, do not include any explanation."
885
+
886
+ # apply template
887
+ prompt = tokenizer.apply_chat_template(
888
+ [{
889
+ "role": "user",
890
+ "content": prompt
891
+ }],
892
+ add_generation_prompt=True,
893
+ tokenize=False,
894
+ )
895
+
896
prompt_len = len(tokenizer(prompt).input_ids)
897
sampled_requests.append(
898
SampleRequest(