A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Benchmark serving script used global parameter 'args' in function 'sample_random_requests' (#6428)
Fish committed
Jul 15, 2024 at 10:27 UTC
ccb20db8bd2abb302a8dc0fbc0901f284df9b3f5
1 file changed
+1
-1
benchmarks/benchmark_serving.py
+1
-1
@@ -203,7 +203,7 @@ def sample_random_requests(
203
)
204
offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
205
input_requests = []
206
- for i in range(args.num_prompts):
206
+ for i in range(num_prompts):
207
prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
208
for j in range(input_lens[i])])
209
input_requests.append(