@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ccb20db8bd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Benchmark serving script used global parameter 'args' in function 'sample_random_requests' (#6428)

Fish committed Jul 15, 2024 at 10:27 UTC ccb20db8bd2abb302a8dc0fbc0901f284df9b3f5
1 file changed +1 -1
benchmarks/benchmark_serving.py
+1 -1
@@ -203,7 +203,7 @@ def sample_random_requests(
203 )
204 offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
205 input_requests = []
206 - for i in range(args.num_prompts):
206 + for i in range(num_prompts):
207 prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
208 for j in range(input_lens[i])])
209 input_requests.append(