@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 94bf9ae4e9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Fix sampling from sonnet for long context case (#9235)

Andy Dai committed Oct 10, 2024 at 17:33 UTC 94bf9ae4e9b8199636668ccbe4dabcdc3b9e5ae6
1 file changed +4 -4
benchmarks/benchmark_serving.py
+4 -4
@@ -176,9 +176,9 @@ def sample_sonnet_requests(
176 # Sample the rest of lines per request.
177 sampled_requests: List[Tuple[str, int, int]] = []
178 for _ in range(num_requests):
179 - sampled_lines = "".join(
180 - prefix_lines +
181 - random.sample(poem_lines, num_input_lines - num_prefix_lines))
179 + num_lines_needed = num_input_lines - num_prefix_lines
180 + sampled_lines = "".join(prefix_lines +
181 + random.choices(poem_lines, k=num_lines_needed))
182
183 prompt = f"{base_prompt}{sampled_lines}"
184 message = [
@@ -536,7 +536,7 @@ async def benchmark(
536 # E.g., "Time to First Token"
537 metric_header: str,
538 ):
539 - # This function print and add statistics of the specified
539 + # This function prints and adds statistics of the specified
540 # metric.
541 if metric_attribute_name not in selected_percentile_metrics:
542 return