A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 13:15 UTC
Multi turn benchmark progress bar for synthetic conversation generation (#28394)
Signed-off-by: Ido Segev <idos@pliops.com>
Ido Segev committed
Nov 11, 2025 at 13:06 UTC
d381eb967f171ea8824357075b15bf2895619609
2 files changed
+17
-4
benchmarks/multi_turn/bench_dataset.py
+15
-3
@@ -11,6 +11,7 @@ from bench_utils import (
11
Color,
12
logger,
13
)
14
+from tqdm import tqdm
15
from transformers import AutoTokenizer # type: ignore
16
17
# Conversation ID is a string (e.g: "UzTK34D")
@@ -417,6 +418,10 @@ def generate_conversations(
418
data = file.read()
419
tokens_in_file = tokenizer.encode(data, add_special_tokens=False)
420
list_of_tokens.extend(tokens_in_file)
421
+ logger.info(
422
+ f"Loaded {len(tokens_in_file)} tokens from file {filename}, "
423
+ f"total tokens so far: {len(list_of_tokens)}"
424
+ )
425
426
conversations: ConversationsMap = {}
427
conv_id = 0
@@ -449,18 +454,25 @@ def generate_conversations(
454
)
455
base_offset += common_prefix_tokens
456
452
- for conv_id in range(args.num_conversations):
457
+ for conv_id in tqdm(
458
+ range(args.num_conversations),
459
+ total=args.num_conversations,
460
+ desc="Generating conversations",
461
+ unit="conv",
462
+ ):
463
# Generate a single conversation
464
messages: MessagesList = []
465
466
nturns = turn_count[conv_id]
467
468
# User prompt token count per turn (with lower limit)
459
- input_token_count: np.ndarray = args.input_num_tokens.sample(nturns)
469
+ input_token_count: np.ndarray = args.input_num_tokens.sample(nturns).astype(int)
470
input_token_count = np.maximum(input_token_count, base_prompt_token_count)
471
472
# Assistant answer token count per turn (with lower limit)
463
- output_token_count: np.ndarray = args.output_num_tokens.sample(nturns)
473
+ output_token_count: np.ndarray = args.output_num_tokens.sample(nturns).astype(
474
+ int
475
+ )
476
output_token_count = np.maximum(output_token_count, 1)
477
478
user_turn = True
benchmarks/multi_turn/requirements.txt
+2
-1
@@ -2,4 +2,5 @@ numpy>=1.24
2
pandas>=2.0.0
3
aiohttp>=3.10
4
transformers>=4.46
5
-xlsxwriter>=3.2.1
\ No newline at end of file
5
+xlsxwriter>=3.2.1
6
+tqdm>=4.66