A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Adding int4 and int8 models for CPU benchmarking (#23709)
Adding int4 and int8 models for CPU benchmarking (#23709) Signed-off-by: Tsai, Louie <louie.tsai@intel.com>
Louie Tsai committed
Sep 5, 2025 at 05:08 UTC
006e7a34aeb3e905ca4131a3251fe079f0511e2f
3 files changed
+1066
-13
.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc2.json
+414
-6
@@ -1,6 +1,6 @@
1
[
2
{
3
- "test_name": "serving_llama8B_tp1_sharegpt",
3
+ "test_name": "serving_llama8B_bf16_tp1_sharegpt",
4
"qps_list": ["inf"],
5
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
6
"server_environment_variables": {
@@ -32,7 +32,7 @@
32
}
33
},
34
{
35
- "test_name": "serving_llama8B_tp2_sharegpt",
35
+ "test_name": "serving_llama8B_bf16_tp2_sharegpt",
36
"qps_list": ["inf"],
37
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
38
"server_environment_variables": {
@@ -64,7 +64,7 @@
64
}
65
},
66
{
67
- "test_name": "serving_llama8B_tp4_sharegpt",
67
+ "test_name": "serving_llama8B_bf16_tp4_sharegpt",
68
"qps_list": ["inf"],
69
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
70
"server_environment_variables": {
@@ -96,7 +96,7 @@
96
}
97
},
98
{
99
- "test_name": "serving_llama8B_tp1_random_128_128",
99
+ "test_name": "serving_llama8B_bf16_tp1_random_128_128",
100
"qps_list": ["inf"],
101
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
102
"server_environment_variables": {
@@ -131,7 +131,7 @@
131
}
132
},
133
{
134
- "test_name": "serving_llama8B_tp2_random_128_128",
134
+ "test_name": "serving_llama8B_bf16_tp2_random_128_128",
135
"qps_list": ["inf"],
136
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
137
"server_environment_variables": {
@@ -166,7 +166,7 @@
166
}
167
},
168
{
169
- "test_name": "serving_llama8B_tp4_random_128_128",
169
+ "test_name": "serving_llama8B_bf16_tp4_random_128_128",
170
"qps_list": ["inf"],
171
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
172
"server_environment_variables": {
@@ -198,5 +198,413 @@
198
"random-output-len": 128,
199
"num_prompts": 1000
200
}
201
+ },
202
+ {
203
+ "test_name": "serving_llama8B_int8_tp1_sharegpt",
204
+ "qps_list": ["inf"],
205
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
206
+ "server_environment_variables": {
207
+ "VLLM_RPC_TIMEOUT": 100000,
208
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
209
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
210
+ "VLLM_CPU_SGL_KERNEL": 1,
211
+ "VLLM_CPU_KVCACHE_SPACE": 40
212
+ },
213
+ "server_parameters": {
214
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
215
+ "tensor_parallel_size": 1,
216
+ "dtype": "bfloat16",
217
+ "distributed_executor_backend": "mp",
218
+ "block_size": 128,
219
+ "trust_remote_code": "",
220
+ "disable_log_stats": "",
221
+ "enforce_eager": "",
222
+ "max_num_batched_tokens": 2048,
223
+ "max_num_seqs": 256,
224
+ "load_format": "dummy"
225
+ },
226
+ "client_parameters": {
227
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
228
+ "backend": "vllm",
229
+ "dataset_name": "sharegpt",
230
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
231
+ "num_prompts": 200
232
+ }
233
+ },
234
+ {
235
+ "test_name": "serving_llama8B_int8_tp2_sharegpt",
236
+ "qps_list": ["inf"],
237
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
238
+ "server_environment_variables": {
239
+ "VLLM_RPC_TIMEOUT": 100000,
240
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
241
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
242
+ "VLLM_CPU_SGL_KERNEL": 1,
243
+ "VLLM_CPU_KVCACHE_SPACE": 40
244
+ },
245
+ "server_parameters": {
246
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
247
+ "tensor_parallel_size": 2,
248
+ "dtype": "bfloat16",
249
+ "distributed_executor_backend": "mp",
250
+ "block_size": 128,
251
+ "trust_remote_code": "",
252
+ "disable_log_stats": "",
253
+ "enforce_eager": "",
254
+ "max_num_batched_tokens": 2048,
255
+ "max_num_seqs": 256,
256
+ "load_format": "dummy"
257
+ },
258
+ "client_parameters": {
259
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
260
+ "backend": "vllm",
261
+ "dataset_name": "sharegpt",
262
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
263
+ "num_prompts": 200
264
+ }
265
+ },
266
+ {
267
+ "test_name": "serving_llama8B_int8_tp4_sharegpt",
268
+ "qps_list": ["inf"],
269
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
270
+ "server_environment_variables": {
271
+ "VLLM_RPC_TIMEOUT": 100000,
272
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
273
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
274
+ "VLLM_CPU_SGL_KERNEL": 1,
275
+ "VLLM_CPU_KVCACHE_SPACE": 40
276
+ },
277
+ "server_parameters": {
278
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
279
+ "tensor_parallel_size": 4,
280
+ "dtype": "bfloat16",
281
+ "distributed_executor_backend": "mp",
282
+ "block_size": 128,
283
+ "trust_remote_code": "",
284
+ "disable_log_stats": "",
285
+ "enforce_eager": "",
286
+ "max_num_batched_tokens": 2048,
287
+ "max_num_seqs": 256,
288
+ "load_format": "dummy"
289
+ },
290
+ "client_parameters": {
291
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
292
+ "backend": "vllm",
293
+ "dataset_name": "sharegpt",
294
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
295
+ "num_prompts": 200
296
+ }
297
+ },
298
+ {
299
+ "test_name": "serving_llama8B_int8_tp1_random_128_128",
300
+ "qps_list": ["inf"],
301
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
302
+ "server_environment_variables": {
303
+ "VLLM_RPC_TIMEOUT": 100000,
304
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
305
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
306
+ "VLLM_CPU_SGL_KERNEL": 1,
307
+ "VLLM_CPU_KVCACHE_SPACE": 40
308
+ },
309
+ "server_parameters": {
310
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
311
+ "tensor_parallel_size": 1,
312
+ "dtype": "bfloat16",
313
+ "distributed_executor_backend": "mp",
314
+ "block_size": 128,
315
+ "trust_remote_code": "",
316
+ "enable_chunked_prefill": "",
317
+ "disable_log_stats": "",
318
+ "enforce_eager": "",
319
+ "max_num_batched_tokens": 2048,
320
+ "max_num_seqs": 256,
321
+ "load_format": "dummy"
322
+ },
323
+ "client_parameters": {
324
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
325
+ "backend": "vllm",
326
+ "dataset_name": "random",
327
+ "random-input-len": 128,
328
+ "random-output-len": 128,
329
+ "ignore-eos": "",
330
+ "num_prompts": 1000
331
+ }
332
+ },
333
+ {
334
+ "test_name": "serving_llama8B_int8_tp2_random_128_128",
335
+ "qps_list": ["inf"],
336
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
337
+ "server_environment_variables": {
338
+ "VLLM_RPC_TIMEOUT": 100000,
339
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
340
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
341
+ "VLLM_CPU_SGL_KERNEL": 1,
342
+ "VLLM_CPU_KVCACHE_SPACE": 40
343
+ },
344
+ "server_parameters": {
345
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
346
+ "tensor_parallel_size": 2,
347
+ "dtype": "bfloat16",
348
+ "distributed_executor_backend": "mp",
349
+ "block_size": 128,
350
+ "trust_remote_code": "",
351
+ "enable_chunked_prefill": "",
352
+ "disable_log_stats": "",
353
+ "enforce_eager": "",
354
+ "max_num_batched_tokens": 2048,
355
+ "max_num_seqs": 256,
356
+ "load_format": "dummy"
357
+ },
358
+ "client_parameters": {
359
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
360
+ "backend": "vllm",
361
+ "dataset_name": "random",
362
+ "random-input-len": 128,
363
+ "random-output-len": 128,
364
+ "ignore-eos": "",
365
+ "num_prompts": 1000
366
+ }
367
+ },
368
+ {
369
+ "test_name": "serving_llama8B_int8_tp4_random_128_128",
370
+ "qps_list": ["inf"],
371
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
372
+ "server_environment_variables": {
373
+ "VLLM_RPC_TIMEOUT": 100000,
374
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
375
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
376
+ "VLLM_CPU_SGL_KERNEL": 1,
377
+ "VLLM_CPU_KVCACHE_SPACE": 40
378
+ },
379
+ "server_parameters": {
380
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
381
+ "tensor_parallel_size": 4,
382
+ "dtype": "bfloat16",
383
+ "distributed_executor_backend": "mp",
384
+ "block_size": 128,
385
+ "trust_remote_code": "",
386
+ "enable_chunked_prefill": "",
387
+ "disable_log_stats": "",
388
+ "enforce_eager": "",
389
+ "max_num_batched_tokens": 2048,
390
+ "max_num_seqs": 256,
391
+ "load_format": "dummy"
392
+ },
393
+ "client_parameters": {
394
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
395
+ "backend": "vllm",
396
+ "dataset_name": "random",
397
+ "random-input-len": 128,
398
+ "random-output-len": 128,
399
+ "ignore-eos": "",
400
+ "num_prompts": 1000
401
+ }
402
+ },
403
+ {
404
+ "test_name": "serving_llama8B_int4_tp1_sharegpt",
405
+ "qps_list": ["inf"],
406
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
407
+ "server_environment_variables": {
408
+ "VLLM_RPC_TIMEOUT": 100000,
409
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
410
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
411
+ "VLLM_CPU_SGL_KERNEL": 1,
412
+ "VLLM_CPU_KVCACHE_SPACE": 40
413
+ },
414
+ "server_parameters": {
415
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
416
+ "quantization": "awq",
417
+ "tensor_parallel_size": 1,
418
+ "dtype": "bfloat16",
419
+ "distributed_executor_backend": "mp",
420
+ "block_size": 128,
421
+ "trust_remote_code": "",
422
+ "disable_log_stats": "",
423
+ "enforce_eager": "",
424
+ "max_num_batched_tokens": 2048,
425
+ "max_num_seqs": 256,
426
+ "load_format": "dummy"
427
+ },
428
+ "client_parameters": {
429
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
430
+ "backend": "vllm",
431
+ "dataset_name": "sharegpt",
432
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
433
+ "num_prompts": 200
434
+ }
435
+ },
436
+ {
437
+ "test_name": "serving_llama8B_int4_tp2_sharegpt",
438
+ "qps_list": ["inf"],
439
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
440
+ "server_environment_variables": {
441
+ "VLLM_RPC_TIMEOUT": 100000,
442
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
443
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
444
+ "VLLM_CPU_SGL_KERNEL": 1,
445
+ "VLLM_CPU_KVCACHE_SPACE": 40
446
+ },
447
+ "server_parameters": {
448
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
449
+ "quantization": "awq",
450
+ "tensor_parallel_size": 2,
451
+ "dtype": "bfloat16",
452
+ "distributed_executor_backend": "mp",
453
+ "block_size": 128,
454
+ "trust_remote_code": "",
455
+ "disable_log_stats": "",
456
+ "enforce_eager": "",
457
+ "max_num_batched_tokens": 2048,
458
+ "max_num_seqs": 256,
459
+ "load_format": "dummy"
460
+ },
461
+ "client_parameters": {
462
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
463
+ "backend": "vllm",
464
+ "dataset_name": "sharegpt",
465
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
466
+ "num_prompts": 200
467
+ }
468
+ },
469
+ {
470
+ "test_name": "serving_llama8B_int4_tp4_sharegpt",
471
+ "qps_list": ["inf"],
472
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
473
+ "server_environment_variables": {
474
+ "VLLM_RPC_TIMEOUT": 100000,
475
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
476
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
477
+ "VLLM_CPU_SGL_KERNEL": 1,
478
+ "VLLM_CPU_KVCACHE_SPACE": 40
479
+ },
480
+ "server_parameters": {
481
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
482
+ "quantization": "awq",
483
+ "tensor_parallel_size": 4,
484
+ "dtype": "bfloat16",
485
+ "distributed_executor_backend": "mp",
486
+ "block_size": 128,
487
+ "trust_remote_code": "",
488
+ "disable_log_stats": "",
489
+ "enforce_eager": "",
490
+ "max_num_batched_tokens": 2048,
491
+ "max_num_seqs": 256,
492
+ "load_format": "dummy"
493
+ },
494
+ "client_parameters": {
495
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
496
+ "backend": "vllm",
497
+ "dataset_name": "sharegpt",
498
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
499
+ "num_prompts": 200
500
+ }
501
+ },
502
+ {
503
+ "test_name": "serving_llama8B_int4_tp1_random_128_128",
504
+ "qps_list": ["inf"],
505
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
506
+ "server_environment_variables": {
507
+ "VLLM_RPC_TIMEOUT": 100000,
508
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
509
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
510
+ "VLLM_CPU_SGL_KERNEL": 1,
511
+ "VLLM_CPU_KVCACHE_SPACE": 40
512
+ },
513
+ "server_parameters": {
514
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
515
+ "quantization": "awq",
516
+ "tensor_parallel_size": 1,
517
+ "dtype": "bfloat16",
518
+ "distributed_executor_backend": "mp",
519
+ "block_size": 128,
520
+ "trust_remote_code": "",
521
+ "enable_chunked_prefill": "",
522
+ "disable_log_stats": "",
523
+ "enforce_eager": "",
524
+ "max_num_batched_tokens": 2048,
525
+ "max_num_seqs": 256,
526
+ "load_format": "dummy"
527
+ },
528
+ "client_parameters": {
529
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
530
+ "backend": "vllm",
531
+ "dataset_name": "random",
532
+ "random-input-len": 128,
533
+ "random-output-len": 128,
534
+ "ignore-eos": "",
535
+ "num_prompts": 1000
536
+ }
537
+ },
538
+ {
539
+ "test_name": "serving_llama8B_int4_tp2_random_128_128",
540
+ "qps_list": ["inf"],
541
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
542
+ "server_environment_variables": {
543
+ "VLLM_RPC_TIMEOUT": 100000,
544
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
545
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
546
+ "VLLM_CPU_SGL_KERNEL": 1,
547
+ "VLLM_CPU_KVCACHE_SPACE": 40
548
+ },
549
+ "server_parameters": {
550
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
551
+ "quantization": "awq",
552
+ "tensor_parallel_size": 2,
553
+ "dtype": "bfloat16",
554
+ "distributed_executor_backend": "mp",
555
+ "block_size": 128,
556
+ "trust_remote_code": "",
557
+ "enable_chunked_prefill": "",
558
+ "disable_log_stats": "",
559
+ "enforce_eager": "",
560
+ "max_num_batched_tokens": 2048,
561
+ "max_num_seqs": 256,
562
+ "load_format": "dummy"
563
+ },
564
+ "client_parameters": {
565
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
566
+ "backend": "vllm",
567
+ "dataset_name": "random",
568
+ "random-input-len": 128,
569
+ "random-output-len": 128,
570
+ "ignore-eos": "",
571
+ "num_prompts": 1000
572
+ }
573
+ },
574
+ {
575
+ "test_name": "serving_llama8B_int4_tp4_random_128_128",
576
+ "qps_list": ["inf"],
577
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
578
+ "server_environment_variables": {
579
+ "VLLM_RPC_TIMEOUT": 100000,
580
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
581
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
582
+ "VLLM_CPU_SGL_KERNEL": 1,
583
+ "VLLM_CPU_KVCACHE_SPACE": 40
584
+ },
585
+ "server_parameters": {
586
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
587
+ "quantization": "awq",
588
+ "tensor_parallel_size": 4,
589
+ "dtype": "bfloat16",
590
+ "distributed_executor_backend": "mp",
591
+ "block_size": 128,
592
+ "trust_remote_code": "",
593
+ "enable_chunked_prefill": "",
594
+ "disable_log_stats": "",
595
+ "enforce_eager": "",
596
+ "max_num_batched_tokens": 2048,
597
+ "max_num_seqs": 256,
598
+ "load_format": "dummy"
599
+ },
600
+ "client_parameters": {
601
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
602
+ "backend": "vllm",
603
+ "dataset_name": "random",
604
+ "random-input-len": 128,
605
+ "random-output-len": 128,
606
+ "ignore-eos": "",
607
+ "num_prompts": 1000
608
+ }
609
}
610
]
.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc3.json
+621
-6
@@ -1,6 +1,6 @@
1
[
2
{
3
- "test_name": "serving_llama8B_pp1_sharegpt",
3
+ "test_name": "serving_llama8B_bf16_pp1_sharegpt",
4
"qps_list": ["inf"],
5
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
6
"server_environment_variables": {
@@ -32,7 +32,39 @@
32
}
33
},
34
{
35
- "test_name": "serving_llama8B_pp3_sharegpt",
35
+ "test_name": "serving_llama8B_bf16_tp2_sharegpt",
36
+ "qps_list": ["inf"],
37
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
38
+ "server_environment_variables": {
39
+ "VLLM_RPC_TIMEOUT": 100000,
40
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
41
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
42
+ "VLLM_CPU_SGL_KERNEL": 1,
43
+ "VLLM_CPU_KVCACHE_SPACE": 40
44
+ },
45
+ "server_parameters": {
46
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
47
+ "tensor_parallel_size": 2,
48
+ "dtype": "bfloat16",
49
+ "distributed_executor_backend": "mp",
50
+ "block_size": 128,
51
+ "trust_remote_code": "",
52
+ "disable_log_stats": "",
53
+ "enforce_eager": "",
54
+ "max_num_batched_tokens": 2048,
55
+ "max_num_seqs": 256,
56
+ "load_format": "dummy"
57
+ },
58
+ "client_parameters": {
59
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
60
+ "backend": "vllm",
61
+ "dataset_name": "sharegpt",
62
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
63
+ "num_prompts": 200
64
+ }
65
+ },
66
+ {
67
+ "test_name": "serving_llama8B_bf16_pp3_sharegpt",
68
"qps_list": ["inf"],
69
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
70
"server_environment_variables": {
@@ -64,7 +96,7 @@
96
}
97
},
98
{
67
- "test_name": "serving_llama8B_tp2pp3_sharegpt",
99
+ "test_name": "serving_llama8B_bf16_tp2pp3_sharegpt",
100
"qps_list": ["inf"],
101
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
102
"server_environment_variables": {
@@ -97,7 +129,7 @@
129
}
130
},
131
{
100
- "test_name": "serving_llama8B_pp1_random_128_128",
132
+ "test_name": "serving_llama8B_bf16_pp1_random_128_128",
133
"qps_list": ["inf"],
134
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
135
"server_environment_variables": {
@@ -132,7 +164,42 @@
164
}
165
},
166
{
135
- "test_name": "serving_llama8B_pp3_random_128_128",
167
+ "test_name": "serving_llama8B_bf16_tp2_random_128_128",
168
+ "qps_list": ["inf"],
169
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
170
+ "server_environment_variables": {
171
+ "VLLM_RPC_TIMEOUT": 100000,
172
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
173
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
174
+ "VLLM_CPU_SGL_KERNEL": 1,
175
+ "VLLM_CPU_KVCACHE_SPACE": 40
176
+ },
177
+ "server_parameters": {
178
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
179
+ "tensor_parallel_size": 2,
180
+ "dtype": "bfloat16",
181
+ "distributed_executor_backend": "mp",
182
+ "block_size": 128,
183
+ "trust_remote_code": "",
184
+ "enable_chunked_prefill": "",
185
+ "disable_log_stats": "",
186
+ "enforce_eager": "",
187
+ "max_num_batched_tokens": 2048,
188
+ "max_num_seqs": 256,
189
+ "load_format": "dummy"
190
+ },
191
+ "client_parameters": {
192
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
193
+ "backend": "vllm",
194
+ "dataset_name": "random",
195
+ "random-input-len": 128,
196
+ "random-output-len": 128,
197
+ "ignore-eos": "",
198
+ "num_prompts": 1000
199
+ }
200
+ },
201
+ {
202
+ "test_name": "serving_llama8B_bf16_pp3_random_128_128",
203
"qps_list": ["inf"],
204
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
205
"server_environment_variables": {
@@ -167,7 +234,7 @@
234
}
235
},
236
{
170
- "test_name": "serving_llama8B_tp2pp3_random_128_128",
237
+ "test_name": "serving_llama8B_bf16_tp2pp3_random_128_128",
238
"qps_list": ["inf"],
239
"max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
240
"server_environment_variables": {
@@ -201,5 +268,553 @@
268
"ignore-eos": "",
269
"num_prompts": 1000
270
}
271
+ },
272
+ {
273
+ "test_name": "serving_llama8B_int8_pp1_sharegpt",
274
+ "qps_list": ["inf"],
275
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
276
+ "server_environment_variables": {
277
+ "VLLM_RPC_TIMEOUT": 100000,
278
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
279
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
280
+ "VLLM_CPU_SGL_KERNEL": 1,
281
+ "VLLM_CPU_KVCACHE_SPACE": 40
282
+ },
283
+ "server_parameters": {
284
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
285
+ "pipeline_parallel_size": 1,
286
+ "dtype": "bfloat16",
287
+ "distributed_executor_backend": "mp",
288
+ "block_size": 128,
289
+ "trust_remote_code": "",
290
+ "disable_log_stats": "",
291
+ "enforce_eager": "",
292
+ "max_num_batched_tokens": 2048,
293
+ "max_num_seqs": 256,
294
+ "load_format": "dummy"
295
+ },
296
+ "client_parameters": {
297
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
298
+ "backend": "vllm",
299
+ "dataset_name": "sharegpt",
300
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
301
+ "num_prompts": 200
302
+ }
303
+ },
304
+ {
305
+ "test_name": "serving_llama8B_int8_tp2_sharegpt",
306
+ "qps_list": ["inf"],
307
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
308
+ "server_environment_variables": {
309
+ "VLLM_RPC_TIMEOUT": 100000,
310
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
311
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
312
+ "VLLM_CPU_SGL_KERNEL": 1,
313
+ "VLLM_CPU_KVCACHE_SPACE": 40
314
+ },
315
+ "server_parameters": {
316
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
317
+ "tensor_parallel_size": 2,
318
+ "dtype": "bfloat16",
319
+ "distributed_executor_backend": "mp",
320
+ "block_size": 128,
321
+ "trust_remote_code": "",
322
+ "disable_log_stats": "",
323
+ "enforce_eager": "",
324
+ "max_num_batched_tokens": 2048,
325
+ "max_num_seqs": 256,
326
+ "load_format": "dummy"
327
+ },
328
+ "client_parameters": {
329
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
330
+ "backend": "vllm",
331
+ "dataset_name": "sharegpt",
332
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
333
+ "num_prompts": 200
334
+ }
335
+ },
336
+ {
337
+ "test_name": "serving_llama8B_int8_pp3_sharegpt",
338
+ "qps_list": ["inf"],
339
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
340
+ "server_environment_variables": {
341
+ "VLLM_RPC_TIMEOUT": 100000,
342
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
343
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
344
+ "VLLM_CPU_SGL_KERNEL": 1,
345
+ "VLLM_CPU_KVCACHE_SPACE": 40
346
+ },
347
+ "server_parameters": {
348
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
349
+ "pipeline_parallel_size": 3,
350
+ "dtype": "bfloat16",
351
+ "distributed_executor_backend": "mp",
352
+ "block_size": 128,
353
+ "trust_remote_code": "",
354
+ "disable_log_stats": "",
355
+ "enforce_eager": "",
356
+ "max_num_batched_tokens": 2048,
357
+ "max_num_seqs": 256,
358
+ "load_format": "dummy"
359
+ },
360
+ "client_parameters": {
361
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
362
+ "backend": "vllm",
363
+ "dataset_name": "sharegpt",
364
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
365
+ "num_prompts": 200
366
+ }
367
+ },
368
+ {
369
+ "test_name": "serving_llama8B_int8_tp2pp3_sharegpt",
370
+ "qps_list": ["inf"],
371
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
372
+ "server_environment_variables": {
373
+ "VLLM_RPC_TIMEOUT": 100000,
374
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
375
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
376
+ "VLLM_CPU_SGL_KERNEL": 1,
377
+ "VLLM_CPU_KVCACHE_SPACE": 40
378
+ },
379
+ "server_parameters": {
380
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
381
+ "tensor_parallel_size": 2,
382
+ "pipeline_parallel_size": 3,
383
+ "dtype": "bfloat16",
384
+ "distributed_executor_backend": "mp",
385
+ "block_size": 128,
386
+ "trust_remote_code": "",
387
+ "disable_log_stats": "",
388
+ "enforce_eager": "",
389
+ "max_num_batched_tokens": 2048,
390
+ "max_num_seqs": 256,
391
+ "load_format": "dummy"
392
+ },
393
+ "client_parameters": {
394
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
395
+ "backend": "vllm",
396
+ "dataset_name": "sharegpt",
397
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
398
+ "num_prompts": 200
399
+ }
400
+ },
401
+ {
402
+ "test_name": "serving_llama8B_int8_pp1_random_128_128",
403
+ "qps_list": ["inf"],
404
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
405
+ "server_environment_variables": {
406
+ "VLLM_RPC_TIMEOUT": 100000,
407
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
408
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
409
+ "VLLM_CPU_SGL_KERNEL": 1,
410
+ "VLLM_CPU_KVCACHE_SPACE": 40
411
+ },
412
+ "server_parameters": {
413
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
414
+ "pipeline_parallel_size": 1,
415
+ "dtype": "bfloat16",
416
+ "distributed_executor_backend": "mp",
417
+ "block_size": 128,
418
+ "trust_remote_code": "",
419
+ "enable_chunked_prefill": "",
420
+ "disable_log_stats": "",
421
+ "enforce_eager": "",
422
+ "max_num_batched_tokens": 2048,
423
+ "max_num_seqs": 256,
424
+ "load_format": "dummy"
425
+ },
426
+ "client_parameters": {
427
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
428
+ "backend": "vllm",
429
+ "dataset_name": "random",
430
+ "random-input-len": 128,
431
+ "random-output-len": 128,
432
+ "ignore-eos": "",
433
+ "num_prompts": 1000
434
+ }
435
+ },
436
+ {
437
+ "test_name": "serving_llama8B_int8_tp2_random_128_128",
438
+ "qps_list": ["inf"],
439
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
440
+ "server_environment_variables": {
441
+ "VLLM_RPC_TIMEOUT": 100000,
442
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
443
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
444
+ "VLLM_CPU_SGL_KERNEL": 1,
445
+ "VLLM_CPU_KVCACHE_SPACE": 40
446
+ },
447
+ "server_parameters": {
448
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
449
+ "tensor_parallel_size": 2,
450
+ "dtype": "bfloat16",
451
+ "distributed_executor_backend": "mp",
452
+ "block_size": 128,
453
+ "trust_remote_code": "",
454
+ "enable_chunked_prefill": "",
455
+ "disable_log_stats": "",
456
+ "enforce_eager": "",
457
+ "max_num_batched_tokens": 2048,
458
+ "max_num_seqs": 256,
459
+ "load_format": "dummy"
460
+ },
461
+ "client_parameters": {
462
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
463
+ "backend": "vllm",
464
+ "dataset_name": "random",
465
+ "random-input-len": 128,
466
+ "random-output-len": 128,
467
+ "ignore-eos": "",
468
+ "num_prompts": 1000
469
+ }
470
+ },
471
+ {
472
+ "test_name": "serving_llama8B_int8_pp3_random_128_128",
473
+ "qps_list": ["inf"],
474
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
475
+ "server_environment_variables": {
476
+ "VLLM_RPC_TIMEOUT": 100000,
477
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
478
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
479
+ "VLLM_CPU_SGL_KERNEL": 1,
480
+ "VLLM_CPU_KVCACHE_SPACE": 40
481
+ },
482
+ "server_parameters": {
483
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
484
+ "pipeline_parallel_size": 3,
485
+ "dtype": "bfloat16",
486
+ "distributed_executor_backend": "mp",
487
+ "block_size": 128,
488
+ "trust_remote_code": "",
489
+ "enable_chunked_prefill": "",
490
+ "disable_log_stats": "",
491
+ "enforce_eager": "",
492
+ "max_num_batched_tokens": 2048,
493
+ "max_num_seqs": 256,
494
+ "load_format": "dummy"
495
+ },
496
+ "client_parameters": {
497
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
498
+ "backend": "vllm",
499
+ "dataset_name": "random",
500
+ "random-input-len": 128,
501
+ "random-output-len": 128,
502
+ "ignore-eos": "",
503
+ "num_prompts": 1000
504
+ }
505
+ },
506
+ {
507
+ "test_name": "serving_llama8B_int8_tp2pp3_random_128_128",
508
+ "qps_list": ["inf"],
509
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
510
+ "server_environment_variables": {
511
+ "VLLM_RPC_TIMEOUT": 100000,
512
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
513
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
514
+ "VLLM_CPU_SGL_KERNEL": 1,
515
+ "VLLM_CPU_KVCACHE_SPACE": 40
516
+ },
517
+ "server_parameters": {
518
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
519
+ "tensor_parallel_size": 2,
520
+ "pipeline_parallel_size": 3,
521
+ "dtype": "bfloat16",
522
+ "distributed_executor_backend": "mp",
523
+ "block_size": 128,
524
+ "trust_remote_code": "",
525
+ "enable_chunked_prefill": "",
526
+ "disable_log_stats": "",
527
+ "enforce_eager": "",
528
+ "max_num_batched_tokens": 2048,
529
+ "max_num_seqs": 256,
530
+ "load_format": "dummy"
531
+ },
532
+ "client_parameters": {
533
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
534
+ "backend": "vllm",
535
+ "dataset_name": "random",
536
+ "random-input-len": 128,
537
+ "random-output-len": 128,
538
+ "ignore-eos": "",
539
+ "num_prompts": 1000
540
+ }
541
+ },
542
+ {
543
+ "test_name": "serving_llama8B_int4_pp1_sharegpt",
544
+ "qps_list": ["inf"],
545
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
546
+ "server_environment_variables": {
547
+ "VLLM_RPC_TIMEOUT": 100000,
548
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
549
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
550
+ "VLLM_CPU_SGL_KERNEL": 1,
551
+ "VLLM_CPU_KVCACHE_SPACE": 40
552
+ },
553
+ "server_parameters": {
554
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
555
+ "quantization": "awq",
556
+ "pipeline_parallel_size": 1,
557
+ "dtype": "bfloat16",
558
+ "distributed_executor_backend": "mp",
559
+ "block_size": 128,
560
+ "trust_remote_code": "",
561
+ "disable_log_stats": "",
562
+ "enforce_eager": "",
563
+ "max_num_batched_tokens": 2048,
564
+ "max_num_seqs": 256,
565
+ "load_format": "dummy"
566
+ },
567
+ "client_parameters": {
568
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
569
+ "backend": "vllm",
570
+ "dataset_name": "sharegpt",
571
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
572
+ "num_prompts": 200
573
+ }
574
+ },
575
+ {
576
+ "test_name": "serving_llama8B_int4_tp2_sharegpt",
577
+ "qps_list": ["inf"],
578
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
579
+ "server_environment_variables": {
580
+ "VLLM_RPC_TIMEOUT": 100000,
581
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
582
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
583
+ "VLLM_CPU_SGL_KERNEL": 1,
584
+ "VLLM_CPU_KVCACHE_SPACE": 40
585
+ },
586
+ "server_parameters": {
587
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
588
+ "quantization": "awq",
589
+ "tensor_parallel_size": 2,
590
+ "dtype": "bfloat16",
591
+ "distributed_executor_backend": "mp",
592
+ "block_size": 128,
593
+ "trust_remote_code": "",
594
+ "disable_log_stats": "",
595
+ "enforce_eager": "",
596
+ "max_num_batched_tokens": 2048,
597
+ "max_num_seqs": 256,
598
+ "load_format": "dummy"
599
+ },
600
+ "client_parameters": {
601
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
602
+ "backend": "vllm",
603
+ "dataset_name": "sharegpt",
604
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
605
+ "num_prompts": 200
606
+ }
607
+ },
608
+ {
609
+ "test_name": "serving_llama8B_int4_pp3_sharegpt",
610
+ "qps_list": ["inf"],
611
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
612
+ "server_environment_variables": {
613
+ "VLLM_RPC_TIMEOUT": 100000,
614
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
615
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
616
+ "VLLM_CPU_SGL_KERNEL": 1,
617
+ "VLLM_CPU_KVCACHE_SPACE": 40
618
+ },
619
+ "server_parameters": {
620
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
621
+ "quantization": "awq",
622
+ "pipeline_parallel_size": 3,
623
+ "dtype": "bfloat16",
624
+ "distributed_executor_backend": "mp",
625
+ "block_size": 128,
626
+ "trust_remote_code": "",
627
+ "disable_log_stats": "",
628
+ "enforce_eager": "",
629
+ "max_num_batched_tokens": 2048,
630
+ "max_num_seqs": 256,
631
+ "load_format": "dummy"
632
+ },
633
+ "client_parameters": {
634
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
635
+ "backend": "vllm",
636
+ "dataset_name": "sharegpt",
637
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
638
+ "num_prompts": 200
639
+ }
640
+ },
641
+ {
642
+ "test_name": "serving_llama8B_int4_tp2pp3_sharegpt",
643
+ "qps_list": ["inf"],
644
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
645
+ "server_environment_variables": {
646
+ "VLLM_RPC_TIMEOUT": 100000,
647
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
648
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
649
+ "VLLM_CPU_SGL_KERNEL": 1,
650
+ "VLLM_CPU_KVCACHE_SPACE": 40
651
+ },
652
+ "server_parameters": {
653
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
654
+ "quantization": "awq",
655
+ "tensor_parallel_size": 2,
656
+ "pipeline_parallel_size": 3,
657
+ "dtype": "bfloat16",
658
+ "distributed_executor_backend": "mp",
659
+ "block_size": 128,
660
+ "trust_remote_code": "",
661
+ "disable_log_stats": "",
662
+ "enforce_eager": "",
663
+ "max_num_batched_tokens": 2048,
664
+ "max_num_seqs": 256,
665
+ "load_format": "dummy"
666
+ },
667
+ "client_parameters": {
668
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
669
+ "backend": "vllm",
670
+ "dataset_name": "sharegpt",
671
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
672
+ "num_prompts": 200
673
+ }
674
+ },
675
+ {
676
+ "test_name": "serving_llama8B_int4_pp1_random_128_128",
677
+ "qps_list": ["inf"],
678
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
679
+ "server_environment_variables": {
680
+ "VLLM_RPC_TIMEOUT": 100000,
681
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
682
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
683
+ "VLLM_CPU_SGL_KERNEL": 1,
684
+ "VLLM_CPU_KVCACHE_SPACE": 40
685
+ },
686
+ "server_parameters": {
687
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
688
+ "quantization": "awq",
689
+ "pipeline_parallel_size": 1,
690
+ "dtype": "bfloat16",
691
+ "distributed_executor_backend": "mp",
692
+ "block_size": 128,
693
+ "trust_remote_code": "",
694
+ "enable_chunked_prefill": "",
695
+ "disable_log_stats": "",
696
+ "enforce_eager": "",
697
+ "max_num_batched_tokens": 2048,
698
+ "max_num_seqs": 256,
699
+ "load_format": "dummy"
700
+ },
701
+ "client_parameters": {
702
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
703
+ "backend": "vllm",
704
+ "dataset_name": "random",
705
+ "random-input-len": 128,
706
+ "random-output-len": 128,
707
+ "ignore-eos": "",
708
+ "num_prompts": 1000
709
+ }
710
+ },
711
+ {
712
+ "test_name": "serving_llama8B_int4_tp2_random_128_128",
713
+ "qps_list": ["inf"],
714
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
715
+ "server_environment_variables": {
716
+ "VLLM_RPC_TIMEOUT": 100000,
717
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
718
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
719
+ "VLLM_CPU_SGL_KERNEL": 1,
720
+ "VLLM_CPU_KVCACHE_SPACE": 40
721
+ },
722
+ "server_parameters": {
723
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
724
+ "quantization": "awq",
725
+ "tensor_parallel_size": 2,
726
+ "dtype": "bfloat16",
727
+ "distributed_executor_backend": "mp",
728
+ "block_size": 128,
729
+ "trust_remote_code": "",
730
+ "enable_chunked_prefill": "",
731
+ "disable_log_stats": "",
732
+ "enforce_eager": "",
733
+ "max_num_batched_tokens": 2048,
734
+ "max_num_seqs": 256,
735
+ "load_format": "dummy"
736
+ },
737
+ "client_parameters": {
738
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
739
+ "backend": "vllm",
740
+ "dataset_name": "random",
741
+ "random-input-len": 128,
742
+ "random-output-len": 128,
743
+ "ignore-eos": "",
744
+ "num_prompts": 1000
745
+ }
746
+ },
747
+ {
748
+ "test_name": "serving_llama8B_int4_pp3_random_128_128",
749
+ "qps_list": ["inf"],
750
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
751
+ "server_environment_variables": {
752
+ "VLLM_RPC_TIMEOUT": 100000,
753
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
754
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
755
+ "VLLM_CPU_SGL_KERNEL": 1,
756
+ "VLLM_CPU_KVCACHE_SPACE": 40
757
+ },
758
+ "server_parameters": {
759
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
760
+ "quantization": "awq",
761
+ "pipeline_parallel_size": 3,
762
+ "dtype": "bfloat16",
763
+ "distributed_executor_backend": "mp",
764
+ "block_size": 128,
765
+ "trust_remote_code": "",
766
+ "enable_chunked_prefill": "",
767
+ "disable_log_stats": "",
768
+ "enforce_eager": "",
769
+ "max_num_batched_tokens": 2048,
770
+ "max_num_seqs": 256,
771
+ "load_format": "dummy"
772
+ },
773
+ "client_parameters": {
774
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
775
+ "backend": "vllm",
776
+ "dataset_name": "random",
777
+ "random-input-len": 128,
778
+ "random-output-len": 128,
779
+ "ignore-eos": "",
780
+ "num_prompts": 1000
781
+ }
782
+ },
783
+ {
784
+ "test_name": "serving_llama8B_int4_tp2pp3_random_128_128",
785
+ "qps_list": ["inf"],
786
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
787
+ "server_environment_variables": {
788
+ "VLLM_RPC_TIMEOUT": 100000,
789
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
790
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
791
+ "VLLM_CPU_SGL_KERNEL": 1,
792
+ "VLLM_CPU_KVCACHE_SPACE": 40
793
+ },
794
+ "server_parameters": {
795
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
796
+ "quantization": "awq",
797
+ "tensor_parallel_size": 2,
798
+ "pipeline_parallel_size": 3,
799
+ "dtype": "bfloat16",
800
+ "distributed_executor_backend": "mp",
801
+ "block_size": 128,
802
+ "trust_remote_code": "",
803
+ "enable_chunked_prefill": "",
804
+ "disable_log_stats": "",
805
+ "enforce_eager": "",
806
+ "max_num_batched_tokens": 2048,
807
+ "max_num_seqs": 256,
808
+ "load_format": "dummy"
809
+ },
810
+ "client_parameters": {
811
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
812
+ "backend": "vllm",
813
+ "dataset_name": "random",
814
+ "random-input-len": 128,
815
+ "random-output-len": 128,
816
+ "ignore-eos": "",
817
+ "num_prompts": 1000
818
+ }
819
}
820
]
docs/contributing/benchmarks.md
+31
-1
@@ -11,9 +11,39 @@ vLLM contains two sets of benchmarks:
11
12
The performance benchmarks are used for development to confirm whether new changes improve performance under various workloads. They are triggered on every commit with both the `perf-benchmarks` and `ready` labels, and when a PR is merged into vLLM.
13
14
+### Manually Trigger the benchmark
15
+
16
+Use [vllm-ci-test-repo images](https://gallery.ecr.aws/q9t5s3a7/vllm-ci-test-repo) with vLLM benchmark suite.
17
+For CPU environment, please use the image with "-cpu" postfix.
18
+
19
+Here is an example for docker run command for CPU.
20
+
21
+```bash
22
+docker run -it --entrypoint /bin/bash -v /data/huggingface:/root/.cache/huggingface -e HF_TOKEN='' --shm-size=16g --name vllm-cpu-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:1da94e673c257373280026f75ceb4effac80e892-cpu
23
+```
24
+
25
+Then, run below command inside the docker instance.
26
+
27
+```bash
28
+bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
29
+```
30
+
31
+When run, benchmark script generates results under **benchmark/results** folder, along with the benchmark_results.md and benchmark_results.json.
32
+
33
+#### Runtime environment variables
34
+
35
+- `ON_CPU`: set the value to '1' on Intel® Xeon® Processors. Default value is 0.
36
+- `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file).
37
+- `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file).
38
+- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file).
39
+- `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string.
40
+- `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string.
41
+
42
+For more results visualization, check the [visualizing the results](https://github.com/intel-ai-tce/vllm/blob/more_cpu_models/.buildkite/nightly-benchmarks/README.md#visualizing-the-results).
43
+
44
The latest performance results are hosted on the public [vLLM Performance Dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm).
45
16
-More information on the performance benchmarks and their parameters can be found [here](gh-file:.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md).
46
+More information on the performance benchmarks and their parameters can be found in [Benchmark README](https://github.com/intel-ai-tce/vllm/blob/more_cpu_models/.buildkite/nightly-benchmarks/README.md) and [performance benchmark description](gh-file:.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md).
47
48
[](){ #nightly-benchmarks }
49