@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a21256c463

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 14:45 UTC

Add TP CLI argument to multimodal inference examples (#29301)

Signed-off-by: Lin, Fanli <fanli.lin@intel.com>

Fanli Lin committed Nov 25, 2025 at 14:03 UTC a21256c46327ec366b7804d22ba66ed04c2ae18b

3 files changed +65 -5

examples/offline_inference/audio_language.py

+15

@@ -425,6 +425,13 @@ def parse_args():

425 default=None,

426 help="Set the seed when initializing `vllm.LLM`.",

427 )

428 + parser.add_argument(

429 + "--tensor-parallel-size",

430 + "-tp",

431 + type=int,

432 + default=None,

433 + help="Tensor parallel size to override the model's default setting. ",

434 + )

435

436 return parser.parse_args()

437

@@ -434,6 +441,12 @@ def main(args):

441 if model not in model_example_map:

442 raise ValueError(f"Model type {model} is not supported.")

443

444 + if args.tensor_parallel_size is not None and args.tensor_parallel_size < 1:

445 + raise ValueError(

446 + f"tensor_parallel_size must be a positive integer, "

447 + f"got {args.tensor_parallel_size}"

448 + )

449 +

450 audio_count = args.num_audios

451 req_data = model_example_map[model](

452 question_per_audio_count[audio_count], audio_count

@@ -446,6 +459,8 @@ def main(args):

459 )

460

461 engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

462 + if args.tensor_parallel_size is not None:

463 + engine_args["tensor_parallel_size"] = args.tensor_parallel_size

464 llm = LLM(**engine_args)

465

466 # We set temperature to 0.2 so that outputs can be different

examples/offline_inference/vision_language.py

+15

@@ -2064,6 +2064,13 @@ def parse_args():

2064 help="If True, will send all requests in a second batch with empty mm "

2065 "data to verify cache hits with UUIDs.",

2066 )

2067 + parser.add_argument(

2068 + "--tensor-parallel-size",

2069 + "-tp",

2070 + type=int,

2071 + default=None,

2072 + help="Tensor parallel size to override the model's default setting. ",

2073 + )

2074 return parser.parse_args()

2075

2076

@@ -2072,6 +2079,12 @@ def main(args):

2079 if model not in model_example_map:

2080 raise ValueError(f"Model type {model} is not supported.")

2081

2082 + if args.tensor_parallel_size is not None and args.tensor_parallel_size < 1:

2083 + raise ValueError(

2084 + f"tensor_parallel_size must be a positive integer, "

2085 + f"got {args.tensor_parallel_size}"

2086 + )

2087 +

2088 modality = args.modality

2089 mm_input = get_multi_modal_input(args)

2090 data = mm_input["data"]

@@ -2089,6 +2102,8 @@ def main(args):

2102 "seed": args.seed,

2103 "mm_processor_cache_gb": 0 if args.disable_mm_processor_cache else 4,

2104 }

2105 + if args.tensor_parallel_size is not None:

2106 + engine_args["tensor_parallel_size"] = args.tensor_parallel_size

2107 llm = LLM(**engine_args)

2108

2109 # Don't want to check the flag multiple times, so just hijack `prompts`.

examples/offline_inference/vision_language_multi_image.py

+35 -5

@@ -1352,10 +1352,18 @@ model_example_map = {

1352 }

1353

1354

1355 -def run_generate(model, question: str, image_urls: list[str], seed: int | None):

1355 +def run_generate(

1356 + model,

1357 + question: str,

1358 + image_urls: list[str],

1359 + seed: int | None,

1360 + tensor_parallel_size: int | None,

1361 +):

1362 req_data = model_example_map[model](question, image_urls)

1363

1358 - engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

1364 + engine_args = asdict(req_data.engine_args) | {"seed": seed}

1365 + if tensor_parallel_size is not None:

1366 + engine_args["tensor_parallel_size"] = tensor_parallel_size

1367 llm = LLM(**engine_args)

1368

1369 sampling_params = SamplingParams(

@@ -1378,7 +1386,13 @@ def run_generate(model, question: str, image_urls: list[str], seed: int | None):

1386 print("-" * 50)

1387

1388

1381 -def run_chat(model: str, question: str, image_urls: list[str], seed: int | None):

1389 +def run_chat(

1390 + model: str,

1391 + question: str,

1392 + image_urls: list[str],

1393 + seed: int | None,

1394 + tensor_parallel_size: int | None,

1395 +):

1396 req_data = model_example_map[model](question, image_urls)

1397

1398 # Disable other modalities to save memory

@@ -1388,6 +1402,8 @@ def run_chat(model: str, question: str, image_urls: list[str], seed: int | None)

1402 )

1403

1404 engine_args = asdict(req_data.engine_args) | {"seed": seed}

1405 + if tensor_parallel_size is not None:

1406 + engine_args["tensor_parallel_size"] = tensor_parallel_size

1407 llm = LLM(**engine_args)

1408

1409 sampling_params = (

@@ -1463,6 +1479,13 @@ def parse_args():

1479 default=2,

1480 help="Number of images to use for the demo.",

1481 )

1482 + parser.add_argument(

1483 + "--tensor-parallel-size",

1484 + "-tp",

1485 + type=int,

1486 + default=None,

1487 + help="Tensor parallel size to override the model's default setting. ",

1488 + )

1489 return parser.parse_args()

1490

1491

@@ -1470,13 +1493,20 @@ def main(args: Namespace):

1493 model = args.model_type

1494 method = args.method

1495 seed = args.seed

1496 + tensor_parallel_size = args.tensor_parallel_size

1497 +

1498 + if tensor_parallel_size is not None and tensor_parallel_size < 1:

1499 + raise ValueError(

1500 + f"tensor_parallel_size must be a positive integer, "

1501 + f"got {tensor_parallel_size}"

1502 + )

1503

1504 image_urls = IMAGE_URLS[: args.num_images]

1505

1506 if method == "generate":

1477 - run_generate(model, QUESTION, image_urls, seed)

1507 + run_generate(model, QUESTION, image_urls, seed, tensor_parallel_size)

1508 elif method == "chat":

1479 - run_chat(model, QUESTION, image_urls, seed)

1509 + run_chat(model, QUESTION, image_urls, seed, tensor_parallel_size)

1510 else:

1511 raise ValueError(f"Invalid method: {method}")

1512