A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Move the last arguments in `arg_utils.py` to be in their final groups (#17531)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 1, 2025 at 18:31 UTC
6768ff4a222b89dac3e2f49af8772216db01952b
2 files changed
+135
-138
vllm/config.py
+1
-1
@@ -1637,7 +1637,7 @@ class ParallelConfig:
1637
"""Use expert parallelism instead of tensor parallelism for MoE layers."""
1638
1639
max_parallel_loading_workers: Optional[int] = None
1640
- """Maximum number of parallal loading workers when loading model
1640
+ """Maximum number of parallel loading workers when loading model
1641
sequentially in multiple batches. To avoid RAM OOM when using tensor
1642
parallel and large models."""
1643
vllm/engine/arg_utils.py
+134
-137
@@ -474,15 +474,21 @@ class EngineArgs:
474
title="LoadConfig",
475
description=LoadConfig.__doc__,
476
)
477
- load_group.add_argument('--load-format',
477
+ load_group.add_argument("--load-format",
478
choices=[f.value for f in LoadFormat],
479
**load_kwargs["load_format"])
480
- load_group.add_argument('--download-dir',
480
+ load_group.add_argument("--download-dir",
481
**load_kwargs["download_dir"])
482
- load_group.add_argument('--model-loader-extra-config',
482
+ load_group.add_argument("--model-loader-extra-config",
483
**load_kwargs["model_loader_extra_config"])
484
- load_group.add_argument('--use-tqdm-on-load',
484
+ load_group.add_argument("--ignore-patterns",
485
+ **load_kwargs["ignore_patterns"])
486
+ load_group.add_argument("--use-tqdm-on-load",
487
**load_kwargs["use_tqdm_on_load"])
488
+ load_group.add_argument('--qlora-adapter-name-or-path',
489
+ type=str,
490
+ default=None,
491
+ help='Name or path of the QLoRA adapter.')
492
493
# Guided decoding arguments
494
guided_decoding_kwargs = get_kwargs(DecodingConfig)
@@ -501,6 +507,14 @@ class EngineArgs:
507
guided_decoding_group.add_argument(
508
"--guided-decoding-disable-additional-properties",
509
**guided_decoding_kwargs["disable_additional_properties"])
510
+ guided_decoding_group.add_argument(
511
+ "--enable-reasoning",
512
+ action=argparse.BooleanOptionalAction,
513
+ help="[DEPRECATED] The `--enable-reasoning` flag is deprecated as "
514
+ "of v0.8.6. Use `--reasoning-parser` to specify the reasoning "
515
+ "parser backend insteadThis flag (`--enable-reasoning`) will be "
516
+ "removed in v0.10.0. When `--reasoning-parser` is specified, "
517
+ "reasoning mode is automatically enabled.")
518
guided_decoding_group.add_argument(
519
"--reasoning-parser",
520
# This choices is a special case because it's not static
@@ -514,27 +528,31 @@ class EngineArgs:
528
description=ParallelConfig.__doc__,
529
)
530
parallel_group.add_argument(
517
- '--distributed-executor-backend',
531
+ "--distributed-executor-backend",
532
**parallel_kwargs["distributed_executor_backend"])
533
parallel_group.add_argument(
520
- '--pipeline-parallel-size', '-pp',
534
+ "--pipeline-parallel-size", "-pp",
535
**parallel_kwargs["pipeline_parallel_size"])
522
- parallel_group.add_argument('--tensor-parallel-size', '-tp',
536
+ parallel_group.add_argument("--tensor-parallel-size", "-tp",
537
**parallel_kwargs["tensor_parallel_size"])
524
- parallel_group.add_argument('--data-parallel-size', '-dp',
538
+ parallel_group.add_argument("--data-parallel-size", "-dp",
539
**parallel_kwargs["data_parallel_size"])
540
parallel_group.add_argument(
527
- '--enable-expert-parallel',
541
+ "--enable-expert-parallel",
542
**parallel_kwargs["enable_expert_parallel"])
543
parallel_group.add_argument(
530
- '--max-parallel-loading-workers',
544
+ "--max-parallel-loading-workers",
545
**parallel_kwargs["max_parallel_loading_workers"])
546
parallel_group.add_argument(
533
- '--ray-workers-use-nsight',
547
+ "--ray-workers-use-nsight",
548
**parallel_kwargs["ray_workers_use_nsight"])
549
parallel_group.add_argument(
536
- '--disable-custom-all-reduce',
550
+ "--disable-custom-all-reduce",
551
**parallel_kwargs["disable_custom_all_reduce"])
552
+ parallel_group.add_argument("--worker-cls",
553
+ **parallel_kwargs["worker_cls"])
554
+ parallel_group.add_argument("--worker-extension-cls",
555
+ **parallel_kwargs["worker_extension_cls"])
556
557
# KV cache arguments
558
cache_kwargs = get_kwargs(CacheConfig)
@@ -542,47 +560,34 @@ class EngineArgs:
560
title="CacheConfig",
561
description=CacheConfig.__doc__,
562
)
545
- cache_group.add_argument('--block-size', **cache_kwargs["block_size"])
546
- cache_group.add_argument('--gpu-memory-utilization',
563
+ cache_group.add_argument("--block-size", **cache_kwargs["block_size"])
564
+ cache_group.add_argument("--gpu-memory-utilization",
565
**cache_kwargs["gpu_memory_utilization"])
548
- cache_group.add_argument('--swap-space', **cache_kwargs["swap_space"])
549
- cache_group.add_argument('--kv-cache-dtype',
566
+ cache_group.add_argument("--swap-space", **cache_kwargs["swap_space"])
567
+ cache_group.add_argument("--kv-cache-dtype",
568
**cache_kwargs["cache_dtype"])
551
- cache_group.add_argument('--num-gpu-blocks-override',
569
+ cache_group.add_argument("--num-gpu-blocks-override",
570
**cache_kwargs["num_gpu_blocks_override"])
571
cache_group.add_argument("--enable-prefix-caching",
572
**cache_kwargs["enable_prefix_caching"])
573
cache_group.add_argument("--prefix-caching-hash-algo",
574
**cache_kwargs["prefix_caching_hash_algo"])
557
- cache_group.add_argument('--cpu-offload-gb',
575
+ cache_group.add_argument("--cpu-offload-gb",
576
**cache_kwargs["cpu_offload_gb"])
559
- cache_group.add_argument('--calculate-kv-scales',
577
+ cache_group.add_argument("--calculate-kv-scales",
578
**cache_kwargs["calculate_kv_scales"])
579
562
- parser.add_argument('--use-v2-block-manager',
563
- action='store_true',
564
- default=True,
565
- help='[DEPRECATED] block manager v1 has been '
566
- 'removed and SelfAttnBlockSpaceManager (i.e. '
567
- 'block manager v2) is now the default. '
568
- 'Setting this flag to True or False'
569
- ' has no effect on vLLM behavior.')
570
-
571
- parser.add_argument('--disable-log-stats',
572
- action='store_true',
573
- help='Disable logging statistics.')
574
-
580
# Tokenizer arguments
581
tokenizer_kwargs = get_kwargs(TokenizerPoolConfig)
582
tokenizer_group = parser.add_argument_group(
583
title="TokenizerPoolConfig",
584
description=TokenizerPoolConfig.__doc__,
585
)
581
- tokenizer_group.add_argument('--tokenizer-pool-size',
586
+ tokenizer_group.add_argument("--tokenizer-pool-size",
587
**tokenizer_kwargs["pool_size"])
583
- tokenizer_group.add_argument('--tokenizer-pool-type',
588
+ tokenizer_group.add_argument("--tokenizer-pool-type",
589
**tokenizer_kwargs["pool_type"])
585
- tokenizer_group.add_argument('--tokenizer-pool-extra-config',
590
+ tokenizer_group.add_argument("--tokenizer-pool-extra-config",
591
**tokenizer_kwargs["extra_config"])
592
593
# Multimodal related configs
@@ -591,13 +596,13 @@ class EngineArgs:
596
title="MultiModalConfig",
597
description=MultiModalConfig.__doc__,
598
)
594
- multimodal_group.add_argument('--limit-mm-per-prompt',
599
+ multimodal_group.add_argument("--limit-mm-per-prompt",
600
**multimodal_kwargs["limit_per_prompt"])
601
multimodal_group.add_argument(
597
- '--mm-processor-kwargs',
602
+ "--mm-processor-kwargs",
603
**multimodal_kwargs["mm_processor_kwargs"])
604
multimodal_group.add_argument(
600
- '--disable-mm-preprocessor-cache',
605
+ "--disable-mm-preprocessor-cache",
606
**multimodal_kwargs["disable_mm_preprocessor_cache"])
607
608
# LoRA related configs
@@ -607,25 +612,25 @@ class EngineArgs:
612
description=LoRAConfig.__doc__,
613
)
614
lora_group.add_argument(
610
- '--enable-lora',
615
+ "--enable-lora",
616
action=argparse.BooleanOptionalAction,
612
- help='If True, enable handling of LoRA adapters.')
613
- lora_group.add_argument('--enable-lora-bias',
617
+ help="If True, enable handling of LoRA adapters.")
618
+ lora_group.add_argument("--enable-lora-bias",
619
**lora_kwargs["bias_enabled"])
615
- lora_group.add_argument('--max-loras', **lora_kwargs["max_loras"])
616
- lora_group.add_argument('--max-lora-rank',
620
+ lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
621
+ lora_group.add_argument("--max-lora-rank",
622
**lora_kwargs["max_lora_rank"])
618
- lora_group.add_argument('--lora-extra-vocab-size',
623
+ lora_group.add_argument("--lora-extra-vocab-size",
624
**lora_kwargs["lora_extra_vocab_size"])
625
lora_group.add_argument(
621
- '--lora-dtype',
626
+ "--lora-dtype",
627
**lora_kwargs["lora_dtype"],
628
)
624
- lora_group.add_argument('--long-lora-scaling-factors',
629
+ lora_group.add_argument("--long-lora-scaling-factors",
630
**lora_kwargs["long_lora_scaling_factors"])
626
- lora_group.add_argument('--max-cpu-loras',
631
+ lora_group.add_argument("--max-cpu-loras",
632
**lora_kwargs["max_cpu_loras"])
628
- lora_group.add_argument('--fully-sharded-loras',
633
+ lora_group.add_argument("--fully-sharded-loras",
634
**lora_kwargs["fully_sharded_loras"])
635
636
# PromptAdapter related configs
@@ -635,14 +640,14 @@ class EngineArgs:
640
description=PromptAdapterConfig.__doc__,
641
)
642
prompt_adapter_group.add_argument(
638
- '--enable-prompt-adapter',
643
+ "--enable-prompt-adapter",
644
action=argparse.BooleanOptionalAction,
640
- help='If True, enable handling of PromptAdapters.')
645
+ help="If True, enable handling of PromptAdapters.")
646
prompt_adapter_group.add_argument(
642
- '--max-prompt-adapters',
647
+ "--max-prompt-adapters",
648
**prompt_adapter_kwargs["max_prompt_adapters"])
649
prompt_adapter_group.add_argument(
645
- '--max-prompt-adapter-token',
650
+ "--max-prompt-adapter-token",
651
**prompt_adapter_kwargs["max_prompt_adapter_token"])
652
653
# Device arguments
@@ -659,25 +664,11 @@ class EngineArgs:
664
description=SpeculativeConfig.__doc__,
665
)
666
speculative_group.add_argument(
662
- '--speculative-config',
667
+ "--speculative-config",
668
type=json.loads,
669
default=None,
665
- help='The configurations for speculative decoding.'
666
- ' Should be a JSON string.')
667
-
668
- parser.add_argument(
669
- '--ignore-patterns',
670
- action="append",
671
- type=str,
672
- default=[],
673
- help="The pattern(s) to ignore when loading the model."
674
- "Default to `original/**/*` to avoid repeated loading of llama's "
675
- "checkpoints.")
676
-
677
- parser.add_argument('--qlora-adapter-name-or-path',
678
- type=str,
679
- default=None,
680
- help='Name or path of the QLoRA adapter.')
670
+ help="The configurations for speculative decoding. Should be a "
671
+ "JSON string.")
672
673
# Observability arguments
674
observability_kwargs = get_kwargs(ObservabilityConfig)
@@ -710,9 +701,9 @@ class EngineArgs:
701
description=SchedulerConfig.__doc__,
702
)
703
scheduler_group.add_argument(
713
- '--max-num-batched-tokens',
704
+ "--max-num-batched-tokens",
705
**scheduler_kwargs["max_num_batched_tokens"])
715
- scheduler_group.add_argument('--max-num-seqs',
706
+ scheduler_group.add_argument("--max-num-seqs",
707
**scheduler_kwargs["max_num_seqs"])
708
scheduler_group.add_argument(
709
"--max-num-partial-prefills",
@@ -723,70 +714,78 @@ class EngineArgs:
714
scheduler_group.add_argument(
715
"--long-prefill-token-threshold",
716
**scheduler_kwargs["long_prefill_token_threshold"])
726
- scheduler_group.add_argument('--num-lookahead-slots',
717
+ scheduler_group.add_argument("--num-lookahead-slots",
718
**scheduler_kwargs["num_lookahead_slots"])
728
- scheduler_group.add_argument('--scheduler-delay-factor',
719
+ scheduler_group.add_argument("--scheduler-delay-factor",
720
**scheduler_kwargs["delay_factor"])
730
- scheduler_group.add_argument('--preemption-mode',
721
+ scheduler_group.add_argument("--preemption-mode",
722
**scheduler_kwargs["preemption_mode"])
732
- scheduler_group.add_argument('--num-scheduler-steps',
723
+ scheduler_group.add_argument("--num-scheduler-steps",
724
**scheduler_kwargs["num_scheduler_steps"])
725
scheduler_group.add_argument(
735
- '--multi-step-stream-outputs',
726
+ "--multi-step-stream-outputs",
727
**scheduler_kwargs["multi_step_stream_outputs"])
737
- scheduler_group.add_argument('--scheduling-policy',
728
+ scheduler_group.add_argument("--scheduling-policy",
729
**scheduler_kwargs["policy"])
730
scheduler_group.add_argument(
740
- '--enable-chunked-prefill',
731
+ "--enable-chunked-prefill",
732
**scheduler_kwargs["enable_chunked_prefill"])
733
scheduler_group.add_argument(
734
"--disable-chunked-mm-input",
735
**scheduler_kwargs["disable_chunked_mm_input"])
745
- parser.add_argument('--scheduler-cls',
746
- **scheduler_kwargs["scheduler_cls"])
747
-
748
- parser.add_argument('--compilation-config',
749
- '-O',
750
- type=CompilationConfig.from_cli,
751
- default=None,
752
- help='torch.compile configuration for the model. '
753
- 'When it is a number (0, 1, 2, 3), it will be '
754
- 'interpreted as the optimization level.\n'
755
- 'NOTE: level 0 is the default level without '
756
- 'any optimization. level 1 and 2 are for internal '
757
- 'testing only. level 3 is the recommended level '
758
- 'for production.\n'
759
- 'To specify the full compilation config, '
760
- 'use a JSON string, e.g. ``{"level": 3, '
761
- '"cudagraph_capture_sizes": [1, 2, 4, 8]}``\n'
762
- 'Following the convention of traditional '
763
- 'compilers, using ``-O`` without space is also '
764
- 'supported. ``-O3`` is equivalent to ``-O 3``.')
765
-
766
- parser.add_argument('--kv-transfer-config',
767
- type=KVTransferConfig.from_cli,
768
- default=None,
769
- help='The configurations for distributed KV cache '
770
- 'transfer. Should be a JSON string.')
771
- parser.add_argument('--kv-events-config',
772
- type=KVEventsConfig.from_cli,
773
- default=None,
774
- help='The configurations for event publishing.')
775
-
776
- parser.add_argument(
777
- '--worker-cls',
778
- type=str,
779
- default="auto",
780
- help='The worker class to use for distributed execution.')
781
- parser.add_argument(
782
- '--worker-extension-cls',
783
- type=str,
784
- default="",
785
- help='The worker extension class on top of the worker cls, '
786
- 'it is useful if you just want to add new functions to the worker '
787
- 'class without changing the existing functions.')
788
-
789
- parser.add_argument(
736
+ scheduler_group.add_argument("--scheduler-cls",
737
+ **scheduler_kwargs["scheduler_cls"])
738
+
739
+ # Compilation arguments
740
+ # compilation_kwargs = get_kwargs(CompilationConfig)
741
+ compilation_group = parser.add_argument_group(
742
+ title="CompilationConfig",
743
+ description=CompilationConfig.__doc__,
744
+ )
745
+ compilation_group.add_argument(
746
+ "--compilation-config",
747
+ "-O",
748
+ type=CompilationConfig.from_cli,
749
+ default=None,
750
+ help="torch.compile configuration for the model. "
751
+ "When it is a number (0, 1, 2, 3), it will be "
752
+ "interpreted as the optimization level.\n"
753
+ "NOTE: level 0 is the default level without "
754
+ "any optimization. level 1 and 2 are for internal "
755
+ "testing only. level 3 is the recommended level "
756
+ "for production.\n"
757
+ "To specify the full compilation config, "
758
+ "use a JSON string, e.g. ``{\"level\": 3, "
759
+ "\"cudagraph_capture_sizes\": [1, 2, 4, 8]}``\n"
760
+ "Following the convention of traditional "
761
+ "compilers, using ``-O`` without space is also "
762
+ "supported. ``-O3`` is equivalent to ``-O 3``.")
763
+
764
+ # KVTransfer arguments
765
+ # kv_transfer_kwargs = get_kwargs(KVTransferConfig)
766
+ kv_transfer_group = parser.add_argument_group(
767
+ title="KVTransferConfig",
768
+ description=KVTransferConfig.__doc__,
769
+ )
770
+ kv_transfer_group.add_argument(
771
+ "--kv-transfer-config",
772
+ type=KVTransferConfig.from_cli,
773
+ default=None,
774
+ help="The configurations for distributed KV cache "
775
+ "transfer. Should be a JSON string.")
776
+ kv_transfer_group.add_argument(
777
+ '--kv-events-config',
778
+ type=KVEventsConfig.from_cli,
779
+ default=None,
780
+ help='The configurations for event publishing.')
781
+
782
+ # vLLM arguments
783
+ # vllm_kwargs = get_kwargs(VllmConfig)
784
+ vllm_group = parser.add_argument_group(
785
+ title="VllmConfig",
786
+ description=VllmConfig.__doc__,
787
+ )
788
+ vllm_group.add_argument(
789
"--additional-config",
790
type=json.loads,
791
default=None,
@@ -795,20 +794,18 @@ class EngineArgs:
794
"configs are valid for the platform you are using. The input format"
795
" is like '{\"config_key\":\"config_value\"}'")
796
798
- parser.add_argument(
799
- "--enable-reasoning",
800
- action="store_true",
801
- default=False,
802
- help=
803
- "[DEPRECATED] " \
804
- "The --enable-reasoning flag is deprecated as of v0.8.6. "
805
- "Use --reasoning-parser to specify " \
806
- "the reasoning parser backend instead. "
807
- "This flag (--enable-reasoning) will be " \
808
- "removed in v0.10.0. "
809
- "When --reasoning-parser is specified, " \
810
- "reasoning mode is automatically enabled."
811
- )
797
+ # Other arguments
798
+ parser.add_argument('--use-v2-block-manager',
799
+ action='store_true',
800
+ default=True,
801
+ help='[DEPRECATED] block manager v1 has been '
802
+ 'removed and SelfAttnBlockSpaceManager (i.e. '
803
+ 'block manager v2) is now the default. '
804
+ 'Setting this flag to True or False'
805
+ ' has no effect on vLLM behavior.')
806
+ parser.add_argument('--disable-log-stats',
807
+ action='store_true',
808
+ help='Disable logging statistics.')
809
810
return parser
811