A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Make EngineArgs use named arguments for config construction (#5285)
Michael Goin committed
Jun 5, 2024 at 18:16 UTC
065aff6c16bd563acc449b1dae6a99256cfbdeb8
1 file changed
+40
-26
vllm/engine/arg_utils.py
+40
-26
@@ -619,34 +619,48 @@ class EngineArgs:
619
"BitsAndBytes load format and QLoRA adapter only support "
620
f"'bitsandbytes' quantization, but got {self.quantization}")
621
622
- device_config = DeviceConfig(self.device)
622
+ device_config = DeviceConfig(device=self.device)
623
model_config = ModelConfig(
624
- self.model, self.tokenizer, self.tokenizer_mode,
625
- self.trust_remote_code, self.dtype, self.seed, self.revision,
626
- self.code_revision, self.rope_scaling, self.tokenizer_revision,
627
- self.max_model_len, self.quantization,
628
- self.quantization_param_path, self.enforce_eager,
629
- self.max_context_len_to_capture, self.max_seq_len_to_capture,
630
- self.max_logprobs, self.disable_sliding_window,
631
- self.skip_tokenizer_init, self.served_model_name)
632
- cache_config = CacheConfig(self.block_size,
633
- self.gpu_memory_utilization,
634
- self.swap_space, self.kv_cache_dtype,
635
- self.num_gpu_blocks_override,
636
- model_config.get_sliding_window(),
637
- self.enable_prefix_caching)
624
+ model=self.model,
625
+ tokenizer=self.tokenizer,
626
+ tokenizer_mode=self.tokenizer_mode,
627
+ trust_remote_code=self.trust_remote_code,
628
+ dtype=self.dtype,
629
+ seed=self.seed,
630
+ revision=self.revision,
631
+ code_revision=self.code_revision,
632
+ rope_scaling=self.rope_scaling,
633
+ tokenizer_revision=self.tokenizer_revision,
634
+ max_model_len=self.max_model_len,
635
+ quantization=self.quantization,
636
+ quantization_param_path=self.quantization_param_path,
637
+ enforce_eager=self.enforce_eager,
638
+ max_context_len_to_capture=self.max_context_len_to_capture,
639
+ max_seq_len_to_capture=self.max_seq_len_to_capture,
640
+ max_logprobs=self.max_logprobs,
641
+ disable_sliding_window=self.disable_sliding_window,
642
+ skip_tokenizer_init=self.skip_tokenizer_init,
643
+ served_model_name=self.served_model_name)
644
+ cache_config = CacheConfig(
645
+ block_size=self.block_size,
646
+ gpu_memory_utilization=self.gpu_memory_utilization,
647
+ swap_space=self.swap_space,
648
+ cache_dtype=self.kv_cache_dtype,
649
+ num_gpu_blocks_override=self.num_gpu_blocks_override,
650
+ sliding_window=model_config.get_sliding_window(),
651
+ enable_prefix_caching=self.enable_prefix_caching)
652
parallel_config = ParallelConfig(
639
- self.pipeline_parallel_size,
640
- self.tensor_parallel_size,
641
- self.worker_use_ray,
642
- self.max_parallel_loading_workers,
643
- self.disable_custom_all_reduce,
644
- TokenizerPoolConfig.create_config(
653
+ pipeline_parallel_size=self.pipeline_parallel_size,
654
+ tensor_parallel_size=self.tensor_parallel_size,
655
+ worker_use_ray=self.worker_use_ray,
656
+ max_parallel_loading_workers=self.max_parallel_loading_workers,
657
+ disable_custom_all_reduce=self.disable_custom_all_reduce,
658
+ tokenizer_pool_config=TokenizerPoolConfig.create_config(
659
self.tokenizer_pool_size,
660
self.tokenizer_pool_type,
661
self.tokenizer_pool_extra_config,
662
),
649
- self.ray_workers_use_nsight,
663
+ ray_workers_use_nsight=self.ray_workers_use_nsight,
664
distributed_executor_backend=self.distributed_executor_backend)
665
666
speculative_config = SpeculativeConfig.maybe_create_spec_config(
@@ -665,10 +679,10 @@ class EngineArgs:
679
)
680
681
scheduler_config = SchedulerConfig(
668
- self.max_num_batched_tokens,
669
- self.max_num_seqs,
670
- model_config.max_model_len,
671
- self.use_v2_block_manager,
682
+ max_num_batched_tokens=self.max_num_batched_tokens,
683
+ max_num_seqs=self.max_num_seqs,
684
+ max_model_len=model_config.max_model_len,
685
+ use_v2_block_manager=self.use_v2_block_manager,
686
num_lookahead_slots=(self.num_lookahead_slots
687
if speculative_config is None else
688
speculative_config.num_lookahead_slots),