@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 065aff6c16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Make EngineArgs use named arguments for config construction (#5285)

Michael Goin committed Jun 5, 2024 at 18:16 UTC 065aff6c16bd563acc449b1dae6a99256cfbdeb8
1 file changed +40 -26
vllm/engine/arg_utils.py
+40 -26
@@ -619,34 +619,48 @@ class EngineArgs:
619 "BitsAndBytes load format and QLoRA adapter only support "
620 f"'bitsandbytes' quantization, but got {self.quantization}")
621
622 - device_config = DeviceConfig(self.device)
622 + device_config = DeviceConfig(device=self.device)
623 model_config = ModelConfig(
624 - self.model, self.tokenizer, self.tokenizer_mode,
625 - self.trust_remote_code, self.dtype, self.seed, self.revision,
626 - self.code_revision, self.rope_scaling, self.tokenizer_revision,
627 - self.max_model_len, self.quantization,
628 - self.quantization_param_path, self.enforce_eager,
629 - self.max_context_len_to_capture, self.max_seq_len_to_capture,
630 - self.max_logprobs, self.disable_sliding_window,
631 - self.skip_tokenizer_init, self.served_model_name)
632 - cache_config = CacheConfig(self.block_size,
633 - self.gpu_memory_utilization,
634 - self.swap_space, self.kv_cache_dtype,
635 - self.num_gpu_blocks_override,
636 - model_config.get_sliding_window(),
637 - self.enable_prefix_caching)
624 + model=self.model,
625 + tokenizer=self.tokenizer,
626 + tokenizer_mode=self.tokenizer_mode,
627 + trust_remote_code=self.trust_remote_code,
628 + dtype=self.dtype,
629 + seed=self.seed,
630 + revision=self.revision,
631 + code_revision=self.code_revision,
632 + rope_scaling=self.rope_scaling,
633 + tokenizer_revision=self.tokenizer_revision,
634 + max_model_len=self.max_model_len,
635 + quantization=self.quantization,
636 + quantization_param_path=self.quantization_param_path,
637 + enforce_eager=self.enforce_eager,
638 + max_context_len_to_capture=self.max_context_len_to_capture,
639 + max_seq_len_to_capture=self.max_seq_len_to_capture,
640 + max_logprobs=self.max_logprobs,
641 + disable_sliding_window=self.disable_sliding_window,
642 + skip_tokenizer_init=self.skip_tokenizer_init,
643 + served_model_name=self.served_model_name)
644 + cache_config = CacheConfig(
645 + block_size=self.block_size,
646 + gpu_memory_utilization=self.gpu_memory_utilization,
647 + swap_space=self.swap_space,
648 + cache_dtype=self.kv_cache_dtype,
649 + num_gpu_blocks_override=self.num_gpu_blocks_override,
650 + sliding_window=model_config.get_sliding_window(),
651 + enable_prefix_caching=self.enable_prefix_caching)
652 parallel_config = ParallelConfig(
639 - self.pipeline_parallel_size,
640 - self.tensor_parallel_size,
641 - self.worker_use_ray,
642 - self.max_parallel_loading_workers,
643 - self.disable_custom_all_reduce,
644 - TokenizerPoolConfig.create_config(
653 + pipeline_parallel_size=self.pipeline_parallel_size,
654 + tensor_parallel_size=self.tensor_parallel_size,
655 + worker_use_ray=self.worker_use_ray,
656 + max_parallel_loading_workers=self.max_parallel_loading_workers,
657 + disable_custom_all_reduce=self.disable_custom_all_reduce,
658 + tokenizer_pool_config=TokenizerPoolConfig.create_config(
659 self.tokenizer_pool_size,
660 self.tokenizer_pool_type,
661 self.tokenizer_pool_extra_config,
662 ),
649 - self.ray_workers_use_nsight,
663 + ray_workers_use_nsight=self.ray_workers_use_nsight,
664 distributed_executor_backend=self.distributed_executor_backend)
665
666 speculative_config = SpeculativeConfig.maybe_create_spec_config(
@@ -665,10 +679,10 @@ class EngineArgs:
679 )
680
681 scheduler_config = SchedulerConfig(
668 - self.max_num_batched_tokens,
669 - self.max_num_seqs,
670 - model_config.max_model_len,
671 - self.use_v2_block_manager,
682 + max_num_batched_tokens=self.max_num_batched_tokens,
683 + max_num_seqs=self.max_num_seqs,
684 + max_model_len=model_config.max_model_len,
685 + use_v2_block_manager=self.use_v2_block_manager,
686 num_lookahead_slots=(self.num_lookahead_slots
687 if speculative_config is None else
688 speculative_config.num_lookahead_slots),