@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c664b0e683
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

fix some bugs (#2689)

zspo committed Feb 1, 2024 at 02:09 UTC c664b0e6838644c22839b6e9c88e61b4e9a540f6
2 files changed +8 -3
vllm/config.py
+4 -1
@@ -355,6 +355,9 @@ class ParallelConfig:
355 worker_use_ray: Whether to use Ray for model workers. Will be set to
356 True if either pipeline_parallel_size or tensor_parallel_size is
357 greater than 1.
358 + max_parallel_loading_workers: Maximum number of multiple batches
359 + when load model sequentially. To avoid RAM OOM when using tensor
360 + parallel and large models.
361 disable_custom_all_reduce: Disable the custom all-reduce kernel and
362 fall back to NCCL.
363 """
@@ -470,7 +473,7 @@ class LoRAConfig:
473 elif self.max_cpu_loras < self.max_loras:
474 raise ValueError(
475 f"max_cpu_loras ({self.max_cpu_loras}) must be >= "
473 - f"max_num_seqs ({self.max_loras})")
476 + f"max_loras ({self.max_loras})")
477
478 def verify_with_model_config(self, model_config: ModelConfig):
479 if self.lora_dtype in (None, "auto"):
vllm/engine/async_llm_engine.py
+4 -2
@@ -296,6 +296,8 @@ class AsyncLLMEngine:
296 async frontend will be executed in a separate process as the
297 model workers.
298 log_requests: Whether to log the requests.
299 + max_log_len: Maximum number of prompt characters or prompt ID numbers
300 + being printed in log.
301 start_engine_loop: If True, the background task to run the engine
302 will be automatically started in the generate call.
303 *args: Arguments for LLMEngine.
@@ -431,8 +433,8 @@ class AsyncLLMEngine:
433 logger.info(f"Received request {request_id}: "
434 f"prompt: {shortened_prompt!r}, "
435 f"prefix_pos: {prefix_pos},"
434 - f"sampling params: {sampling_params}, "
435 - f"prompt token ids: {shortened_token_ids}, "
436 + f"sampling_params: {sampling_params}, "
437 + f"prompt_token_ids: {shortened_token_ids}, "
438 f"lora_request: {lora_request}.")
439
440 if not self.is_running: