@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f936657eb6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Provide default max model length (#1224)

Woosuk Kwon committed Sep 28, 2023 at 14:44 UTC f936657eb67671ac594f285ebd5c90c3f8a6dee2
4 files changed +14 -9
vllm/config.py
+11 -7
@@ -164,9 +164,6 @@ class ModelConfig:
164 total_num_attention_heads = self.hf_config.num_attention_heads
165 return total_num_attention_heads // parallel_config.tensor_parallel_size
166
167 - def get_max_model_len(self) -> int:
168 - return self.max_model_len
169 -
167 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
168 total_num_hidden_layers = self.hf_config.num_hidden_layers
169 return total_num_hidden_layers // parallel_config.pipeline_parallel_size
@@ -378,10 +375,17 @@ def _get_and_verify_max_len(
375 if max_len_key is not None:
376 derived_max_model_len = min(derived_max_model_len, max_len_key)
377 if derived_max_model_len == float("inf"):
381 - raise ValueError(
382 - "The model's config.json must contain one of the following keys "
383 - "to determine the original maximum length of the model: "
384 - f"{possible_keys}")
378 + if max_model_len is not None:
379 + # If max_model_len is specified, we use it.
380 + return max_model_len
381 +
382 + default_max_len = 2048
383 + logger.warning(
384 + "The model's config.json does not contain any of the following "
385 + "keys to determine the original maximum length of the model: "
386 + f"{possible_keys}. Assuming the model's maximum length is "
387 + f"{default_max_len}.")
388 + derived_max_model_len = default_max_len
389
390 rope_scaling = getattr(hf_config, "rope_scaling", None)
391 if rope_scaling is not None:
vllm/engine/arg_utils.py
+1 -1
@@ -184,7 +184,7 @@ class EngineArgs:
184 self.worker_use_ray)
185 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
186 self.max_num_seqs,
187 - model_config.get_max_model_len())
187 + model_config.max_model_len)
188 return model_config, cache_config, parallel_config, scheduler_config
189
190
vllm/engine/llm_engine.py
+1
@@ -77,6 +77,7 @@ class LLMEngine:
77 f"revision={model_config.revision}, "
78 f"trust_remote_code={model_config.trust_remote_code}, "
79 f"dtype={model_config.dtype}, "
80 + f"max_seq_len={model_config.max_model_len}, "
81 f"download_dir={model_config.download_dir!r}, "
82 f"load_format={model_config.load_format}, "
83 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
vllm/entrypoints/openai/api_server.py
+1 -1
@@ -615,7 +615,7 @@ if __name__ == "__main__":
615 engine_args = AsyncEngineArgs.from_cli_args(args)
616 engine = AsyncLLMEngine.from_engine_args(engine_args)
617 engine_model_config = asyncio.run(engine.get_model_config())
618 - max_model_len = engine_model_config.get_max_model_len()
618 + max_model_len = engine_model_config.max_model_len
619
620 # A separate tokenizer to map token IDs to strings.
621 tokenizer = get_tokenizer(engine_args.tokenizer,