@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 58a072be15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Add model sequence length into model config (#575)

Zhuohan Li committed Jul 25, 2023 at 23:46 UTC 58a072be15a4e36bee006d1c9a962e527819cf18
3 files changed +27 -18
vllm/config.py
+20
@@ -109,6 +109,26 @@ class ModelConfig:
109 total_num_attention_heads = self.hf_config.num_attention_heads
110 return total_num_attention_heads // parallel_config.tensor_parallel_size
111
112 + def get_max_model_len(self) -> int:
113 + max_model_len = float("inf")
114 + possible_keys = [
115 + # OPT
116 + "max_position_embeddings",
117 + # GPT-2
118 + "n_positions",
119 + # MPT
120 + "max_seq_len",
121 + # Others
122 + "max_sequence_length",
123 + "max_seq_length",
124 + "seq_len",
125 + ]
126 + for key in possible_keys:
127 + max_len_key = getattr(self.hf_config, key, None)
128 + if max_len_key is not None:
129 + max_model_len = min(max_model_len, max_len_key)
130 + return max_model_len
131 +
132 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
133 total_num_hidden_layers = self.hf_config.num_hidden_layers
134 return total_num_hidden_layers // parallel_config.pipeline_parallel_size
vllm/engine/arg_utils.py
+2 -3
@@ -155,10 +155,9 @@ class EngineArgs:
155 parallel_config = ParallelConfig(self.pipeline_parallel_size,
156 self.tensor_parallel_size,
157 self.worker_use_ray)
158 - max_model_len = getattr(model_config.hf_config,
159 - 'max_position_embeddings', float('inf'))
158 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
161 - self.max_num_seqs, max_model_len)
159 + self.max_num_seqs,
160 + model_config.get_max_model_len())
161 return model_config, cache_config, parallel_config, scheduler_config
162
163
vllm/entrypoints/openai/api_server.py
+5 -15
@@ -107,25 +107,14 @@ async def get_gen_prompt(request) -> str:
107 return prompt
108
109
110 -async def check_length(request, prompt, model_config):
111 - if hasattr(model_config.hf_config, "max_sequence_length"):
112 - context_len = model_config.hf_config.max_sequence_length
113 - elif hasattr(model_config.hf_config, "seq_length"):
114 - context_len = model_config.hf_config.seq_length
115 - elif hasattr(model_config.hf_config, "max_position_embeddings"):
116 - context_len = model_config.hf_config.max_position_embeddings
117 - elif hasattr(model_config.hf_config, "seq_length"):
118 - context_len = model_config.hf_config.seq_length
119 - else:
120 - context_len = 2048
121 -
110 +async def check_length(request, prompt):
111 input_ids = tokenizer(prompt).input_ids
112 token_num = len(input_ids)
113
125 - if token_num + request.max_tokens > context_len:
114 + if token_num + request.max_tokens > max_model_len:
115 return create_error_response(
116 HTTPStatus.BAD_REQUEST,
128 - f"This model's maximum context length is {context_len} tokens. "
117 + f"This model's maximum context length is {max_model_len} tokens. "
118 f"However, you requested {request.max_tokens + token_num} tokens "
119 f"({token_num} in the messages, "
120 f"{request.max_tokens} in the completion). "
@@ -194,7 +183,7 @@ async def create_chat_completion(raw_request: Request):
183 "logit_bias is not currently supported")
184
185 prompt = await get_gen_prompt(request)
197 - error_check_ret = await check_length(request, prompt, engine_model_config)
186 + error_check_ret = await check_length(request, prompt)
187 if error_check_ret is not None:
188 return error_check_ret
189
@@ -591,6 +580,7 @@ if __name__ == "__main__":
580 engine_args = AsyncEngineArgs.from_cli_args(args)
581 engine = AsyncLLMEngine.from_engine_args(engine_args)
582 engine_model_config = asyncio.run(engine.get_model_config())
583 + max_model_len = engine_model_config.get_max_model_len()
584
585 # A separate tokenizer to map token IDs to strings.
586 tokenizer = get_tokenizer(engine_args.tokenizer,