A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Add model sequence length into model config (#575)
Zhuohan Li committed
Jul 25, 2023 at 23:46 UTC
58a072be15a4e36bee006d1c9a962e527819cf18
3 files changed
+27
-18
vllm/config.py
+20
@@ -109,6 +109,26 @@ class ModelConfig:
109
total_num_attention_heads = self.hf_config.num_attention_heads
110
return total_num_attention_heads // parallel_config.tensor_parallel_size
111
112
+ def get_max_model_len(self) -> int:
113
+ max_model_len = float("inf")
114
+ possible_keys = [
115
+ # OPT
116
+ "max_position_embeddings",
117
+ # GPT-2
118
+ "n_positions",
119
+ # MPT
120
+ "max_seq_len",
121
+ # Others
122
+ "max_sequence_length",
123
+ "max_seq_length",
124
+ "seq_len",
125
+ ]
126
+ for key in possible_keys:
127
+ max_len_key = getattr(self.hf_config, key, None)
128
+ if max_len_key is not None:
129
+ max_model_len = min(max_model_len, max_len_key)
130
+ return max_model_len
131
+
132
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
133
total_num_hidden_layers = self.hf_config.num_hidden_layers
134
return total_num_hidden_layers // parallel_config.pipeline_parallel_size
vllm/engine/arg_utils.py
+2
-3
@@ -155,10 +155,9 @@ class EngineArgs:
155
parallel_config = ParallelConfig(self.pipeline_parallel_size,
156
self.tensor_parallel_size,
157
self.worker_use_ray)
158
- max_model_len = getattr(model_config.hf_config,
159
- 'max_position_embeddings', float('inf'))
158
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
161
- self.max_num_seqs, max_model_len)
159
+ self.max_num_seqs,
160
+ model_config.get_max_model_len())
161
return model_config, cache_config, parallel_config, scheduler_config
162
163
vllm/entrypoints/openai/api_server.py
+5
-15
@@ -107,25 +107,14 @@ async def get_gen_prompt(request) -> str:
107
return prompt
108
109
110
-async def check_length(request, prompt, model_config):
111
- if hasattr(model_config.hf_config, "max_sequence_length"):
112
- context_len = model_config.hf_config.max_sequence_length
113
- elif hasattr(model_config.hf_config, "seq_length"):
114
- context_len = model_config.hf_config.seq_length
115
- elif hasattr(model_config.hf_config, "max_position_embeddings"):
116
- context_len = model_config.hf_config.max_position_embeddings
117
- elif hasattr(model_config.hf_config, "seq_length"):
118
- context_len = model_config.hf_config.seq_length
119
- else:
120
- context_len = 2048
121
-
110
+async def check_length(request, prompt):
111
input_ids = tokenizer(prompt).input_ids
112
token_num = len(input_ids)
113
125
- if token_num + request.max_tokens > context_len:
114
+ if token_num + request.max_tokens > max_model_len:
115
return create_error_response(
116
HTTPStatus.BAD_REQUEST,
128
- f"This model's maximum context length is {context_len} tokens. "
117
+ f"This model's maximum context length is {max_model_len} tokens. "
118
f"However, you requested {request.max_tokens + token_num} tokens "
119
f"({token_num} in the messages, "
120
f"{request.max_tokens} in the completion). "
@@ -194,7 +183,7 @@ async def create_chat_completion(raw_request: Request):
183
"logit_bias is not currently supported")
184
185
prompt = await get_gen_prompt(request)
197
- error_check_ret = await check_length(request, prompt, engine_model_config)
186
+ error_check_ret = await check_length(request, prompt)
187
if error_check_ret is not None:
188
return error_check_ret
189
@@ -591,6 +580,7 @@ if __name__ == "__main__":
580
engine_args = AsyncEngineArgs.from_cli_args(args)
581
engine = AsyncLLMEngine.from_engine_args(engine_args)
582
engine_model_config = asyncio.run(engine.get_model_config())
583
+ max_model_len = engine_model_config.get_max_model_len()
584
585
# A separate tokenizer to map token IDs to strings.
586
tokenizer = get_tokenizer(engine_args.tokenizer,