@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 58fd57ff1d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix score api for missing max_model_len validation (#12119)

Signed-off-by: Wallas Santos <wallashss@ibm.com>

Wallas Henrique committed Jan 17, 2025 at 13:24 UTC 58fd57ff1d6d4ed56bed40aaaf9fe133b93b2efa
3 files changed +80 -33
tests/entrypoints/openai/test_score.py
+39 -6
@@ -12,6 +12,9 @@ MODEL_NAME = "BAAI/bge-reranker-v2-m3"
12 def server():
13 args = [
14 "--enforce-eager",
15 + # Will be used on tests to compare prompt input length
16 + "--max-model-len",
17 + "100"
18 ]
19
20 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -20,8 +23,7 @@ def server():
23
24 @pytest.mark.asyncio
25 @pytest.mark.parametrize("model_name", [MODEL_NAME])
23 -async def test_text_1_str_text_2_list(server: RemoteOpenAIServer,
24 - model_name: str):
26 +def test_text_1_str_text_2_list(server: RemoteOpenAIServer, model_name: str):
27 text_1 = "What is the capital of France?"
28 text_2 = [
29 "The capital of Brazil is Brasilia.", "The capital of France is Paris."
@@ -45,8 +47,7 @@ async def test_text_1_str_text_2_list(server: RemoteOpenAIServer,
47
48 @pytest.mark.asyncio
49 @pytest.mark.parametrize("model_name", [MODEL_NAME])
48 -async def test_text_1_list_text_2_list(server: RemoteOpenAIServer,
49 - model_name: str):
50 +def test_text_1_list_text_2_list(server: RemoteOpenAIServer, model_name: str):
51 text_1 = [
52 "What is the capital of the United States?",
53 "What is the capital of France?"
@@ -73,8 +74,7 @@ async def test_text_1_list_text_2_list(server: RemoteOpenAIServer,
74
75 @pytest.mark.asyncio
76 @pytest.mark.parametrize("model_name", [MODEL_NAME])
76 -async def test_text_1_str_text_2_str(server: RemoteOpenAIServer,
77 - model_name: str):
77 +def test_text_1_str_text_2_str(server: RemoteOpenAIServer, model_name: str):
78 text_1 = "What is the capital of France?"
79 text_2 = "The capital of France is Paris."
80
@@ -91,3 +91,36 @@ async def test_text_1_str_text_2_str(server: RemoteOpenAIServer,
91 assert score.data is not None
92 assert len(score.data) == 1
93 assert score.data[0].score >= 0.9
94 +
95 +
96 +@pytest.mark.asyncio
97 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
98 +def test_score_max_model_len(server: RemoteOpenAIServer, model_name: str):
99 +
100 + text_1 = "What is the capital of France?" * 20
101 + text_2 = [
102 + "The capital of Brazil is Brasilia.", "The capital of France is Paris."
103 + ]
104 +
105 + score_response = requests.post(server.url_for("score"),
106 + json={
107 + "model": model_name,
108 + "text_1": text_1,
109 + "text_2": text_2,
110 + })
111 + assert score_response.status_code == 400
112 + # Assert just a small fragments of the response
113 + assert "Please reduce the length of the input." in \
114 + score_response.text
115 +
116 + # Test truncation
117 + score_response = requests.post(server.url_for("score"),
118 + json={
119 + "model": model_name,
120 + "text_1": text_1,
121 + "text_2": text_2,
122 + "truncate_prompt_tokens": 101
123 + })
124 + assert score_response.status_code == 400
125 + assert "Please, select a smaller truncation size." in \
126 + score_response.text
vllm/entrypoints/openai/serving_engine.py
+9 -5
@@ -203,15 +203,19 @@ class OpenAIServing:
203 ) -> TextTokensPrompt:
204 token_num = len(input_ids)
205
206 - # Note: EmbeddingRequest doesn't have max_tokens
207 - if isinstance(request,
208 - (EmbeddingChatRequest, EmbeddingCompletionRequest)):
206 + # Note: EmbeddingRequest and ScoreRequest doesn't have max_tokens
207 + if isinstance(
208 + request,
209 + (EmbeddingChatRequest, EmbeddingCompletionRequest, ScoreRequest)):
210 +
211 + operation = "score" if isinstance(request, ScoreRequest) \
212 + else "embedding generation"
213 if token_num > self.max_model_len:
214 raise ValueError(
215 f"This model's maximum context length is "
216 f"{self.max_model_len} tokens. However, you requested "
213 - f"{token_num} tokens in the input for embedding "
214 - f"generation. Please reduce the length of the input.")
217 + f"{token_num} tokens in the input for {operation}. "
218 + f"Please reduce the length of the input.")
219 return TextTokensPrompt(prompt=input_text,
220 prompt_token_ids=input_ids)
221
vllm/entrypoints/openai/serving_score.py
+32 -22
@@ -101,6 +101,38 @@ class OpenAIServingScores(OpenAIServing):
101 if not self.model_config.is_cross_encoder:
102 raise ValueError("Model is not cross encoder.")
103
104 + if truncate_prompt_tokens is not None and \
105 + truncate_prompt_tokens > self.max_model_len:
106 + raise ValueError(
107 + f"truncate_prompt_tokens value ({truncate_prompt_tokens}) "
108 + f"is greater than max_model_len ({self.max_model_len})."
109 + f" Please, select a smaller truncation size.")
110 +
111 + input_pairs = make_pairs(request.text_1, request.text_2)
112 + for q, t in input_pairs:
113 + request_prompt = f"{q}{tokenizer.sep_token}{t}"
114 +
115 + tokenization_kwargs: Dict[str, Any] = {}
116 + if truncate_prompt_tokens is not None:
117 + tokenization_kwargs["truncation"] = True
118 + tokenization_kwargs["max_length"] = truncate_prompt_tokens
119 +
120 + tokenize_async = make_async(tokenizer.__call__,
121 + executor=self._tokenizer_executor)
122 + prompt_inputs = await tokenize_async(text=q,
123 + text_pair=t,
124 + **tokenization_kwargs)
125 +
126 + input_ids = prompt_inputs["input_ids"]
127 + text_token_prompt = \
128 + self._validate_input(request, input_ids, request_prompt)
129 + engine_prompt = TokensPrompt(
130 + prompt_token_ids=text_token_prompt["prompt_token_ids"],
131 + token_type_ids=prompt_inputs.get("token_type_ids"))
132 +
133 + request_prompts.append(request_prompt)
134 + engine_prompts.append(engine_prompt)
135 +
136 except ValueError as e:
137 logger.exception("Error in preprocessing prompt inputs")
138 return self.create_error_response(str(e))
@@ -108,28 +140,6 @@ class OpenAIServingScores(OpenAIServing):
140 # Schedule the request and get the result generator.
141 generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []
142
111 - input_pairs = make_pairs(request.text_1, request.text_2)
112 -
113 - for q, t in input_pairs:
114 - request_prompt = f"{q}{tokenizer.sep_token}{t}"
115 -
116 - tokenization_kwargs: Dict[str, Any] = {}
117 - if truncate_prompt_tokens is not None:
118 - tokenization_kwargs["truncation"] = True
119 - tokenization_kwargs["max_length"] = truncate_prompt_tokens
120 -
121 - tokenize_async = make_async(tokenizer.__call__,
122 - executor=self._tokenizer_executor)
123 - prompt_inputs = await tokenize_async(text=q,
124 - text_pair=t,
125 - **tokenization_kwargs)
126 - engine_prompt = TokensPrompt(
127 - prompt_token_ids=prompt_inputs["input_ids"],
128 - token_type_ids=prompt_inputs.get("token_type_ids"))
129 -
130 - request_prompts.append(request_prompt)
131 - engine_prompts.append(engine_prompt)
132 -
143 try:
144 pooling_params = request.to_pooling_params()
145