A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
migrate pydantic from v1 to v2 (#2531)
migrate pydantic from v1 to v2 (#2531)
Jannis Schönleber committed
Jan 22, 2024 at 01:05 UTC
71d63ed72e8e528e3131af79b2c1ad853065ccb8
7 files changed
+26
-22
requirements-neuron.txt
+1
-1
@@ -5,5 +5,5 @@ torch-neuronx >= 2.1.0
5
neuronx-cc
6
fastapi
7
uvicorn[standard]
8
-pydantic == 1.10.13 # Required for OpenAI server.
8
+pydantic >= 2.0 # Required for OpenAI server.
9
aioprometheus[starlette]
requirements-rocm.txt
+1
-1
@@ -9,5 +9,5 @@ tokenizers>=0.15.0
9
transformers >= 4.36.0 # Required for Mixtral.
10
fastapi
11
uvicorn[standard]
12
-pydantic == 1.10.13 # Required for OpenAI server.
12
+pydantic >= 2.0 # Required for OpenAI server.
13
aioprometheus[starlette]
requirements.txt
+1
-1
@@ -8,5 +8,5 @@ transformers >= 4.36.0 # Required for Mixtral.
8
xformers == 0.0.23.post1 # Required for CUDA 12.1.
9
fastapi
10
uvicorn[standard]
11
-pydantic == 1.10.13 # Required for OpenAI server.
11
+pydantic >= 2.0 # Required for OpenAI server.
12
aioprometheus[starlette]
vllm/entrypoints/openai/api_server.py
+12
-6
@@ -106,7 +106,7 @@ app.add_route("/metrics", metrics) # Exposes HTTP metrics
106
@app.exception_handler(RequestValidationError)
107
async def validation_exception_handler(_, exc):
108
err = openai_serving_chat.create_error_response(message=str(exc))
109
- return JSONResponse(err.dict(), status_code=HTTPStatus.BAD_REQUEST)
109
+ return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
110
111
112
@app.get("/health")
@@ -118,7 +118,7 @@ async def health() -> Response:
118
@app.get("/v1/models")
119
async def show_available_models():
120
models = await openai_serving_chat.show_available_models()
121
- return JSONResponse(content=models.dict())
121
+ return JSONResponse(content=models.model_dump())
122
123
124
@app.post("/v1/chat/completions")
@@ -126,22 +126,28 @@ async def create_chat_completion(request: ChatCompletionRequest,
126
raw_request: Request):
127
generator = await openai_serving_chat.create_chat_completion(
128
request, raw_request)
129
- if request.stream and not isinstance(generator, ErrorResponse):
129
+ if isinstance(generator, ErrorResponse):
130
+ return JSONResponse(content=generator.model_dump(),
131
+ status_code=generator.code)
132
+ if request.stream:
133
return StreamingResponse(content=generator,
134
media_type="text/event-stream")
135
else:
133
- return JSONResponse(content=generator.dict())
136
+ return JSONResponse(content=generator.model_dump())
137
138
139
@app.post("/v1/completions")
140
async def create_completion(request: CompletionRequest, raw_request: Request):
141
generator = await openai_serving_completion.create_completion(
142
request, raw_request)
140
- if request.stream and not isinstance(generator, ErrorResponse):
143
+ if isinstance(generator, ErrorResponse):
144
+ return JSONResponse(content=generator.model_dump(),
145
+ status_code=generator.code)
146
+ if request.stream:
147
return StreamingResponse(content=generator,
148
media_type="text/event-stream")
149
else:
144
- return JSONResponse(content=generator.dict())
150
+ return JSONResponse(content=generator.model_dump())
151
152
153
if __name__ == "__main__":
vllm/entrypoints/openai/protocol.py
+3
-4
@@ -14,7 +14,7 @@ class ErrorResponse(BaseModel):
14
message: str
15
type: str
16
param: Optional[str] = None
17
- code: Optional[str] = None
17
+ code: int
18
19
20
class ModelPermission(BaseModel):
@@ -189,7 +189,7 @@ class CompletionStreamResponse(BaseModel):
189
created: int = Field(default_factory=lambda: int(time.time()))
190
model: str
191
choices: List[CompletionResponseStreamChoice]
192
- usage: Optional[UsageInfo]
192
+ usage: Optional[UsageInfo] = Field(default=None)
193
194
195
class ChatMessage(BaseModel):
@@ -229,5 +229,4 @@ class ChatCompletionStreamResponse(BaseModel):
229
created: int = Field(default_factory=lambda: int(time.time()))
230
model: str
231
choices: List[ChatCompletionResponseStreamChoice]
232
- usage: Optional[UsageInfo] = Field(
233
- default=None, description="data about request and response")
232
+ usage: Optional[UsageInfo] = Field(default=None)
vllm/entrypoints/openai/serving_chat.py
+5
-6
@@ -102,7 +102,7 @@ class OpenAIServingChat(OpenAIServing):
102
created=created_time,
103
choices=[choice_data],
104
model=model_name)
105
- data = chunk.json(exclude_unset=True, ensure_ascii=False)
105
+ data = chunk.model_dump_json(exclude_unset=True)
106
yield f"data: {data}\n\n"
107
108
# Send response to echo the input portion of the last message
@@ -125,7 +125,7 @@ class OpenAIServingChat(OpenAIServing):
125
created=created_time,
126
choices=[choice_data],
127
model=model_name)
128
- data = chunk.json(exclude_unset=True, ensure_ascii=False)
128
+ data = chunk.model_dump_json(exclude_unset=True)
129
yield f"data: {data}\n\n"
130
131
# Send response for each token for each request.n (index)
@@ -156,7 +156,7 @@ class OpenAIServingChat(OpenAIServing):
156
created=created_time,
157
choices=[choice_data],
158
model=model_name)
159
- data = chunk.json(exclude_unset=True, ensure_ascii=False)
159
+ data = chunk.model_dump_json(exclude_unset=True)
160
yield f"data: {data}\n\n"
161
else:
162
# Send the finish response for each request.n only once
@@ -178,9 +178,8 @@ class OpenAIServingChat(OpenAIServing):
178
model=model_name)
179
if final_usage is not None:
180
chunk.usage = final_usage
181
- data = chunk.json(exclude_unset=True,
182
- exclude_none=True,
183
- ensure_ascii=False)
181
+ data = chunk.model_dump_json(exclude_unset=True,
182
+ exclude_none=True)
183
yield f"data: {data}\n\n"
184
finish_reason_sent[i] = True
185
# Send the final done message after all response.n are finished
vllm/entrypoints/openai/serving_completion.py
+3
-3
@@ -74,7 +74,7 @@ async def completion_stream_generator(
74
logprobs=logprobs,
75
finish_reason=finish_reason,
76
)
77
- ]).json(exclude_unset=True, ensure_ascii=False)
77
+ ]).model_dump_json(exclude_unset=True)
78
yield f"data: {response_json}\n\n"
79
80
if output.finish_reason is not None:
@@ -99,7 +99,7 @@ async def completion_stream_generator(
99
)
100
],
101
usage=final_usage,
102
- ).json(exclude_unset=True, ensure_ascii=False)
102
+ ).model_dump_json(exclude_unset=True)
103
yield f"data: {response_json}\n\n"
104
105
yield "data: [DONE]\n\n"
@@ -279,7 +279,7 @@ class OpenAIServingCompletion(OpenAIServing):
279
# When user requests streaming but we don't stream, we still need to
280
# return a streaming response with a single event.
281
if request.stream:
282
- response_json = response.json(ensure_ascii=False)
282
+ response_json = response.model_dump_json()
283
284
async def fake_stream_generator() -> AsyncGenerator[str, None]:
285
yield f"data: {response_json}\n\n"