@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 71d63ed72e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

migrate pydantic from v1 to v2 (#2531)

migrate pydantic from v1 to v2 (#2531)

Jannis Schönleber committed Jan 22, 2024 at 01:05 UTC 71d63ed72e8e528e3131af79b2c1ad853065ccb8
7 files changed +26 -22
requirements-neuron.txt
+1 -1
@@ -5,5 +5,5 @@ torch-neuronx >= 2.1.0
5 neuronx-cc
6 fastapi
7 uvicorn[standard]
8 -pydantic == 1.10.13 # Required for OpenAI server.
8 +pydantic >= 2.0 # Required for OpenAI server.
9 aioprometheus[starlette]
requirements-rocm.txt
+1 -1
@@ -9,5 +9,5 @@ tokenizers>=0.15.0
9 transformers >= 4.36.0 # Required for Mixtral.
10 fastapi
11 uvicorn[standard]
12 -pydantic == 1.10.13 # Required for OpenAI server.
12 +pydantic >= 2.0 # Required for OpenAI server.
13 aioprometheus[starlette]
requirements.txt
+1 -1
@@ -8,5 +8,5 @@ transformers >= 4.36.0 # Required for Mixtral.
8 xformers == 0.0.23.post1 # Required for CUDA 12.1.
9 fastapi
10 uvicorn[standard]
11 -pydantic == 1.10.13 # Required for OpenAI server.
11 +pydantic >= 2.0 # Required for OpenAI server.
12 aioprometheus[starlette]
vllm/entrypoints/openai/api_server.py
+12 -6
@@ -106,7 +106,7 @@ app.add_route("/metrics", metrics) # Exposes HTTP metrics
106 @app.exception_handler(RequestValidationError)
107 async def validation_exception_handler(_, exc):
108 err = openai_serving_chat.create_error_response(message=str(exc))
109 - return JSONResponse(err.dict(), status_code=HTTPStatus.BAD_REQUEST)
109 + return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
110
111
112 @app.get("/health")
@@ -118,7 +118,7 @@ async def health() -> Response:
118 @app.get("/v1/models")
119 async def show_available_models():
120 models = await openai_serving_chat.show_available_models()
121 - return JSONResponse(content=models.dict())
121 + return JSONResponse(content=models.model_dump())
122
123
124 @app.post("/v1/chat/completions")
@@ -126,22 +126,28 @@ async def create_chat_completion(request: ChatCompletionRequest,
126 raw_request: Request):
127 generator = await openai_serving_chat.create_chat_completion(
128 request, raw_request)
129 - if request.stream and not isinstance(generator, ErrorResponse):
129 + if isinstance(generator, ErrorResponse):
130 + return JSONResponse(content=generator.model_dump(),
131 + status_code=generator.code)
132 + if request.stream:
133 return StreamingResponse(content=generator,
134 media_type="text/event-stream")
135 else:
133 - return JSONResponse(content=generator.dict())
136 + return JSONResponse(content=generator.model_dump())
137
138
139 @app.post("/v1/completions")
140 async def create_completion(request: CompletionRequest, raw_request: Request):
141 generator = await openai_serving_completion.create_completion(
142 request, raw_request)
140 - if request.stream and not isinstance(generator, ErrorResponse):
143 + if isinstance(generator, ErrorResponse):
144 + return JSONResponse(content=generator.model_dump(),
145 + status_code=generator.code)
146 + if request.stream:
147 return StreamingResponse(content=generator,
148 media_type="text/event-stream")
149 else:
144 - return JSONResponse(content=generator.dict())
150 + return JSONResponse(content=generator.model_dump())
151
152
153 if __name__ == "__main__":
vllm/entrypoints/openai/protocol.py
+3 -4
@@ -14,7 +14,7 @@ class ErrorResponse(BaseModel):
14 message: str
15 type: str
16 param: Optional[str] = None
17 - code: Optional[str] = None
17 + code: int
18
19
20 class ModelPermission(BaseModel):
@@ -189,7 +189,7 @@ class CompletionStreamResponse(BaseModel):
189 created: int = Field(default_factory=lambda: int(time.time()))
190 model: str
191 choices: List[CompletionResponseStreamChoice]
192 - usage: Optional[UsageInfo]
192 + usage: Optional[UsageInfo] = Field(default=None)
193
194
195 class ChatMessage(BaseModel):
@@ -229,5 +229,4 @@ class ChatCompletionStreamResponse(BaseModel):
229 created: int = Field(default_factory=lambda: int(time.time()))
230 model: str
231 choices: List[ChatCompletionResponseStreamChoice]
232 - usage: Optional[UsageInfo] = Field(
233 - default=None, description="data about request and response")
232 + usage: Optional[UsageInfo] = Field(default=None)
vllm/entrypoints/openai/serving_chat.py
+5 -6
@@ -102,7 +102,7 @@ class OpenAIServingChat(OpenAIServing):
102 created=created_time,
103 choices=[choice_data],
104 model=model_name)
105 - data = chunk.json(exclude_unset=True, ensure_ascii=False)
105 + data = chunk.model_dump_json(exclude_unset=True)
106 yield f"data: {data}\n\n"
107
108 # Send response to echo the input portion of the last message
@@ -125,7 +125,7 @@ class OpenAIServingChat(OpenAIServing):
125 created=created_time,
126 choices=[choice_data],
127 model=model_name)
128 - data = chunk.json(exclude_unset=True, ensure_ascii=False)
128 + data = chunk.model_dump_json(exclude_unset=True)
129 yield f"data: {data}\n\n"
130
131 # Send response for each token for each request.n (index)
@@ -156,7 +156,7 @@ class OpenAIServingChat(OpenAIServing):
156 created=created_time,
157 choices=[choice_data],
158 model=model_name)
159 - data = chunk.json(exclude_unset=True, ensure_ascii=False)
159 + data = chunk.model_dump_json(exclude_unset=True)
160 yield f"data: {data}\n\n"
161 else:
162 # Send the finish response for each request.n only once
@@ -178,9 +178,8 @@ class OpenAIServingChat(OpenAIServing):
178 model=model_name)
179 if final_usage is not None:
180 chunk.usage = final_usage
181 - data = chunk.json(exclude_unset=True,
182 - exclude_none=True,
183 - ensure_ascii=False)
181 + data = chunk.model_dump_json(exclude_unset=True,
182 + exclude_none=True)
183 yield f"data: {data}\n\n"
184 finish_reason_sent[i] = True
185 # Send the final done message after all response.n are finished
vllm/entrypoints/openai/serving_completion.py
+3 -3
@@ -74,7 +74,7 @@ async def completion_stream_generator(
74 logprobs=logprobs,
75 finish_reason=finish_reason,
76 )
77 - ]).json(exclude_unset=True, ensure_ascii=False)
77 + ]).model_dump_json(exclude_unset=True)
78 yield f"data: {response_json}\n\n"
79
80 if output.finish_reason is not None:
@@ -99,7 +99,7 @@ async def completion_stream_generator(
99 )
100 ],
101 usage=final_usage,
102 - ).json(exclude_unset=True, ensure_ascii=False)
102 + ).model_dump_json(exclude_unset=True)
103 yield f"data: {response_json}\n\n"
104
105 yield "data: [DONE]\n\n"
@@ -279,7 +279,7 @@ class OpenAIServingCompletion(OpenAIServing):
279 # When user requests streaming but we don't stream, we still need to
280 # return a streaming response with a single event.
281 if request.stream:
282 - response_json = response.json(ensure_ascii=False)
282 + response_json = response.model_dump_json()
283
284 async def fake_stream_generator() -> AsyncGenerator[str, None]:
285 yield f"data: {response_json}\n\n"