A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix] Fix pydantic.errors.PydanticUserError (#17962)
Signed-off-by: wangli <wangli858794774@gmail.com>
Li Wang committed
May 12, 2025 at 12:58 UTC
19a3c78d1ff8c6cfab078227bad1c8cb79f47887
1 file changed
+7
-1
vllm/entrypoints/openai/serving_engine.py
+7
-1
@@ -1,18 +1,24 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
+import sys
5
import time
6
from collections.abc import (AsyncGenerator, Iterable, Iterator, Mapping,
7
Sequence)
8
from concurrent.futures.thread import ThreadPoolExecutor
9
from http import HTTPStatus
10
from typing import (Annotated, Any, Callable, ClassVar, Generic, Optional,
10
- TypedDict, TypeVar, Union)
11
+ TypeVar, Union)
12
13
from fastapi import Request
14
from pydantic import BaseModel, ConfigDict, Field
15
from starlette.datastructures import Headers
16
17
+if sys.version_info >= (3, 12):
18
+ from typing import TypedDict
19
+else:
20
+ from typing_extensions import TypedDict
21
+
22
import vllm.envs as envs
23
from vllm.config import ModelConfig
24
from vllm.engine.protocol import EngineClient