A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[mypy] Fix mypy warnings in api_server.py (#11941)
Signed-off-by: Fred Reiss <frreiss@us.ibm.com>
Fred Reiss committed
Jan 10, 2025 at 17:04 UTC
c9f09a4fe83ef13824ea1663214ac7aad08d2b31
1 file changed
+5
-3
vllm/entrypoints/openai/api_server.py
+5
-3
@@ -14,7 +14,7 @@ from argparse import Namespace
14
from contextlib import asynccontextmanager
15
from functools import partial
16
from http import HTTPStatus
17
-from typing import AsyncIterator, Optional, Set, Tuple
17
+from typing import AsyncIterator, Dict, Optional, Set, Tuple, Union
18
19
import uvloop
20
from fastapi import APIRouter, FastAPI, HTTPException, Request
@@ -420,6 +420,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
420
"use the Pooling API (`/pooling`) instead.")
421
422
res = await fallback_handler.create_pooling(request, raw_request)
423
+
424
+ generator: Union[ErrorResponse, EmbeddingResponse]
425
if isinstance(res, PoolingResponse):
426
generator = EmbeddingResponse(
427
id=res.id,
@@ -494,7 +496,7 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
496
return await create_score(request, raw_request)
497
498
497
-TASK_HANDLERS = {
499
+TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {
500
"generate": {
501
"messages": (ChatCompletionRequest, create_chat_completion),
502
"default": (CompletionRequest, create_completion),
@@ -652,7 +654,7 @@ def build_app(args: Namespace) -> FastAPI:
654
module_path, object_name = middleware.rsplit(".", 1)
655
imported = getattr(importlib.import_module(module_path), object_name)
656
if inspect.isclass(imported):
655
- app.add_middleware(imported)
657
+ app.add_middleware(imported) # type: ignore[arg-type]
658
elif inspect.iscoroutinefunction(imported):
659
app.middleware("http")(imported)
660
else: