@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9f09a4fe8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[mypy] Fix mypy warnings in api_server.py (#11941)

Signed-off-by: Fred Reiss <frreiss@us.ibm.com>

Fred Reiss committed Jan 10, 2025 at 17:04 UTC c9f09a4fe83ef13824ea1663214ac7aad08d2b31
1 file changed +5 -3
vllm/entrypoints/openai/api_server.py
+5 -3
@@ -14,7 +14,7 @@ from argparse import Namespace
14 from contextlib import asynccontextmanager
15 from functools import partial
16 from http import HTTPStatus
17 -from typing import AsyncIterator, Optional, Set, Tuple
17 +from typing import AsyncIterator, Dict, Optional, Set, Tuple, Union
18
19 import uvloop
20 from fastapi import APIRouter, FastAPI, HTTPException, Request
@@ -420,6 +420,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
420 "use the Pooling API (`/pooling`) instead.")
421
422 res = await fallback_handler.create_pooling(request, raw_request)
423 +
424 + generator: Union[ErrorResponse, EmbeddingResponse]
425 if isinstance(res, PoolingResponse):
426 generator = EmbeddingResponse(
427 id=res.id,
@@ -494,7 +496,7 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
496 return await create_score(request, raw_request)
497
498
497 -TASK_HANDLERS = {
499 +TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {
500 "generate": {
501 "messages": (ChatCompletionRequest, create_chat_completion),
502 "default": (CompletionRequest, create_completion),
@@ -652,7 +654,7 @@ def build_app(args: Namespace) -> FastAPI:
654 module_path, object_name = middleware.rsplit(".", 1)
655 imported = getattr(importlib.import_module(module_path), object_name)
656 if inspect.isclass(imported):
655 - app.add_middleware(imported)
657 + app.add_middleware(imported) # type: ignore[arg-type]
658 elif inspect.iscoroutinefunction(imported):
659 app.middleware("http")(imported)
660 else: