A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fix `asyncio.Task` not being subscriptable (#4623)
Cyrus Leung committed
May 7, 2024 at 00:31 UTC
323f27b9048713cdbab31995265975842a937167
2 files changed
+5
-5
vllm/engine/async_llm_engine.py
+3
-3
@@ -1,8 +1,8 @@
1
import asyncio
2
import time
3
from functools import partial
4
-from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
5
- Optional, Set, Tuple, Type, Union)
4
+from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
5
+ Set, Tuple, Type, Union)
6
7
from transformers import PreTrainedTokenizer
8
@@ -327,7 +327,7 @@ class AsyncLLMEngine:
327
# We need to keep a reference to unshielded
328
# task as well to prevent it from being garbage
329
# collected
330
- self._background_loop_unshielded: Optional[asyncio.Task[Any]] = None
330
+ self._background_loop_unshielded: Optional[asyncio.Task] = None
331
self.start_engine_loop = start_engine_loop
332
self._errored_with: Optional[BaseException] = None
333
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -4,7 +4,7 @@ import inspect
4
import re
5
from contextlib import asynccontextmanager
6
from http import HTTPStatus
7
-from typing import Any, Set
7
+from typing import Set
8
9
import fastapi
10
import uvicorn
@@ -34,7 +34,7 @@ openai_serving_chat: OpenAIServingChat
34
openai_serving_completion: OpenAIServingCompletion
35
logger = init_logger(__name__)
36
37
-_running_tasks: Set[asyncio.Task[Any]] = set()
37
+_running_tasks: Set[asyncio.Task] = set()
38
39
40
@asynccontextmanager