@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 323f27b904
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fix `asyncio.Task` not being subscriptable (#4623)

Cyrus Leung committed May 7, 2024 at 00:31 UTC 323f27b9048713cdbab31995265975842a937167
2 files changed +5 -5
vllm/engine/async_llm_engine.py
+3 -3
@@ -1,8 +1,8 @@
1 import asyncio
2 import time
3 from functools import partial
4 -from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
5 - Optional, Set, Tuple, Type, Union)
4 +from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
5 + Set, Tuple, Type, Union)
6
7 from transformers import PreTrainedTokenizer
8
@@ -327,7 +327,7 @@ class AsyncLLMEngine:
327 # We need to keep a reference to unshielded
328 # task as well to prevent it from being garbage
329 # collected
330 - self._background_loop_unshielded: Optional[asyncio.Task[Any]] = None
330 + self._background_loop_unshielded: Optional[asyncio.Task] = None
331 self.start_engine_loop = start_engine_loop
332 self._errored_with: Optional[BaseException] = None
333
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -4,7 +4,7 @@ import inspect
4 import re
5 from contextlib import asynccontextmanager
6 from http import HTTPStatus
7 -from typing import Any, Set
7 +from typing import Set
8
9 import fastapi
10 import uvicorn
@@ -34,7 +34,7 @@ openai_serving_chat: OpenAIServingChat
34 openai_serving_completion: OpenAIServingCompletion
35 logger = init_logger(__name__)
36
37 -_running_tasks: Set[asyncio.Task[Any]] = set()
37 +_running_tasks: Set[asyncio.Task] = set()
38
39
40 @asynccontextmanager