A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix typing in AsyncLLMEngine & add toml to requirements-dev (#2100)
mezuzza committed
Dec 14, 2023 at 03:19 UTC
6774bd50b0c19b3f33717d15763022df385a1b70
2 files changed
+8
-6
requirements-dev.txt
+1
@@ -1,5 +1,6 @@
1
# formatting
2
yapf==0.32.0
3
+toml==0.10.2
4
ruff==0.1.5
5
6
# type checking
vllm/engine/async_llm_engine.py
+7
-6
@@ -2,7 +2,7 @@ import asyncio
2
import time
3
from functools import partial
4
from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5
- Union)
5
+ Union, AsyncIterator)
6
7
from vllm.config import ModelConfig
8
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -401,11 +401,12 @@ class AsyncLLMEngine:
401
return stream
402
403
async def generate(
404
- self,
405
- prompt: Optional[str],
406
- sampling_params: SamplingParams,
407
- request_id: str,
408
- prompt_token_ids: Optional[List[int]] = None) -> RequestOutput:
404
+ self,
405
+ prompt: Optional[str],
406
+ sampling_params: SamplingParams,
407
+ request_id: str,
408
+ prompt_token_ids: Optional[List[int]] = None
409
+ ) -> AsyncIterator[RequestOutput]:
410
"""Generate outputs for a request.
411
412
Generate outputs for a request. This method is a coroutine. It adds the