@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6774bd50b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix typing in AsyncLLMEngine & add toml to requirements-dev (#2100)

mezuzza committed Dec 14, 2023 at 03:19 UTC 6774bd50b0c19b3f33717d15763022df385a1b70
2 files changed +8 -6
requirements-dev.txt
+1
@@ -1,5 +1,6 @@
1 # formatting
2 yapf==0.32.0
3 +toml==0.10.2
4 ruff==0.1.5
5
6 # type checking
vllm/engine/async_llm_engine.py
+7 -6
@@ -2,7 +2,7 @@ import asyncio
2 import time
3 from functools import partial
4 from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5 - Union)
5 + Union, AsyncIterator)
6
7 from vllm.config import ModelConfig
8 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -401,11 +401,12 @@ class AsyncLLMEngine:
401 return stream
402
403 async def generate(
404 - self,
405 - prompt: Optional[str],
406 - sampling_params: SamplingParams,
407 - request_id: str,
408 - prompt_token_ids: Optional[List[int]] = None) -> RequestOutput:
404 + self,
405 + prompt: Optional[str],
406 + sampling_params: SamplingParams,
407 + request_id: str,
408 + prompt_token_ids: Optional[List[int]] = None
409 + ) -> AsyncIterator[RequestOutput]:
410 """Generate outputs for a request.
411
412 Generate outputs for a request. This method is a coroutine. It adds the