A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Doc] Add typing hints / mypy types cleanup (#3816)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Michael Feil committed
Apr 11, 2024 at 17:17 UTC
c2b4a1bce9a7707179cdfab2fb498c20b2b221e6
11 files changed
+90
-64
benchmarks/backend_request_func.py
+33
-29
@@ -27,8 +27,8 @@ class RequestFuncInput:
27
class RequestFuncOutput:
28
generated_text: str = ""
29
success: bool = False
30
- latency: float = 0
31
- ttft: float = 0 # Time to first token
30
+ latency: float = 0.0
31
+ ttft: float = 0.0 # Time to first token
32
itl: List[float] = field(
33
default_factory=list) # List of inter-token latencies
34
prompt_len: int = 0
@@ -58,23 +58,24 @@ async def async_request_tgi(
58
output = RequestFuncOutput()
59
output.prompt_len = request_func_input.prompt_len
60
61
- ttft = 0
61
+ ttft = 0.0
62
st = time.perf_counter()
63
most_recent_timestamp = st
64
try:
65
async with session.post(url=api_url, json=payload) as response:
66
if response.status == 200:
67
- async for chunk in response.content:
68
- chunk = chunk.strip()
69
- if not chunk:
67
+ async for chunk_bytes in response.content:
68
+ chunk_bytes = chunk_bytes.strip()
69
+ if not chunk_bytes:
70
continue
71
72
- chunk = remove_prefix(chunk.decode("utf-8"), "data:")
72
+ chunk = remove_prefix(chunk_bytes.decode("utf-8"),
73
+ "data:")
74
75
data = json.loads(chunk)
76
timestamp = time.perf_counter()
77
# First token
77
- if ttft == 0:
78
+ if ttft == 0.0:
79
ttft = time.perf_counter() - st
80
output.ttft = ttft
81
@@ -119,23 +120,24 @@ async def async_request_trt_llm(
120
output = RequestFuncOutput()
121
output.prompt_len = request_func_input.prompt_len
122
122
- ttft = 0
123
+ ttft = 0.0
124
st = time.perf_counter()
125
most_recent_timestamp = st
126
try:
127
async with session.post(url=api_url, json=payload) as response:
128
if response.status == 200:
128
- async for chunk in response.content:
129
- chunk = chunk.strip()
130
- if not chunk:
129
+ async for chunk_bytes in response.content:
130
+ chunk_bytes = chunk_bytes.strip()
131
+ if not chunk_bytes:
132
continue
133
133
- chunk = remove_prefix(chunk.decode("utf-8"), "data:")
134
+ chunk = remove_prefix(chunk_bytes.decode("utf-8"),
135
+ "data:")
136
137
data = json.loads(chunk)
138
timestamp = time.perf_counter()
139
# First token
138
- if ttft == 0:
140
+ if ttft == 0.0:
141
ttft = time.perf_counter() - st
142
output.ttft = ttft
143
@@ -151,7 +153,7 @@ async def async_request_trt_llm(
153
output.success = True
154
155
else:
154
- output.error = response.reason
156
+ output.error = response.reason or ""
157
output.success = False
158
except Exception:
159
output.success = False
@@ -195,7 +197,7 @@ async def async_request_deepspeed_mii(
197
output.generated_text = parsed_resp["text"][0]
198
output.success = True
199
else:
198
- output.error = response.reason
200
+ output.error = response.reason or ""
201
output.success = False
202
except Exception:
203
output.success = False
@@ -234,19 +236,20 @@ async def async_request_openai_completions(
236
output.prompt_len = request_func_input.prompt_len
237
238
generated_text = ""
237
- ttft = 0
239
+ ttft = 0.0
240
st = time.perf_counter()
241
most_recent_timestamp = st
242
try:
243
async with session.post(url=api_url, json=payload,
244
headers=headers) as response:
245
if response.status == 200:
244
- async for chunk in response.content:
245
- chunk = chunk.strip()
246
- if not chunk:
246
+ async for chunk_bytes in response.content:
247
+ chunk_bytes = chunk_bytes.strip()
248
+ if not chunk_bytes:
249
continue
250
249
- chunk = remove_prefix(chunk.decode("utf-8"), "data: ")
251
+ chunk = remove_prefix(chunk_bytes.decode("utf-8"),
252
+ "data: ")
253
if chunk == "[DONE]":
254
latency = time.perf_counter() - st
255
else:
@@ -255,7 +258,7 @@ async def async_request_openai_completions(
258
if data["choices"][0]["text"]:
259
timestamp = time.perf_counter()
260
# First token
258
- if ttft == 0:
261
+ if ttft == 0.0:
262
ttft = time.perf_counter() - st
263
output.ttft = ttft
264
@@ -315,19 +318,20 @@ async def async_request_openai_chat_completions(
318
output.prompt_len = request_func_input.prompt_len
319
320
generated_text = ""
318
- ttft = 0
321
+ ttft = 0.0
322
st = time.perf_counter()
323
most_recent_timestamp = st
324
try:
325
async with session.post(url=api_url, json=payload,
326
headers=headers) as response:
327
if response.status == 200:
325
- async for chunk in response.content:
326
- chunk = chunk.strip()
327
- if not chunk:
328
+ async for chunk_bytes in response.content:
329
+ chunk_bytes = chunk_bytes.strip()
330
+ if not chunk_bytes:
331
continue
332
330
- chunk = remove_prefix(chunk.decode("utf-8"), "data: ")
333
+ chunk = remove_prefix(chunk_bytes.decode("utf-8"),
334
+ "data: ")
335
if chunk == "[DONE]":
336
latency = time.perf_counter() - st
337
else:
@@ -337,7 +341,7 @@ async def async_request_openai_chat_completions(
341
delta = data["choices"][0]["delta"]
342
if delta.get("content", None):
343
# First token
340
- if ttft == 0:
344
+ if ttft == 0.0:
345
ttft = time.perf_counter() - st
346
output.ttft = ttft
347
@@ -354,7 +358,7 @@ async def async_request_openai_chat_completions(
358
output.success = True
359
output.latency = latency
360
else:
357
- output.error = response.reason
361
+ output.error = response.reason or ""
362
output.success = False
363
except Exception:
364
output.success = False
docs/source/conf.py
+2
-1
@@ -12,6 +12,7 @@
12
13
import logging
14
import sys
15
+from typing import List
16
17
from sphinx.ext import autodoc
18
@@ -45,7 +46,7 @@ templates_path = ['_templates']
46
# List of patterns, relative to source directory, that match files and
47
# directories to ignore when looking for source files.
48
# This pattern also affects html_static_path and html_extra_path.
48
-exclude_patterns = []
49
+exclude_patterns: List[str] = []
50
51
# Exclude the prompt "$" when copying code
52
copybutton_prompt_text = r"\$ "
setup.py
+3
-2
@@ -5,7 +5,7 @@ import re
5
import subprocess
6
import sys
7
from shutil import which
8
-from typing import List
8
+from typing import Dict, List
9
10
import torch
11
from packaging.version import Version, parse
@@ -52,7 +52,7 @@ class CMakeExtension(Extension):
52
53
class cmake_build_ext(build_ext):
54
# A dict of extension directories that have been configured.
55
- did_config = {}
55
+ did_config: Dict[str, bool] = {}
56
57
#
58
# Determine number of compilation jobs and optionally nvcc compile threads.
@@ -261,6 +261,7 @@ def get_nvcc_cuda_version() -> Version:
261
262
Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
263
"""
264
+ assert CUDA_HOME is not None, "CUDA_HOME is not set"
265
nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
266
universal_newlines=True)
267
output = nvcc_output.split()
vllm/core/block/interfaces.py
+19
-12
@@ -1,5 +1,5 @@
1
-from abc import ABC, abstractmethod, abstractproperty
2
-from typing import Dict, List, Optional, Protocol
1
+from abc import ABC, abstractmethod
2
+from typing import Dict, FrozenSet, List, Optional, Protocol
3
4
from vllm.utils import Device
5
@@ -10,23 +10,28 @@ class Block(ABC):
10
def append_token_ids(self, token_ids: List[int]) -> None:
11
pass
12
13
- @abstractproperty
13
+ @property
14
+ @abstractmethod
15
def block_id(self) -> Optional[int]:
16
pass
17
17
- @abstractproperty
18
+ @property
19
+ @abstractmethod
20
def token_ids(self) -> List[int]:
21
pass
22
21
- @abstractproperty
23
+ @property
24
+ @abstractmethod
25
def num_empty_slots(self) -> int:
26
pass
27
25
- @abstractproperty
28
+ @property
29
+ @abstractmethod
30
def is_full(self) -> bool:
31
pass
32
29
- @abstractproperty
33
+ @property
34
+ @abstractmethod
35
def prev_block(self) -> Optional["Block"]:
36
pass
37
@@ -47,12 +52,13 @@ class Block(ABC):
52
class BlockAllocator(ABC):
53
54
@abstractmethod
50
- def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
55
+ def allocate_mutable(self, prev_block: Optional[Block],
56
+ device: Device) -> Block:
57
pass
58
59
@abstractmethod
60
def allocate_immutable(self, prev_block: Optional[Block],
55
- token_ids: List[int]) -> Block:
61
+ token_ids: List[int], device: Device) -> Block:
62
pass
63
64
@abstractmethod
@@ -64,11 +70,12 @@ class BlockAllocator(ABC):
70
pass
71
72
@abstractmethod
67
- def get_num_free_blocks(self) -> int:
73
+ def get_num_free_blocks(self, device: Device) -> int:
74
pass
75
70
- @abstractproperty
71
- def all_block_ids(self) -> frozenset[int]:
76
+ @property
77
+ @abstractmethod
78
+ def all_block_ids(self) -> FrozenSet[int]:
79
pass
80
81
@abstractmethod
vllm/engine/metrics.py
+8
-2
@@ -1,6 +1,6 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import Dict, List
3
+from typing import Dict, List, Protocol
4
5
import numpy as np
6
from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
@@ -119,6 +119,12 @@ class Stats:
119
time_e2e_requests: List[float]
120
121
122
+class SupportsMetricsInfo(Protocol):
123
+
124
+ def metrics_info(self) -> Dict[str, str]:
125
+ ...
126
+
127
+
128
class StatLogger:
129
"""StatLogger is used LLMEngine to log to Promethus and Stdout."""
130
@@ -135,7 +141,7 @@ class StatLogger:
141
self.labels = labels
142
self.metrics = Metrics(labelnames=list(labels.keys()))
143
138
- def info(self, type: str, obj: object) -> None:
144
+ def info(self, type: str, obj: SupportsMetricsInfo) -> None:
145
if type == "cache_config":
146
self.metrics.info_cache_config.info(obj.metrics_info())
147
vllm/logger.py
+7
-1
@@ -4,6 +4,7 @@
4
import logging
5
import os
6
import sys
7
+from typing import Optional
8
9
VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
10
@@ -26,7 +27,7 @@ class NewLineFormatter(logging.Formatter):
27
28
29
_root_logger = logging.getLogger("vllm")
29
-_default_handler = None
30
+_default_handler: Optional[logging.Handler] = None
31
32
33
def _setup_logger():
@@ -55,7 +56,12 @@ def init_logger(name: str):
56
# Use the same settings as above for root logger
57
logger = logging.getLogger(name)
58
logger.setLevel(os.getenv("LOG_LEVEL", "DEBUG"))
59
+
60
if VLLM_CONFIGURE_LOGGING:
61
+ if _default_handler is None:
62
+ raise ValueError(
63
+ "_default_handler is not set up. This should never happen!"
64
+ " Please open an issue on Github.")
65
logger.addHandler(_default_handler)
66
logger.propagate = False
67
return logger
vllm/model_executor/layers/rotary_embedding.py
+8
-7
@@ -247,11 +247,12 @@ def _yarn_find_correction_dim(num_rotations: int,
247
248
249
# Find dim range bounds based on rotations
250
-def _yarn_find_correction_range(low_rot: int,
251
- high_rot: int,
252
- dim: int,
253
- base: float = 10000,
254
- max_position_embeddings: int = 2048) -> int:
250
+def _yarn_find_correction_range(
251
+ low_rot: int,
252
+ high_rot: int,
253
+ dim: int,
254
+ base: float = 10000,
255
+ max_position_embeddings: int = 2048) -> Tuple[int, int]:
256
low = math.floor(
257
_yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
258
high = math.ceil(
@@ -293,8 +294,8 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
294
*,
295
extrapolation_factor: float = 1,
296
attn_factor: float = 1,
296
- beta_fast: float = 32,
297
- beta_slow: float = 1,
297
+ beta_fast: int = 32,
298
+ beta_slow: int = 1,
299
) -> None:
300
self.scaling_factor = scaling_factor
301
self.extrapolation_factor = extrapolation_factor
vllm/transformers_utils/config.py
+2
-2
@@ -1,10 +1,10 @@
1
-from typing import Optional
1
+from typing import Dict, Optional
2
3
from transformers import AutoConfig, PretrainedConfig
4
5
from vllm.transformers_utils.configs import *
6
7
-_CONFIG_REGISTRY = {
7
+_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
8
"chatglm": ChatGLMConfig,
9
"dbrx": DbrxConfig,
10
"mpt": MPTConfig,
vllm/transformers_utils/configs/dbrx.py
+1
-1
@@ -12,7 +12,7 @@ from transformers.utils import logging
12
13
logger = logging.get_logger(__name__)
14
15
-DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
15
+DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore
16
17
18
class DbrxAttentionConfig(PretrainedConfig):
vllm/transformers_utils/tokenizers/baichuan.py
+5
-5
@@ -16,11 +16,11 @@ logger = logging.get_logger(__name__)
16
17
VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
18
19
-PRETRAINED_VOCAB_FILES_MAP = {
19
+PRETRAINED_VOCAB_FILES_MAP = { # type: ignore
20
"vocab_file": {},
21
"tokenizer_file": {},
22
}
23
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
23
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} # type: ignore
24
25
26
class BaichuanTokenizer(PreTrainedTokenizer):
@@ -148,9 +148,9 @@ class BaichuanTokenizer(PreTrainedTokenizer):
148
`Tuple(str)`: Paths to the files saved.
149
"""
150
if not os.path.isdir(save_directory):
151
- logger.error(f"Vocabulary path ({save_directory}) "
152
- "should be a directory")
153
- return
151
+ raise ValueError(f"Vocabulary path ({save_directory}) "
152
+ "should be a directory")
153
+
154
out_vocab_file = os.path.join(
155
save_directory,
156
(filename_prefix + "-" if filename_prefix else "") +
vllm/utils.py
+2
-2
@@ -294,7 +294,7 @@ def create_kv_caches_with_random(
294
head_size: int,
295
cache_dtype: Optional[Union[str, torch.dtype]],
296
model_dtype: Optional[Union[str, torch.dtype]] = None,
297
- seed: Optional[int] = 0,
297
+ seed: int = 0,
298
device: Optional[str] = "cuda",
299
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
300
torch.random.manual_seed(seed)
@@ -400,7 +400,7 @@ class CudaMemoryProfiler:
400
gc.collect()
401
402
403
-def str_to_int_tuple(s: str) -> Tuple[int]:
403
+def str_to_int_tuple(s: str) -> Tuple[int, ...]:
404
"""Convert a string to a tuple of integers."""
405
try:
406
return tuple(map(int, s.split(",")))