@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2b4a1bce9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Doc] Add typing hints / mypy types cleanup (#3816)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Michael Feil committed Apr 11, 2024 at 17:17 UTC c2b4a1bce9a7707179cdfab2fb498c20b2b221e6
11 files changed +90 -64
benchmarks/backend_request_func.py
+33 -29
@@ -27,8 +27,8 @@ class RequestFuncInput:
27 class RequestFuncOutput:
28 generated_text: str = ""
29 success: bool = False
30 - latency: float = 0
31 - ttft: float = 0 # Time to first token
30 + latency: float = 0.0
31 + ttft: float = 0.0 # Time to first token
32 itl: List[float] = field(
33 default_factory=list) # List of inter-token latencies
34 prompt_len: int = 0
@@ -58,23 +58,24 @@ async def async_request_tgi(
58 output = RequestFuncOutput()
59 output.prompt_len = request_func_input.prompt_len
60
61 - ttft = 0
61 + ttft = 0.0
62 st = time.perf_counter()
63 most_recent_timestamp = st
64 try:
65 async with session.post(url=api_url, json=payload) as response:
66 if response.status == 200:
67 - async for chunk in response.content:
68 - chunk = chunk.strip()
69 - if not chunk:
67 + async for chunk_bytes in response.content:
68 + chunk_bytes = chunk_bytes.strip()
69 + if not chunk_bytes:
70 continue
71
72 - chunk = remove_prefix(chunk.decode("utf-8"), "data:")
72 + chunk = remove_prefix(chunk_bytes.decode("utf-8"),
73 + "data:")
74
75 data = json.loads(chunk)
76 timestamp = time.perf_counter()
77 # First token
77 - if ttft == 0:
78 + if ttft == 0.0:
79 ttft = time.perf_counter() - st
80 output.ttft = ttft
81
@@ -119,23 +120,24 @@ async def async_request_trt_llm(
120 output = RequestFuncOutput()
121 output.prompt_len = request_func_input.prompt_len
122
122 - ttft = 0
123 + ttft = 0.0
124 st = time.perf_counter()
125 most_recent_timestamp = st
126 try:
127 async with session.post(url=api_url, json=payload) as response:
128 if response.status == 200:
128 - async for chunk in response.content:
129 - chunk = chunk.strip()
130 - if not chunk:
129 + async for chunk_bytes in response.content:
130 + chunk_bytes = chunk_bytes.strip()
131 + if not chunk_bytes:
132 continue
133
133 - chunk = remove_prefix(chunk.decode("utf-8"), "data:")
134 + chunk = remove_prefix(chunk_bytes.decode("utf-8"),
135 + "data:")
136
137 data = json.loads(chunk)
138 timestamp = time.perf_counter()
139 # First token
138 - if ttft == 0:
140 + if ttft == 0.0:
141 ttft = time.perf_counter() - st
142 output.ttft = ttft
143
@@ -151,7 +153,7 @@ async def async_request_trt_llm(
153 output.success = True
154
155 else:
154 - output.error = response.reason
156 + output.error = response.reason or ""
157 output.success = False
158 except Exception:
159 output.success = False
@@ -195,7 +197,7 @@ async def async_request_deepspeed_mii(
197 output.generated_text = parsed_resp["text"][0]
198 output.success = True
199 else:
198 - output.error = response.reason
200 + output.error = response.reason or ""
201 output.success = False
202 except Exception:
203 output.success = False
@@ -234,19 +236,20 @@ async def async_request_openai_completions(
236 output.prompt_len = request_func_input.prompt_len
237
238 generated_text = ""
237 - ttft = 0
239 + ttft = 0.0
240 st = time.perf_counter()
241 most_recent_timestamp = st
242 try:
243 async with session.post(url=api_url, json=payload,
244 headers=headers) as response:
245 if response.status == 200:
244 - async for chunk in response.content:
245 - chunk = chunk.strip()
246 - if not chunk:
246 + async for chunk_bytes in response.content:
247 + chunk_bytes = chunk_bytes.strip()
248 + if not chunk_bytes:
249 continue
250
249 - chunk = remove_prefix(chunk.decode("utf-8"), "data: ")
251 + chunk = remove_prefix(chunk_bytes.decode("utf-8"),
252 + "data: ")
253 if chunk == "[DONE]":
254 latency = time.perf_counter() - st
255 else:
@@ -255,7 +258,7 @@ async def async_request_openai_completions(
258 if data["choices"][0]["text"]:
259 timestamp = time.perf_counter()
260 # First token
258 - if ttft == 0:
261 + if ttft == 0.0:
262 ttft = time.perf_counter() - st
263 output.ttft = ttft
264
@@ -315,19 +318,20 @@ async def async_request_openai_chat_completions(
318 output.prompt_len = request_func_input.prompt_len
319
320 generated_text = ""
318 - ttft = 0
321 + ttft = 0.0
322 st = time.perf_counter()
323 most_recent_timestamp = st
324 try:
325 async with session.post(url=api_url, json=payload,
326 headers=headers) as response:
327 if response.status == 200:
325 - async for chunk in response.content:
326 - chunk = chunk.strip()
327 - if not chunk:
328 + async for chunk_bytes in response.content:
329 + chunk_bytes = chunk_bytes.strip()
330 + if not chunk_bytes:
331 continue
332
330 - chunk = remove_prefix(chunk.decode("utf-8"), "data: ")
333 + chunk = remove_prefix(chunk_bytes.decode("utf-8"),
334 + "data: ")
335 if chunk == "[DONE]":
336 latency = time.perf_counter() - st
337 else:
@@ -337,7 +341,7 @@ async def async_request_openai_chat_completions(
341 delta = data["choices"][0]["delta"]
342 if delta.get("content", None):
343 # First token
340 - if ttft == 0:
344 + if ttft == 0.0:
345 ttft = time.perf_counter() - st
346 output.ttft = ttft
347
@@ -354,7 +358,7 @@ async def async_request_openai_chat_completions(
358 output.success = True
359 output.latency = latency
360 else:
357 - output.error = response.reason
361 + output.error = response.reason or ""
362 output.success = False
363 except Exception:
364 output.success = False
docs/source/conf.py
+2 -1
@@ -12,6 +12,7 @@
12
13 import logging
14 import sys
15 +from typing import List
16
17 from sphinx.ext import autodoc
18
@@ -45,7 +46,7 @@ templates_path = ['_templates']
46 # List of patterns, relative to source directory, that match files and
47 # directories to ignore when looking for source files.
48 # This pattern also affects html_static_path and html_extra_path.
48 -exclude_patterns = []
49 +exclude_patterns: List[str] = []
50
51 # Exclude the prompt "$" when copying code
52 copybutton_prompt_text = r"\$ "
setup.py
+3 -2
@@ -5,7 +5,7 @@ import re
5 import subprocess
6 import sys
7 from shutil import which
8 -from typing import List
8 +from typing import Dict, List
9
10 import torch
11 from packaging.version import Version, parse
@@ -52,7 +52,7 @@ class CMakeExtension(Extension):
52
53 class cmake_build_ext(build_ext):
54 # A dict of extension directories that have been configured.
55 - did_config = {}
55 + did_config: Dict[str, bool] = {}
56
57 #
58 # Determine number of compilation jobs and optionally nvcc compile threads.
@@ -261,6 +261,7 @@ def get_nvcc_cuda_version() -> Version:
261
262 Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
263 """
264 + assert CUDA_HOME is not None, "CUDA_HOME is not set"
265 nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
266 universal_newlines=True)
267 output = nvcc_output.split()
vllm/core/block/interfaces.py
+19 -12
@@ -1,5 +1,5 @@
1 -from abc import ABC, abstractmethod, abstractproperty
2 -from typing import Dict, List, Optional, Protocol
1 +from abc import ABC, abstractmethod
2 +from typing import Dict, FrozenSet, List, Optional, Protocol
3
4 from vllm.utils import Device
5
@@ -10,23 +10,28 @@ class Block(ABC):
10 def append_token_ids(self, token_ids: List[int]) -> None:
11 pass
12
13 - @abstractproperty
13 + @property
14 + @abstractmethod
15 def block_id(self) -> Optional[int]:
16 pass
17
17 - @abstractproperty
18 + @property
19 + @abstractmethod
20 def token_ids(self) -> List[int]:
21 pass
22
21 - @abstractproperty
23 + @property
24 + @abstractmethod
25 def num_empty_slots(self) -> int:
26 pass
27
25 - @abstractproperty
28 + @property
29 + @abstractmethod
30 def is_full(self) -> bool:
31 pass
32
29 - @abstractproperty
33 + @property
34 + @abstractmethod
35 def prev_block(self) -> Optional["Block"]:
36 pass
37
@@ -47,12 +52,13 @@ class Block(ABC):
52 class BlockAllocator(ABC):
53
54 @abstractmethod
50 - def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
55 + def allocate_mutable(self, prev_block: Optional[Block],
56 + device: Device) -> Block:
57 pass
58
59 @abstractmethod
60 def allocate_immutable(self, prev_block: Optional[Block],
55 - token_ids: List[int]) -> Block:
61 + token_ids: List[int], device: Device) -> Block:
62 pass
63
64 @abstractmethod
@@ -64,11 +70,12 @@ class BlockAllocator(ABC):
70 pass
71
72 @abstractmethod
67 - def get_num_free_blocks(self) -> int:
73 + def get_num_free_blocks(self, device: Device) -> int:
74 pass
75
70 - @abstractproperty
71 - def all_block_ids(self) -> frozenset[int]:
76 + @property
77 + @abstractmethod
78 + def all_block_ids(self) -> FrozenSet[int]:
79 pass
80
81 @abstractmethod
vllm/engine/metrics.py
+8 -2
@@ -1,6 +1,6 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import Dict, List
3 +from typing import Dict, List, Protocol
4
5 import numpy as np
6 from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
@@ -119,6 +119,12 @@ class Stats:
119 time_e2e_requests: List[float]
120
121
122 +class SupportsMetricsInfo(Protocol):
123 +
124 + def metrics_info(self) -> Dict[str, str]:
125 + ...
126 +
127 +
128 class StatLogger:
129 """StatLogger is used LLMEngine to log to Promethus and Stdout."""
130
@@ -135,7 +141,7 @@ class StatLogger:
141 self.labels = labels
142 self.metrics = Metrics(labelnames=list(labels.keys()))
143
138 - def info(self, type: str, obj: object) -> None:
144 + def info(self, type: str, obj: SupportsMetricsInfo) -> None:
145 if type == "cache_config":
146 self.metrics.info_cache_config.info(obj.metrics_info())
147
vllm/logger.py
+7 -1
@@ -4,6 +4,7 @@
4 import logging
5 import os
6 import sys
7 +from typing import Optional
8
9 VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
10
@@ -26,7 +27,7 @@ class NewLineFormatter(logging.Formatter):
27
28
29 _root_logger = logging.getLogger("vllm")
29 -_default_handler = None
30 +_default_handler: Optional[logging.Handler] = None
31
32
33 def _setup_logger():
@@ -55,7 +56,12 @@ def init_logger(name: str):
56 # Use the same settings as above for root logger
57 logger = logging.getLogger(name)
58 logger.setLevel(os.getenv("LOG_LEVEL", "DEBUG"))
59 +
60 if VLLM_CONFIGURE_LOGGING:
61 + if _default_handler is None:
62 + raise ValueError(
63 + "_default_handler is not set up. This should never happen!"
64 + " Please open an issue on Github.")
65 logger.addHandler(_default_handler)
66 logger.propagate = False
67 return logger
vllm/model_executor/layers/rotary_embedding.py
+8 -7
@@ -247,11 +247,12 @@ def _yarn_find_correction_dim(num_rotations: int,
247
248
249 # Find dim range bounds based on rotations
250 -def _yarn_find_correction_range(low_rot: int,
251 - high_rot: int,
252 - dim: int,
253 - base: float = 10000,
254 - max_position_embeddings: int = 2048) -> int:
250 +def _yarn_find_correction_range(
251 + low_rot: int,
252 + high_rot: int,
253 + dim: int,
254 + base: float = 10000,
255 + max_position_embeddings: int = 2048) -> Tuple[int, int]:
256 low = math.floor(
257 _yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
258 high = math.ceil(
@@ -293,8 +294,8 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
294 *,
295 extrapolation_factor: float = 1,
296 attn_factor: float = 1,
296 - beta_fast: float = 32,
297 - beta_slow: float = 1,
297 + beta_fast: int = 32,
298 + beta_slow: int = 1,
299 ) -> None:
300 self.scaling_factor = scaling_factor
301 self.extrapolation_factor = extrapolation_factor
vllm/transformers_utils/config.py
+2 -2
@@ -1,10 +1,10 @@
1 -from typing import Optional
1 +from typing import Dict, Optional
2
3 from transformers import AutoConfig, PretrainedConfig
4
5 from vllm.transformers_utils.configs import *
6
7 -_CONFIG_REGISTRY = {
7 +_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
8 "chatglm": ChatGLMConfig,
9 "dbrx": DbrxConfig,
10 "mpt": MPTConfig,
vllm/transformers_utils/configs/dbrx.py
+1 -1
@@ -12,7 +12,7 @@ from transformers.utils import logging
12
13 logger = logging.get_logger(__name__)
14
15 -DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
15 +DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore
16
17
18 class DbrxAttentionConfig(PretrainedConfig):
vllm/transformers_utils/tokenizers/baichuan.py
+5 -5
@@ -16,11 +16,11 @@ logger = logging.get_logger(__name__)
16
17 VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
18
19 -PRETRAINED_VOCAB_FILES_MAP = {
19 +PRETRAINED_VOCAB_FILES_MAP = { # type: ignore
20 "vocab_file": {},
21 "tokenizer_file": {},
22 }
23 -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
23 +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} # type: ignore
24
25
26 class BaichuanTokenizer(PreTrainedTokenizer):
@@ -148,9 +148,9 @@ class BaichuanTokenizer(PreTrainedTokenizer):
148 `Tuple(str)`: Paths to the files saved.
149 """
150 if not os.path.isdir(save_directory):
151 - logger.error(f"Vocabulary path ({save_directory}) "
152 - "should be a directory")
153 - return
151 + raise ValueError(f"Vocabulary path ({save_directory}) "
152 + "should be a directory")
153 +
154 out_vocab_file = os.path.join(
155 save_directory,
156 (filename_prefix + "-" if filename_prefix else "") +
vllm/utils.py
+2 -2
@@ -294,7 +294,7 @@ def create_kv_caches_with_random(
294 head_size: int,
295 cache_dtype: Optional[Union[str, torch.dtype]],
296 model_dtype: Optional[Union[str, torch.dtype]] = None,
297 - seed: Optional[int] = 0,
297 + seed: int = 0,
298 device: Optional[str] = "cuda",
299 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
300 torch.random.manual_seed(seed)
@@ -400,7 +400,7 @@ class CudaMemoryProfiler:
400 gc.collect()
401
402
403 -def str_to_int_tuple(s: str) -> Tuple[int]:
403 +def str_to_int_tuple(s: str) -> Tuple[int, ...]:
404 """Convert a string to a tuple of integers."""
405 try:
406 return tuple(map(int, s.split(",")))