A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc] Use dataclass for InputMetadata (#3452)
Co-authored-by: youkaichao <youkaichao@126.com>
Woosuk Kwon committed
Mar 17, 2024 at 03:02 UTC
abfc4f3387c436d46d6701e9ba916de8f9ed9329
3 files changed
+24
-63
setup.py
-1
@@ -2,7 +2,6 @@ import contextlib
2
import io
3
import os
4
import re
5
-import shutil
5
import subprocess
6
import warnings
7
from pathlib import Path
vllm/model_executor/input_metadata.py
+14
-35
@@ -1,8 +1,10 @@
1
+from dataclasses import dataclass
2
from typing import Optional
3
4
import torch
5
6
7
+@dataclass
8
class InputMetadata:
9
"""Metadata for input sequences. Used in PagedAttention.
10
@@ -15,40 +17,17 @@ class InputMetadata:
17
kv_cache_dtype: Data type to store kv cache.
18
"""
19
18
- def __init__(
19
- self,
20
- is_prompt: bool,
21
- slot_mapping: torch.Tensor,
22
- prompt_lens: Optional[torch.Tensor],
23
- max_seq_len: Optional[int],
24
- start_loc: Optional[torch.Tensor],
25
- max_context_len: Optional[int],
26
- context_lens: Optional[torch.Tensor],
27
- block_tables: Optional[torch.Tensor],
28
- use_cuda_graph: bool,
29
- kv_cache_dtype: str,
30
- ) -> None:
31
- self.is_prompt = is_prompt
32
- self.prompt_lens = prompt_lens
33
- self.max_seq_len = max_seq_len
34
- self.start_loc = start_loc
35
- self.max_context_len = max_context_len
36
- self.slot_mapping = slot_mapping
37
- self.context_lens = context_lens
38
- self.block_tables = block_tables
39
- self.use_cuda_graph = use_cuda_graph
40
- self.kv_cache_dtype = kv_cache_dtype
20
+ is_prompt: bool
21
+ slot_mapping: torch.Tensor
22
+ prompt_lens: Optional[torch.Tensor]
23
+ max_seq_len: Optional[int]
24
+ start_loc: Optional[torch.Tensor]
25
+ max_context_len: Optional[int]
26
+ context_lens: Optional[torch.Tensor]
27
+ block_tables: Optional[torch.Tensor]
28
+ use_cuda_graph: bool
29
+ kv_cache_dtype: str
30
42
- # Set during the execution of the first attention op.
43
- # FIXME(woosuk): This is a hack.
31
+ def __post_init__(self):
32
+ # will not appear in the __repr__ and __init__
33
self.attn_bias = None
45
-
46
- def __repr__(self) -> str:
47
- return ("InputMetadata("
48
- f"is_prompt={self.is_prompt}, "
49
- f"max_context_len={self.max_context_len}, "
50
- f"slot_mapping={self.slot_mapping}, "
51
- f"context_lens={self.context_lens}, "
52
- f"block_tables={self.block_tables}, "
53
- f"use_cuda_graph={self.use_cuda_graph}, "
54
- f"kv_cache_dtype={self.kv_cache_dtype})")
vllm/worker/model_runner.py
+10
-27
@@ -1,4 +1,5 @@
1
import contextlib
2
+import dataclasses
3
import time
4
from typing import Dict, List, Optional, Tuple, Set, Union
5
@@ -521,45 +522,27 @@ class ModelRunner:
522
metadata_dict = {
523
"input_tokens": input_tokens,
524
"input_positions": input_positions,
524
- "is_prompt": input_metadata.is_prompt,
525
- "slot_mapping": input_metadata.slot_mapping,
526
- "prompt_lens": input_metadata.prompt_lens,
527
- "max_seq_len": input_metadata.max_seq_len,
528
- "start_loc": input_metadata.start_loc,
529
- "max_context_len": input_metadata.max_context_len,
530
- "context_lens": input_metadata.context_lens,
531
- "block_tables": input_metadata.block_tables,
532
- "use_cuda_graph": input_metadata.use_cuda_graph,
533
- "kv_cache_dtype": input_metadata.kv_cache_dtype,
525
"selected_token_indices":
526
sampling_metadata.selected_token_indices,
527
"lora_requests": lora_requests,
528
"lora_mapping": lora_mapping,
529
}
530
+ metadata_dict.update(dataclasses.asdict(input_metadata))
531
broadcast_tensor_dict(metadata_dict, src=0)
532
else:
533
metadata_dict = broadcast_tensor_dict(src=0)
542
- input_tokens = metadata_dict["input_tokens"]
543
- input_positions = metadata_dict["input_positions"]
544
- lora_mapping = metadata_dict["lora_mapping"]
545
- lora_requests = metadata_dict["lora_requests"]
546
- input_metadata = InputMetadata(
547
- is_prompt=metadata_dict["is_prompt"],
548
- slot_mapping=metadata_dict["slot_mapping"],
549
- prompt_lens=metadata_dict["prompt_lens"],
550
- max_seq_len=metadata_dict["max_seq_len"],
551
- start_loc=metadata_dict["start_loc"],
552
- max_context_len=metadata_dict["max_context_len"],
553
- context_lens=metadata_dict["context_lens"],
554
- block_tables=metadata_dict["block_tables"],
555
- use_cuda_graph=metadata_dict["use_cuda_graph"],
556
- kv_cache_dtype=metadata_dict["kv_cache_dtype"],
557
- )
534
+ input_tokens = metadata_dict.pop("input_tokens")
535
+ input_positions = metadata_dict.pop("input_positions")
536
+ selected_token_indices = metadata_dict.pop(
537
+ "selected_token_indices")
538
+ lora_mapping = metadata_dict.pop("lora_mapping")
539
+ lora_requests = metadata_dict.pop("lora_requests")
540
+ input_metadata = InputMetadata(**metadata_dict)
541
sampling_metadata = SamplingMetadata(
542
seq_groups=None,
543
seq_data=None,
544
prompt_lens=None,
562
- selected_token_indices=metadata_dict["selected_token_indices"],
545
+ selected_token_indices=selected_token_indices,
546
categorized_sample_indices=None,
547
generators=None,
548
perform_sampling=False,