@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / abfc4f3387
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] Use dataclass for InputMetadata (#3452)

Co-authored-by: youkaichao <youkaichao@126.com>

Woosuk Kwon committed Mar 17, 2024 at 03:02 UTC abfc4f3387c436d46d6701e9ba916de8f9ed9329
3 files changed +24 -63
setup.py
-1
@@ -2,7 +2,6 @@ import contextlib
2 import io
3 import os
4 import re
5 -import shutil
5 import subprocess
6 import warnings
7 from pathlib import Path
vllm/model_executor/input_metadata.py
+14 -35
@@ -1,8 +1,10 @@
1 +from dataclasses import dataclass
2 from typing import Optional
3
4 import torch
5
6
7 +@dataclass
8 class InputMetadata:
9 """Metadata for input sequences. Used in PagedAttention.
10
@@ -15,40 +17,17 @@ class InputMetadata:
17 kv_cache_dtype: Data type to store kv cache.
18 """
19
18 - def __init__(
19 - self,
20 - is_prompt: bool,
21 - slot_mapping: torch.Tensor,
22 - prompt_lens: Optional[torch.Tensor],
23 - max_seq_len: Optional[int],
24 - start_loc: Optional[torch.Tensor],
25 - max_context_len: Optional[int],
26 - context_lens: Optional[torch.Tensor],
27 - block_tables: Optional[torch.Tensor],
28 - use_cuda_graph: bool,
29 - kv_cache_dtype: str,
30 - ) -> None:
31 - self.is_prompt = is_prompt
32 - self.prompt_lens = prompt_lens
33 - self.max_seq_len = max_seq_len
34 - self.start_loc = start_loc
35 - self.max_context_len = max_context_len
36 - self.slot_mapping = slot_mapping
37 - self.context_lens = context_lens
38 - self.block_tables = block_tables
39 - self.use_cuda_graph = use_cuda_graph
40 - self.kv_cache_dtype = kv_cache_dtype
20 + is_prompt: bool
21 + slot_mapping: torch.Tensor
22 + prompt_lens: Optional[torch.Tensor]
23 + max_seq_len: Optional[int]
24 + start_loc: Optional[torch.Tensor]
25 + max_context_len: Optional[int]
26 + context_lens: Optional[torch.Tensor]
27 + block_tables: Optional[torch.Tensor]
28 + use_cuda_graph: bool
29 + kv_cache_dtype: str
30
42 - # Set during the execution of the first attention op.
43 - # FIXME(woosuk): This is a hack.
31 + def __post_init__(self):
32 + # will not appear in the __repr__ and __init__
33 self.attn_bias = None
45 -
46 - def __repr__(self) -> str:
47 - return ("InputMetadata("
48 - f"is_prompt={self.is_prompt}, "
49 - f"max_context_len={self.max_context_len}, "
50 - f"slot_mapping={self.slot_mapping}, "
51 - f"context_lens={self.context_lens}, "
52 - f"block_tables={self.block_tables}, "
53 - f"use_cuda_graph={self.use_cuda_graph}, "
54 - f"kv_cache_dtype={self.kv_cache_dtype})")
vllm/worker/model_runner.py
+10 -27
@@ -1,4 +1,5 @@
1 import contextlib
2 +import dataclasses
3 import time
4 from typing import Dict, List, Optional, Tuple, Set, Union
5
@@ -521,45 +522,27 @@ class ModelRunner:
522 metadata_dict = {
523 "input_tokens": input_tokens,
524 "input_positions": input_positions,
524 - "is_prompt": input_metadata.is_prompt,
525 - "slot_mapping": input_metadata.slot_mapping,
526 - "prompt_lens": input_metadata.prompt_lens,
527 - "max_seq_len": input_metadata.max_seq_len,
528 - "start_loc": input_metadata.start_loc,
529 - "max_context_len": input_metadata.max_context_len,
530 - "context_lens": input_metadata.context_lens,
531 - "block_tables": input_metadata.block_tables,
532 - "use_cuda_graph": input_metadata.use_cuda_graph,
533 - "kv_cache_dtype": input_metadata.kv_cache_dtype,
525 "selected_token_indices":
526 sampling_metadata.selected_token_indices,
527 "lora_requests": lora_requests,
528 "lora_mapping": lora_mapping,
529 }
530 + metadata_dict.update(dataclasses.asdict(input_metadata))
531 broadcast_tensor_dict(metadata_dict, src=0)
532 else:
533 metadata_dict = broadcast_tensor_dict(src=0)
542 - input_tokens = metadata_dict["input_tokens"]
543 - input_positions = metadata_dict["input_positions"]
544 - lora_mapping = metadata_dict["lora_mapping"]
545 - lora_requests = metadata_dict["lora_requests"]
546 - input_metadata = InputMetadata(
547 - is_prompt=metadata_dict["is_prompt"],
548 - slot_mapping=metadata_dict["slot_mapping"],
549 - prompt_lens=metadata_dict["prompt_lens"],
550 - max_seq_len=metadata_dict["max_seq_len"],
551 - start_loc=metadata_dict["start_loc"],
552 - max_context_len=metadata_dict["max_context_len"],
553 - context_lens=metadata_dict["context_lens"],
554 - block_tables=metadata_dict["block_tables"],
555 - use_cuda_graph=metadata_dict["use_cuda_graph"],
556 - kv_cache_dtype=metadata_dict["kv_cache_dtype"],
557 - )
534 + input_tokens = metadata_dict.pop("input_tokens")
535 + input_positions = metadata_dict.pop("input_positions")
536 + selected_token_indices = metadata_dict.pop(
537 + "selected_token_indices")
538 + lora_mapping = metadata_dict.pop("lora_mapping")
539 + lora_requests = metadata_dict.pop("lora_requests")
540 + input_metadata = InputMetadata(**metadata_dict)
541 sampling_metadata = SamplingMetadata(
542 seq_groups=None,
543 seq_data=None,
544 prompt_lens=None,
562 - selected_token_indices=metadata_dict["selected_token_indices"],
545 + selected_token_indices=selected_token_indices,
546 categorized_sample_indices=None,
547 generators=None,
548 perform_sampling=False,