@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 49eedea373
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] Zero-copy asdict for InputMetadata (#3475)

Antoni Baum committed Mar 18, 2024 at 15:56 UTC 49eedea373043ee9d1b11b81b6c5b3bc24af5b77
2 files changed +12 -4
vllm/model_executor/input_metadata.py
+11 -2
@@ -1,5 +1,5 @@
1 -from dataclasses import dataclass
2 -from typing import Optional
1 +from dataclasses import dataclass, fields
2 +from typing import Optional, Any, Dict
3
4 import torch
5
@@ -31,3 +31,12 @@ class InputMetadata:
31 def __post_init__(self):
32 # will not appear in the __repr__ and __init__
33 self.attn_bias = None
34 +
35 + def asdict_zerocopy(self) -> Dict[str, Any]:
36 + """Similar to dataclasses.asdict, but avoids deepcopying."""
37 + # Note that if we add dataclasses as fields, they will need
38 + # similar handling.
39 + return {
40 + field.name: getattr(self, field.name)
41 + for field in fields(self)
42 + }
vllm/worker/model_runner.py
+1 -2
@@ -1,5 +1,4 @@
1 import contextlib
2 -import dataclasses
2 import time
3 from typing import Dict, List, Optional, Tuple, Set, Union
4
@@ -527,7 +526,7 @@ class ModelRunner:
526 "lora_requests": lora_requests,
527 "lora_mapping": lora_mapping,
528 }
530 - metadata_dict.update(dataclasses.asdict(input_metadata))
529 + metadata_dict.update(input_metadata.asdict_zerocopy())
530 broadcast_tensor_dict(metadata_dict, src=0)
531 else:
532 metadata_dict = broadcast_tensor_dict(src=0)