A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Core] Zero-copy asdict for InputMetadata (#3475)
Antoni Baum committed
Mar 18, 2024 at 15:56 UTC
49eedea373043ee9d1b11b81b6c5b3bc24af5b77
2 files changed
+12
-4
vllm/model_executor/input_metadata.py
+11
-2
@@ -1,5 +1,5 @@
1
-from dataclasses import dataclass
2
-from typing import Optional
1
+from dataclasses import dataclass, fields
2
+from typing import Optional, Any, Dict
3
4
import torch
5
@@ -31,3 +31,12 @@ class InputMetadata:
31
def __post_init__(self):
32
# will not appear in the __repr__ and __init__
33
self.attn_bias = None
34
+
35
+ def asdict_zerocopy(self) -> Dict[str, Any]:
36
+ """Similar to dataclasses.asdict, but avoids deepcopying."""
37
+ # Note that if we add dataclasses as fields, they will need
38
+ # similar handling.
39
+ return {
40
+ field.name: getattr(self, field.name)
41
+ for field in fields(self)
42
+ }
vllm/worker/model_runner.py
+1
-2
@@ -1,5 +1,4 @@
1
import contextlib
2
-import dataclasses
2
import time
3
from typing import Dict, List, Optional, Tuple, Set, Union
4
@@ -527,7 +526,7 @@ class ModelRunner:
526
"lora_requests": lora_requests,
527
"lora_mapping": lora_mapping,
528
}
530
- metadata_dict.update(dataclasses.asdict(input_metadata))
529
+ metadata_dict.update(input_metadata.asdict_zerocopy())
530
broadcast_tensor_dict(metadata_dict, src=0)
531
else:
532
metadata_dict = broadcast_tensor_dict(src=0)