A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Misc] Don't dump contents of kvcache tensors on errors (#8527)
Nick Hill committed
Sep 17, 2024 at 20:24 UTC
56c3de018c35580fd088655c2f9951cd4da5335d
1 file changed
+20
-2
vllm/worker/model_runner_base.py
+20
-2
@@ -3,11 +3,13 @@ import pickle
3
from abc import ABC, abstractmethod
4
from datetime import datetime
5
from functools import wraps
6
-from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
7
- TypeVar)
6
+from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
7
+ Optional, Type, TypeVar)
8
9
import torch
10
+from torch import is_tensor
11
12
+from vllm.logger import init_logger
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
from vllm.platforms import current_platform
15
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
@@ -17,6 +19,8 @@ if TYPE_CHECKING:
19
from vllm.attention.backends.abstract import AttentionBackend
20
from vllm.model_executor import SamplingMetadata
21
22
+logger = init_logger(__name__)
23
+
24
T = TypeVar('T', bound="BroadcastableModelInput")
25
26
@@ -113,6 +117,8 @@ def dump_input_when_exception(exclude_args: Optional[List[int]] = None,
117
except Exception as err:
118
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
119
filename = f"/tmp/err_{func.__name__}_input_{timestamp}.pkl"
120
+ logger.info("Writing input of failed execution to %s...",
121
+ filename)
122
with open(filename, "wb") as filep:
123
dumped_inputs = {
124
k: v
@@ -122,7 +128,19 @@ def dump_input_when_exception(exclude_args: Optional[List[int]] = None,
128
for i, arg in enumerate(args):
129
if i not in (exclude_args or []):
130
dumped_inputs[f"arg_{i}"] = arg
131
+
132
+ # Only persist dtype and shape for kvcache tensors
133
+ # (can be way to big otherwise)
134
+ if (kv_caches := dumped_inputs.get("kv_caches")) \
135
+ and isinstance(kv_caches, Iterable):
136
+ dumped_inputs["kv_caches"] = [(t.dtype, t.shape)
137
+ for t in kv_caches
138
+ if is_tensor(t)]
139
+
140
pickle.dump(dumped_inputs, filep)
141
+ logger.info(
142
+ "Completed writing input of failed execution to %s.",
143
+ filename)
144
raise type(err)(
145
f"Error in model execution (input dumped to {filename}): "
146
f"{str(err)}") from err