@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56c3de018c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Misc] Don't dump contents of kvcache tensors on errors (#8527)

Nick Hill committed Sep 17, 2024 at 20:24 UTC 56c3de018c35580fd088655c2f9951cd4da5335d
1 file changed +20 -2
vllm/worker/model_runner_base.py
+20 -2
@@ -3,11 +3,13 @@ import pickle
3 from abc import ABC, abstractmethod
4 from datetime import datetime
5 from functools import wraps
6 -from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
7 - TypeVar)
6 +from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
7 + Optional, Type, TypeVar)
8
9 import torch
10 +from torch import is_tensor
11
12 +from vllm.logger import init_logger
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.platforms import current_platform
15 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
@@ -17,6 +19,8 @@ if TYPE_CHECKING:
19 from vllm.attention.backends.abstract import AttentionBackend
20 from vllm.model_executor import SamplingMetadata
21
22 +logger = init_logger(__name__)
23 +
24 T = TypeVar('T', bound="BroadcastableModelInput")
25
26
@@ -113,6 +117,8 @@ def dump_input_when_exception(exclude_args: Optional[List[int]] = None,
117 except Exception as err:
118 timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
119 filename = f"/tmp/err_{func.__name__}_input_{timestamp}.pkl"
120 + logger.info("Writing input of failed execution to %s...",
121 + filename)
122 with open(filename, "wb") as filep:
123 dumped_inputs = {
124 k: v
@@ -122,7 +128,19 @@ def dump_input_when_exception(exclude_args: Optional[List[int]] = None,
128 for i, arg in enumerate(args):
129 if i not in (exclude_args or []):
130 dumped_inputs[f"arg_{i}"] = arg
131 +
132 + # Only persist dtype and shape for kvcache tensors
133 + # (can be way to big otherwise)
134 + if (kv_caches := dumped_inputs.get("kv_caches")) \
135 + and isinstance(kv_caches, Iterable):
136 + dumped_inputs["kv_caches"] = [(t.dtype, t.shape)
137 + for t in kv_caches
138 + if is_tensor(t)]
139 +
140 pickle.dump(dumped_inputs, filep)
141 + logger.info(
142 + "Completed writing input of failed execution to %s.",
143 + filename)
144 raise type(err)(
145 f"Error in model execution (input dumped to {filename}): "
146 f"{str(err)}") from err