A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[MISC] Skip dumping inputs when unpicklable (#8744)
Cody Yu committed
Sep 23, 2024 at 23:10 UTC
b8747e8a7c318ab774862f94ccbdbba5b7d9dd4a
1 file changed
+9
-1
vllm/worker/model_runner_base.py
+9
-1
@@ -137,7 +137,15 @@ def dump_input_when_exception(exclude_args: Optional[List[int]] = None,
137
for t in kv_caches
138
if is_tensor(t)]
139
140
- pickle.dump(dumped_inputs, filep)
140
+ try:
141
+ pickle.dump(dumped_inputs, filep)
142
+ except Exception as pickle_err:
143
+ logger.warning(
144
+ "Failed to pickle inputs of failed execution: %s",
145
+ str(pickle_err))
146
+ raise type(err)(f"Error in model execution: "
147
+ f"{str(err)}") from err
148
+
149
logger.info(
150
"Completed writing input of failed execution to %s.",
151
filename)