@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b8747e8a7c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[MISC] Skip dumping inputs when unpicklable (#8744)

Cody Yu committed Sep 23, 2024 at 23:10 UTC b8747e8a7c318ab774862f94ccbdbba5b7d9dd4a
1 file changed +9 -1
vllm/worker/model_runner_base.py
+9 -1
@@ -137,7 +137,15 @@ def dump_input_when_exception(exclude_args: Optional[List[int]] = None,
137 for t in kv_caches
138 if is_tensor(t)]
139
140 - pickle.dump(dumped_inputs, filep)
140 + try:
141 + pickle.dump(dumped_inputs, filep)
142 + except Exception as pickle_err:
143 + logger.warning(
144 + "Failed to pickle inputs of failed execution: %s",
145 + str(pickle_err))
146 + raise type(err)(f"Error in model execution: "
147 + f"{str(err)}") from err
148 +
149 logger.info(
150 "Completed writing input of failed execution to %s.",
151 filename)