A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Fix pickle of input when async output processing is on (#9931)
Signed-off-by: Wallas Santos <wallashss@ibm.com>
Wallas Henrique committed
Nov 5, 2024 at 21:39 UTC
966e31697bdeb47b33b3e26b4aab5999c85f3e90
2 files changed
+38
tests/basic_correctness/test_basic_correctness.py
+26
@@ -156,3 +156,29 @@ def test_model_with_failure(vllm_runner) -> None:
156
ModelInputForGPUWithSamplingMetadata)
157
finally:
158
os.remove(filename)
159
+
160
+
161
+def test_failure_with_async_out_proc(vllm_runner) -> None:
162
+
163
+ filename = None
164
+ try:
165
+ with vllm_runner("facebook/opt-125m",
166
+ dtype="half",
167
+ enforce_eager=False,
168
+ gpu_memory_utilization=0.7) as vllm_model,\
169
+ patch("vllm.model_executor.models.opt.OPTForCausalLM.forward",
170
+ side_effect=ValueError()):
171
+ model_config = vllm_model.model.llm_engine.model_config
172
+ assert model_config.use_async_output_proc
173
+ with pytest.raises(ValueError) as exc_info:
174
+ vllm_model.generate_greedy('how to make pizza?', 250)
175
+ matches = re.search(r"input dumped to (.+).pkl",
176
+ str(exc_info.value))
177
+ assert matches is not None
178
+
179
+ filename = f"{matches.group(1)}.pkl"
180
+ finally:
181
+ # Clean up
182
+ if filename is not None:
183
+ os.remove(filename)
184
+ pass
vllm/worker/model_runner.py
+12
@@ -136,6 +136,18 @@ class ModelInputForGPU(ModelRunnerInputBase):
136
attn_backend, tensor_dict)
137
return cls(**tensor_dict)
138
139
+ # Exclude `async_callback` to be able to pickle this object
140
+ def __getstate__(self):
141
+ state = self.__dict__.copy()
142
+ del state["async_callback"]
143
+ return state
144
+
145
+ # TODO: What happens when we depickle this object?
146
+ # How can we update this callback to properly pass it to the engine?
147
+ def __setstate__(self, state):
148
+ self.__dict__.update(state)
149
+ self.__dict__.update({'async_callback': None})
150
+
151
152
@dataclass(frozen=True)
153
class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):