A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] Fix CI tests on V1 engine (#10272)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 12, 2024 at 16:17 UTC
112fa0bbe5e5354f592a42913a4e6d72e0407b93
3 files changed
+7
-1
tests/v1/engine/test_engine_core.py
+3
@@ -27,6 +27,9 @@ def make_request() -> EngineCoreRequest:
27
request_id=uuid.uuid4(),
28
prompt=PROMPT,
29
prompt_token_ids=PROMPT_TOKENS,
30
+ mm_data=None,
31
+ mm_placeholders=None,
32
+ mm_processor_kwargs=None,
33
sampling_params=SamplingParams(),
34
eos_token_id=None,
35
arrival_time=time.time(),
tests/v1/engine/test_engine_core_client.py
+3
@@ -29,6 +29,9 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
29
request_id=str(uuid.uuid4()),
30
prompt=PROMPT,
31
prompt_token_ids=PROMPT_TOKENS,
32
+ mm_data=None,
33
+ mm_placeholders=None,
34
+ mm_processor_kwargs=None,
35
sampling_params=params,
36
eos_token_id=None,
37
arrival_time=time.time(),
vllm/v1/engine/core.py
+1
-1
@@ -317,7 +317,7 @@ class EngineCoreProc(EngineCore):
317
318
# Msgpack serialization decoding.
319
decoder_add_req = PickleEncoder()
320
- decoder_abort_req = msgpack.Decoder(list[str])
320
+ decoder_abort_req = PickleEncoder()
321
322
with self.make_socket(input_path, zmq.constants.PULL) as socket:
323
while True: