@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 112fa0bbe5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] Fix CI tests on V1 engine (#10272)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 12, 2024 at 16:17 UTC 112fa0bbe5e5354f592a42913a4e6d72e0407b93
3 files changed +7 -1
tests/v1/engine/test_engine_core.py
+3
@@ -27,6 +27,9 @@ def make_request() -> EngineCoreRequest:
27 request_id=uuid.uuid4(),
28 prompt=PROMPT,
29 prompt_token_ids=PROMPT_TOKENS,
30 + mm_data=None,
31 + mm_placeholders=None,
32 + mm_processor_kwargs=None,
33 sampling_params=SamplingParams(),
34 eos_token_id=None,
35 arrival_time=time.time(),
tests/v1/engine/test_engine_core_client.py
+3
@@ -29,6 +29,9 @@ def make_request(params: SamplingParams) -> EngineCoreRequest:
29 request_id=str(uuid.uuid4()),
30 prompt=PROMPT,
31 prompt_token_ids=PROMPT_TOKENS,
32 + mm_data=None,
33 + mm_placeholders=None,
34 + mm_processor_kwargs=None,
35 sampling_params=params,
36 eos_token_id=None,
37 arrival_time=time.time(),
vllm/v1/engine/core.py
+1 -1
@@ -317,7 +317,7 @@ class EngineCoreProc(EngineCore):
317
318 # Msgpack serialization decoding.
319 decoder_add_req = PickleEncoder()
320 - decoder_abort_req = msgpack.Decoder(list[str])
320 + decoder_abort_req = PickleEncoder()
321
322 with self.make_socket(input_path, zmq.constants.PULL) as socket:
323 while True: