A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core] Remove duplicate processing in async engine (#5525)
Cyrus Leung committed
Jun 15, 2024 at 01:04 UTC
77490c6f2f1e99982d2553832a42980bbdee820c
1 file changed
+1
-13
vllm/engine/async_llm_engine.py
+1
-13
@@ -580,21 +580,9 @@ class AsyncLLMEngine:
580
if arrival_time is None:
581
arrival_time = time.time()
582
583
- if self.engine_use_ray:
584
- processed_inputs = await self.engine.process_model_inputs_async \
585
- .remote( # type: ignore
586
- request_id=request_id,
587
- inputs=inputs,
588
- lora_request=lora_request)
589
- else:
590
- processed_inputs = await self.engine.process_model_inputs_async(
591
- request_id=request_id,
592
- inputs=inputs,
593
- lora_request=lora_request)
594
-
583
stream = self._request_tracker.add_request(
584
request_id,
597
- inputs=processed_inputs,
585
+ inputs=inputs,
586
params=params,
587
arrival_time=arrival_time,
588
lora_request=lora_request,