A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix 'Actor methods cannot be called directly' when using `--engine-use-ray` (#2664)
* fix: engine-useray complain * fix: typo
Wen Sun committed
Jan 31, 2024 at 00:17 UTC
d79ced3292445d8471b3c4e5ce2dbf311834ec1b
1 file changed
+13
-5
vllm/engine/async_llm_engine.py
+13
-5
@@ -447,11 +447,19 @@ class AsyncLLMEngine:
447
448
if arrival_time is None:
449
arrival_time = time.time()
450
- prompt_token_ids = await self.engine.encode_request_async(
451
- request_id=request_id,
452
- prompt=prompt,
453
- prompt_token_ids=prompt_token_ids,
454
- lora_request=lora_request)
450
+
451
+ if self.engine_use_ray:
452
+ prompt_token_ids = await self.engine.encode_request_async.remote(
453
+ request_id=request_id,
454
+ prompt=prompt,
455
+ prompt_token_ids=prompt_token_ids,
456
+ lora_request=lora_request)
457
+ else:
458
+ prompt_token_ids = await self.engine.encode_request_async(
459
+ request_id=request_id,
460
+ prompt=prompt,
461
+ prompt_token_ids=prompt_token_ids,
462
+ lora_request=lora_request)
463
464
stream = self._request_tracker.add_request(
465
request_id,