@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d79ced3292
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix 'Actor methods cannot be called directly' when using `--engine-use-ray` (#2664)

* fix: engine-useray complain * fix: typo

Wen Sun committed Jan 31, 2024 at 00:17 UTC d79ced3292445d8471b3c4e5ce2dbf311834ec1b
1 file changed +13 -5
vllm/engine/async_llm_engine.py
+13 -5
@@ -447,11 +447,19 @@ class AsyncLLMEngine:
447
448 if arrival_time is None:
449 arrival_time = time.time()
450 - prompt_token_ids = await self.engine.encode_request_async(
451 - request_id=request_id,
452 - prompt=prompt,
453 - prompt_token_ids=prompt_token_ids,
454 - lora_request=lora_request)
450 +
451 + if self.engine_use_ray:
452 + prompt_token_ids = await self.engine.encode_request_async.remote(
453 + request_id=request_id,
454 + prompt=prompt,
455 + prompt_token_ids=prompt_token_ids,
456 + lora_request=lora_request)
457 + else:
458 + prompt_token_ids = await self.engine.encode_request_async(
459 + request_id=request_id,
460 + prompt=prompt,
461 + prompt_token_ids=prompt_token_ids,
462 + lora_request=lora_request)
463
464 stream = self._request_tracker.add_request(
465 request_id,