A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Use RayActorError for older versions of Ray in RayTokenizerGroupPool (#6039)
Antoni Baum committed
Jul 1, 2024 at 13:12 UTC
dec6fc6f3bc56a487af4d8d23e18dd227360ef98
1 file changed
+5
-1
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+5
-1
@@ -2,7 +2,11 @@ import asyncio
2
import os
3
from typing import List, Optional
4
5
-from ray.exceptions import ActorDiedError
5
+try:
6
+ from ray.exceptions import ActorDiedError
7
+except ImportError:
8
+ # For older versions of Ray
9
+ from ray.exceptions import RayActorError as ActorDiedError
10
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
11
from transformers import PreTrainedTokenizer
12