A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core] Add fault tolerance for `RayTokenizerGroupPool` (#5748)
Antoni Baum committed
Jun 25, 2024 at 10:15 UTC
67882dbb44186d781ab6db9eaec08f6616dc86bd
5 files changed
+195
-24
tests/tokenization/test_tokenizer_group.py
+99
@@ -1,5 +1,7 @@
1
import asyncio
2
import os
3
+import sys
4
+from typing import List, Optional
5
from unittest.mock import patch
6
7
import pytest
@@ -100,3 +102,100 @@ async def test_tokenizer_group_ray_pool_env_var_propagation(
102
max_num_seqs=1,
103
max_input_length=None)
104
tokenizer_pool.ping()
105
+
106
+
107
+@pytest.mark.asyncio
108
+@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
109
+async def test_tokenizer_group_ray_pool_fault_tolerance(tokenizer_group_type):
110
+ """Test that Ray tokenizer pool group can recover from failures and
111
+ if that's not possible, mark itself as unhealthy."""
112
+
113
+ class FailingTokenizerGroup(TokenizerGroup):
114
+
115
+ def __init__(self,
116
+ *args,
117
+ fail_at: Optional[List[int]] = None,
118
+ **kwargs):
119
+ super().__init__(*args, **kwargs)
120
+ self.i = 0
121
+ self.fail_at = fail_at or []
122
+
123
+ def encode(self, *args, **kwargs):
124
+ self.i += 1
125
+ if self.i in self.fail_at:
126
+ sys.exit(1)
127
+ return super().encode(*args, **kwargs)
128
+
129
+ class FailingRayTokenizerGroupPool(RayTokenizerGroupPool):
130
+ _worker_cls = FailingTokenizerGroup
131
+
132
+ # Fail at first iteration
133
+ fail_at = [1]
134
+ tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
135
+ tokenizer_group_pool = FailingRayTokenizerGroupPool.from_config(
136
+ tokenizer_pool_config,
137
+ tokenizer_id="gpt2",
138
+ enable_lora=False,
139
+ max_num_seqs=1,
140
+ max_input_length=None,
141
+ fail_at=fail_at)
142
+ tokenizer_actors = tokenizer_group_pool.tokenizer_actors.copy()
143
+
144
+ # Modify fail at to not fail at all (will be re-read when actor is
145
+ # re-initialized).
146
+ fail_at[0] = 1000
147
+
148
+ # We should recover successfully.
149
+ await tokenizer_group_pool.encode_async(request_id="1",
150
+ prompt="prompt",
151
+ lora_request=None)
152
+ await tokenizer_group_pool.encode_async(request_id="1",
153
+ prompt="prompt",
154
+ lora_request=None)
155
+
156
+ # Check that we have a new actor
157
+ assert len(tokenizer_group_pool.tokenizer_actors) == len(tokenizer_actors)
158
+ assert tokenizer_group_pool.tokenizer_actors != tokenizer_actors
159
+
160
+ # Fail at first iteration
161
+ fail_at = [1]
162
+ tokenizer_group_pool = FailingRayTokenizerGroupPool.from_config(
163
+ tokenizer_pool_config,
164
+ tokenizer_id="gpt2",
165
+ enable_lora=False,
166
+ max_num_seqs=1,
167
+ max_input_length=None,
168
+ fail_at=fail_at)
169
+
170
+ # We should fail after re-initialization.
171
+ with pytest.raises(RuntimeError):
172
+ await tokenizer_group_pool.encode_async(request_id="1",
173
+ prompt="prompt",
174
+ lora_request=None)
175
+
176
+ # check_health should raise the same thing
177
+ with pytest.raises(RuntimeError):
178
+ tokenizer_group_pool.check_health()
179
+
180
+ # Ensure that non-ActorDiedErrors are still propagated correctly and do not
181
+ # cause a re-initialization.
182
+ fail_at = []
183
+ tokenizer_group_pool = FailingRayTokenizerGroupPool.from_config(
184
+ tokenizer_pool_config,
185
+ tokenizer_id="gpt2",
186
+ enable_lora=False,
187
+ max_num_seqs=1,
188
+ max_input_length=2,
189
+ fail_at=fail_at)
190
+ tokenizer_actors = tokenizer_group_pool.tokenizer_actors.copy()
191
+
192
+ # Prompt too long error
193
+ with pytest.raises(ValueError):
194
+ await tokenizer_group_pool.encode_async(request_id="1",
195
+ prompt="prompt" * 100,
196
+ lora_request=None)
197
+ await tokenizer_group_pool.encode_async(request_id="1",
198
+ prompt="prompt",
199
+ lora_request=None)
200
+ # Actors should stay the same.
201
+ assert tokenizer_group_pool.tokenizer_actors == tokenizer_actors
vllm/engine/async_llm_engine.py
+2
@@ -310,6 +310,8 @@ class _AsyncLLMEngine(LLMEngine):
310
)
311
312
async def check_health_async(self) -> None:
313
+ if self.tokenizer:
314
+ self.tokenizer.check_health()
315
self.model_executor.check_health()
316
317
vllm/engine/llm_engine.py
+2
@@ -1013,6 +1013,8 @@ class LLMEngine:
1013
return self.model_executor.pin_lora(lora_id)
1014
1015
def check_health(self) -> None:
1016
+ if self.tokenizer:
1017
+ self.tokenizer.check_health()
1018
self.model_executor.check_health()
1019
1020
def is_tracing_enabled(self) -> bool:
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+4
@@ -53,3 +53,7 @@ class BaseTokenizerGroup(ABC):
53
) -> "PreTrainedTokenizer":
54
"""Get a tokenizer for a LoRA request."""
55
pass
56
+
57
+ def check_health(self):
58
+ """Raise exception if the tokenizer group is unhealthy."""
59
+ return
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+88
-24
@@ -2,17 +2,21 @@ import asyncio
2
import os
3
from typing import List, Optional
4
5
+from ray.exceptions import ActorDiedError
6
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
7
from transformers import PreTrainedTokenizer
8
9
from vllm.config import TokenizerPoolConfig
10
from vllm.executor.ray_utils import ray
11
+from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
14
BaseTokenizerGroup)
15
from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
16
TokenizerGroup)
17
18
+logger = init_logger(__name__)
19
+
20
21
class RayTokenizerGroupPool(BaseTokenizerGroup):
22
"""A Ray-based pool of TokenizerGroups for async tokenization."""
@@ -46,24 +50,28 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
50
ray_actor_options: dict, **tokenizer_config):
51
# Store a local copy of the TokenizerGroup for quick access
52
# to underlying HF tokenizers.
53
+ self._tokenizer_config = {
54
+ "tokenizer_id": tokenizer_id,
55
+ "enable_lora": enable_lora,
56
+ "max_num_seqs": max_num_seqs,
57
+ "max_input_length": max_input_length,
58
+ **tokenizer_config
59
+ }
60
self._local_tokenizer_group = self._worker_cls(
50
- tokenizer_id=tokenizer_id,
51
- enable_lora=enable_lora,
52
- max_num_seqs=max_num_seqs,
53
- max_input_length=max_input_length,
54
- **tokenizer_config,
55
- )
56
-
57
- ray_tokenizer_group_cls = ray.remote(
61
+ **self._tokenizer_config, )
62
+
63
+ self._ray_tokenizer_group_cls = ray.remote(
64
self._worker_cls).options(**ray_actor_options)
59
- self.tokenizer_actors = [
60
- ray_tokenizer_group_cls.remote(tokenizer_id, enable_lora,
61
- max_num_seqs, max_input_length,
62
- **tokenizer_config)
63
- for _ in range(num_actors)
64
- ]
65
+ self.tokenizer_actors = [self._init_actor() for _ in range(num_actors)]
66
self._idle_actors: Optional[asyncio.Queue] = None
67
68
+ # If set, actor is unhealthy. Will reraise on the next
69
+ # check_health call.
70
+ self._exception: Optional[ActorDiedError] = None
71
+
72
+ def _init_actor(self) -> ray.ObjectRef:
73
+ return self._ray_tokenizer_group_cls.remote(**self._tokenizer_config)
74
+
75
@property
76
def pool_size(self) -> int:
77
return len(self.tokenizer_actors)
@@ -78,6 +86,22 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
86
for actor in self.tokenizer_actors:
87
self._idle_actors.put_nowait(actor)
88
89
+ def _finalize_encode(self, actor: ray.ObjectRef,
90
+ original_actor: ray.ObjectRef, actor_is_alive: bool):
91
+ assert self._idle_actors is not None
92
+ # Cleanup the dead actor.
93
+ if not actor_is_alive or original_actor is not actor:
94
+ self.tokenizer_actors.remove(original_actor)
95
+ if actor_is_alive:
96
+ # Put the actor back in the queue.
97
+ # This is done in a finally block to ensure that the actor is
98
+ # always put back in the queue, even if an exception/cancellation
99
+ # is raised.
100
+ self._idle_actors.put_nowait(actor)
101
+ # Add back the new actor.
102
+ if original_actor is not actor:
103
+ self.tokenizer_actors.append(actor)
104
+
105
def encode(self,
106
prompt: str,
107
request_id: Optional[str] = None,
@@ -88,23 +112,41 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
112
The actor is then put back in the queue for future use.
113
This is blocking.
114
"""
115
+ self.check_health()
116
self._ensure_queue_initialized()
117
assert self._idle_actors is not None
118
119
if self._idle_actors.empty():
120
raise RuntimeError("No idle actors available.")
121
actor = self._idle_actors.get_nowait()
122
+ actor_is_alive = True
123
+ original_actor = actor
124
try:
125
ret = ray.get(
126
actor.encode.remote(request_id=request_id,
127
prompt=prompt,
128
lora_request=lora_request))
129
+ except ActorDiedError as e:
130
+ # If the actor is dead, we first try to reinitialize it.
131
+ logger.warning("%s died with ActorDiedError, reinitializing.",
132
+ actor,
133
+ exc_info=e)
134
+ actor = self._init_actor()
135
+ try:
136
+ ret = ray.get(
137
+ actor.encode.remote(request_id=request_id,
138
+ prompt=prompt,
139
+ lora_request=lora_request))
140
+ except ActorDiedError as e:
141
+ logger.error(
142
+ "%s died for second time in a row, marking "
143
+ "RayTokenizerGroupPool as unhealthy.", actor)
144
+ actor_is_alive = False
145
+ if not self._exception:
146
+ self._exception = e
147
+ self.check_health()
148
finally:
103
- # Put the actor back in the queue.
104
- # This is done in a finally block to ensure that the actor is
105
- # always put back in the queue, even if an exception/cancellation
106
- # is raised.
107
- self._idle_actors.put_nowait(actor)
149
+ self._finalize_encode(actor, original_actor, actor_is_alive)
150
return ret
151
152
async def encode_async(
@@ -120,20 +162,37 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
162
The actor is then put back in the queue for future use.
163
This is non-blocking.
164
"""
165
+ self.check_health()
166
self._ensure_queue_initialized()
167
assert self._idle_actors is not None
168
169
actor = await self._idle_actors.get()
170
+ actor_is_alive = True
171
+ original_actor = actor
172
try:
173
ret = await actor.encode.remote(request_id=request_id,
174
prompt=prompt,
175
lora_request=lora_request)
176
+ except ActorDiedError as e:
177
+ # If the actor is dead, we first try to reinitialize it.
178
+ logger.warning("%s died with ActorDiedError, reinitializing.",
179
+ actor,
180
+ exc_info=e)
181
+ actor = self._init_actor()
182
+ try:
183
+ ret = await actor.encode.remote(request_id=request_id,
184
+ prompt=prompt,
185
+ lora_request=lora_request)
186
+ except ActorDiedError as e:
187
+ logger.error(
188
+ "%s died for second time in a row, marking "
189
+ "RayTokenizerGroupPool as unhealthy.", actor)
190
+ actor_is_alive = False
191
+ if not self._exception:
192
+ self._exception = e
193
+ self.check_health()
194
finally:
132
- # Put the actor back in the queue.
133
- # This is done in a finally block to ensure that the actor is
134
- # always put back in the queue, even if an exception/cancellation
135
- # is raised.
136
- self._idle_actors.put_nowait(actor)
195
+ self._finalize_encode(actor, original_actor, actor_is_alive)
196
return ret
197
198
def get_max_input_len(self,
@@ -155,6 +214,11 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
214
return await self._local_tokenizer_group.get_lora_tokenizer_async(
215
lora_request)
216
217
+ def check_health(self):
218
+ if self._exception:
219
+ raise RuntimeError(
220
+ "TokenizerGroupPool is unhealthy.") from self._exception
221
+
222
223
def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None:
224
"""Copy over all current process environment variables to the runtime_env.