A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] Fix get tokenizer when using ray (#3301)
Roy committed
Mar 11, 2024 at 10:17 UTC
9e8744a545f34ca636a5113ae98cec5851af56eb
6 files changed
+23
-7
tests/async_engine/test_async_llm_engine.py
+3
@@ -89,3 +89,6 @@ async def test_new_requests_event():
89
await asyncio.sleep(0.01)
90
assert engine.engine.add_request_calls == 3
91
assert engine.engine.step_calls == old_step_calls + 1
92
+
93
+ engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
94
+ assert engine.get_tokenizer() is not None
vllm/engine/async_llm_engine.py
+7
-2
@@ -5,6 +5,8 @@ from functools import partial
5
from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
6
Union, AsyncIterator, Callable)
7
8
+from transformers import PreTrainedTokenizer
9
+
10
from vllm.lora.request import LoRARequest
11
from vllm.config import ModelConfig
12
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -372,8 +374,11 @@ class AsyncLLMEngine:
374
self.set_errored(exc)
375
self._request_tracker.propagate_exception(exc)
376
375
- def get_tokenizer(self):
376
- return self.engine.tokenizer.tokenizer
377
+ async def get_tokenizer(self) -> "PreTrainedTokenizer":
378
+ if self.engine_use_ray:
379
+ return await self.engine.get_tokenizer.remote()
380
+ else:
381
+ return self.engine.get_tokenizer()
382
383
def start_background_loop(self) -> None:
384
"""Start the background loop."""
vllm/engine/llm_engine.py
+7
-1
@@ -7,6 +7,8 @@ import importlib
7
from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8
Union)
9
10
+from transformers import PreTrainedTokenizer
11
+
12
import vllm
13
from vllm.lora.request import LoRARequest
14
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
@@ -163,7 +165,11 @@ class LLMEngine:
165
# the closure used to initialize Ray worker actors
166
raise RuntimeError("LLMEngine should not be pickled!")
167
166
- def get_tokenizer_for_seq(self, sequence: Sequence):
168
+ def get_tokenizer(self) -> "PreTrainedTokenizer":
169
+ return self.tokenizer.get_lora_tokenizer()
170
+
171
+ def get_tokenizer_for_seq(self,
172
+ sequence: Sequence) -> "PreTrainedTokenizer":
173
return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
174
175
def _dispatch_worker(self):
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -65,7 +65,7 @@ class OpenAIServingChat(OpenAIServing):
65
lora_request = self._maybe_get_lora(request)
66
guided_decode_logits_processor = (
67
await get_guided_decoding_logits_processor(
68
- request, self.engine.get_tokenizer()))
68
+ request, await self.engine.get_tokenizer()))
69
if guided_decode_logits_processor:
70
if sampling_params.logits_processors is None:
71
sampling_params.logits_processors = []
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -126,7 +126,7 @@ class OpenAIServingCompletion(OpenAIServing):
126
lora_request = self._maybe_get_lora(request)
127
guided_decode_logit_processor = (
128
await get_guided_decoding_logits_processor(
129
- request, self.engine.get_tokenizer()))
129
+ request, await self.engine.get_tokenizer()))
130
if guided_decode_logit_processor is not None:
131
if sampling_params.logits_processors is None:
132
sampling_params.logits_processors = []
vllm/transformers_utils/tokenizer.py
+4
-2
@@ -120,7 +120,8 @@ class TokenizerGroup:
120
121
def get_lora_tokenizer(
122
self,
123
- lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
123
+ lora_request: Optional[LoRARequest] = None
124
+ ) -> "PreTrainedTokenizer":
125
if not lora_request or not self.enable_lora:
126
return self.tokenizer
127
if lora_request.lora_int_id not in self.lora_tokenizers:
@@ -133,7 +134,8 @@ class TokenizerGroup:
134
135
async def get_lora_tokenizer_async(
136
self,
136
- lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
137
+ lora_request: Optional[LoRARequest] = None
138
+ ) -> "PreTrainedTokenizer":
139
if not lora_request or not self.enable_lora:
140
return self.tokenizer
141
if lora_request.lora_int_id not in self.lora_tokenizers: