@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e8744a545
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[BugFix] Fix get tokenizer when using ray (#3301)

Roy committed Mar 11, 2024 at 10:17 UTC 9e8744a545f34ca636a5113ae98cec5851af56eb
6 files changed +23 -7
tests/async_engine/test_async_llm_engine.py
+3
@@ -89,3 +89,6 @@ async def test_new_requests_event():
89 await asyncio.sleep(0.01)
90 assert engine.engine.add_request_calls == 3
91 assert engine.engine.step_calls == old_step_calls + 1
92 +
93 + engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
94 + assert engine.get_tokenizer() is not None
vllm/engine/async_llm_engine.py
+7 -2
@@ -5,6 +5,8 @@ from functools import partial
5 from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
6 Union, AsyncIterator, Callable)
7
8 +from transformers import PreTrainedTokenizer
9 +
10 from vllm.lora.request import LoRARequest
11 from vllm.config import ModelConfig
12 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -372,8 +374,11 @@ class AsyncLLMEngine:
374 self.set_errored(exc)
375 self._request_tracker.propagate_exception(exc)
376
375 - def get_tokenizer(self):
376 - return self.engine.tokenizer.tokenizer
377 + async def get_tokenizer(self) -> "PreTrainedTokenizer":
378 + if self.engine_use_ray:
379 + return await self.engine.get_tokenizer.remote()
380 + else:
381 + return self.engine.get_tokenizer()
382
383 def start_background_loop(self) -> None:
384 """Start the background loop."""
vllm/engine/llm_engine.py
+7 -1
@@ -7,6 +7,8 @@ import importlib
7 from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8 Union)
9
10 +from transformers import PreTrainedTokenizer
11 +
12 import vllm
13 from vllm.lora.request import LoRARequest
14 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
@@ -163,7 +165,11 @@ class LLMEngine:
165 # the closure used to initialize Ray worker actors
166 raise RuntimeError("LLMEngine should not be pickled!")
167
166 - def get_tokenizer_for_seq(self, sequence: Sequence):
168 + def get_tokenizer(self) -> "PreTrainedTokenizer":
169 + return self.tokenizer.get_lora_tokenizer()
170 +
171 + def get_tokenizer_for_seq(self,
172 + sequence: Sequence) -> "PreTrainedTokenizer":
173 return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
174
175 def _dispatch_worker(self):
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -65,7 +65,7 @@ class OpenAIServingChat(OpenAIServing):
65 lora_request = self._maybe_get_lora(request)
66 guided_decode_logits_processor = (
67 await get_guided_decoding_logits_processor(
68 - request, self.engine.get_tokenizer()))
68 + request, await self.engine.get_tokenizer()))
69 if guided_decode_logits_processor:
70 if sampling_params.logits_processors is None:
71 sampling_params.logits_processors = []
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -126,7 +126,7 @@ class OpenAIServingCompletion(OpenAIServing):
126 lora_request = self._maybe_get_lora(request)
127 guided_decode_logit_processor = (
128 await get_guided_decoding_logits_processor(
129 - request, self.engine.get_tokenizer()))
129 + request, await self.engine.get_tokenizer()))
130 if guided_decode_logit_processor is not None:
131 if sampling_params.logits_processors is None:
132 sampling_params.logits_processors = []
vllm/transformers_utils/tokenizer.py
+4 -2
@@ -120,7 +120,8 @@ class TokenizerGroup:
120
121 def get_lora_tokenizer(
122 self,
123 - lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
123 + lora_request: Optional[LoRARequest] = None
124 + ) -> "PreTrainedTokenizer":
125 if not lora_request or not self.enable_lora:
126 return self.tokenizer
127 if lora_request.lora_int_id not in self.lora_tokenizers:
@@ -133,7 +134,8 @@ class TokenizerGroup:
134
135 async def get_lora_tokenizer_async(
136 self,
136 - lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
137 + lora_request: Optional[LoRARequest] = None
138 + ) -> "PreTrainedTokenizer":
139 if not lora_request or not self.enable_lora:
140 return self.tokenizer
141 if lora_request.lora_int_id not in self.lora_tokenizers: