@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb96c1e98c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Asynchronous tokenization (#2879)

Antoni Baum committed Mar 15, 2024 at 16:37 UTC fb96c1e98c05ffa35dd48416f68e88edb2f9eb34
17 files changed +658 -153
.buildkite/test-pipeline.yaml
+1 -1
@@ -28,7 +28,7 @@ steps:
28 num_gpus: 2 # only support 1 or 2 for now.
29
30 - label: Engine Test
31 - command: pytest -v -s engine test_sequence.py
31 + command: pytest -v -s engine tokenization test_sequence.py
32
33 - label: Entrypoints Test
34 command: pytest -v -s entrypoints
tests/async_engine/test_api_server.py
+7 -9
@@ -25,23 +25,21 @@ def _query_server_long(prompt: str) -> dict:
25
26
27 @pytest.fixture
28 -def api_server():
28 +def api_server(tokenizer_pool_size: int):
29 script_path = Path(__file__).parent.joinpath(
30 "api_server_async_engine.py").absolute()
31 uvicorn_process = subprocess.Popen([
32 - sys.executable,
33 - "-u",
34 - str(script_path),
35 - "--model",
36 - "facebook/opt-125m",
37 - "--host",
38 - "127.0.0.1",
32 + sys.executable, "-u",
33 + str(script_path), "--model", "facebook/opt-125m", "--host",
34 + "127.0.0.1", "--tokenizer-pool-size",
35 + str(tokenizer_pool_size)
36 ])
37 yield
38 uvicorn_process.terminate()
39
40
44 -def test_api_server(api_server):
41 +@pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
42 +def test_api_server(api_server, tokenizer_pool_size: int):
43 """
44 Run the API server and test it.
45
tests/conftest.py
+11
@@ -7,6 +7,7 @@ from transformers import AutoModelForCausalLM
7
8 from vllm import LLM, SamplingParams
9 from vllm.transformers_utils.tokenizer import get_tokenizer
10 +from vllm.config import TokenizerPoolConfig
11
12 _TEST_DIR = os.path.dirname(__file__)
13 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
@@ -258,3 +259,13 @@ class VllmRunner:
259 @pytest.fixture
260 def vllm_runner():
261 return VllmRunner
262 +
263 +
264 +def get_tokenizer_pool_config(tokenizer_group_type):
265 + if tokenizer_group_type is None:
266 + return None
267 + if tokenizer_group_type == "ray":
268 + return TokenizerPoolConfig(pool_size=1,
269 + pool_type="ray",
270 + extra_config={})
271 + raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
tests/lora/test_tokenizer.py deleted
-69
@@ -1,69 +0,0 @@
1 -import pytest
2 -from transformers import AutoTokenizer, PreTrainedTokenizerBase
3 -
4 -from vllm.lora.request import LoRARequest
5 -from vllm.transformers_utils.tokenizer import TokenizerGroup, get_lora_tokenizer
6 -
7 -
8 -@pytest.mark.asyncio
9 -async def test_transformers_tokenizer():
10 - reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
11 - tokenizer = TokenizerGroup(
12 - tokenizer_id="gpt2",
13 - enable_lora=False,
14 - max_num_seqs=1,
15 - max_input_length=None,
16 - )
17 - assert reference_tokenizer.encode("prompt") == tokenizer.encode(
18 - request_id="request_id", prompt="prompt", lora_request=None)
19 - assert reference_tokenizer.encode(
20 - "prompt") == await tokenizer.encode_async(request_id="request_id",
21 - prompt="prompt",
22 - lora_request=None)
23 - assert isinstance(tokenizer.get_lora_tokenizer(None),
24 - PreTrainedTokenizerBase)
25 - assert tokenizer.get_lora_tokenizer(
26 - None) == await tokenizer.get_lora_tokenizer_async(None)
27 -
28 -
29 -@pytest.mark.asyncio
30 -async def test_transformers_tokenizer_lora(sql_lora_files):
31 - reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
32 - tokenizer = TokenizerGroup(
33 - tokenizer_id="gpt2",
34 - enable_lora=True,
35 - max_num_seqs=1,
36 - max_input_length=None,
37 - )
38 - lora_request = LoRARequest("1", 1, sql_lora_files)
39 - assert reference_tokenizer.encode("prompt") == tokenizer.encode(
40 - request_id="request_id", prompt="prompt", lora_request=lora_request)
41 - assert reference_tokenizer.encode(
42 - "prompt") == await tokenizer.encode_async(request_id="request_id",
43 - prompt="prompt",
44 - lora_request=lora_request)
45 - assert isinstance(tokenizer.get_lora_tokenizer(None),
46 - PreTrainedTokenizerBase)
47 - assert tokenizer.get_lora_tokenizer(
48 - None) == await tokenizer.get_lora_tokenizer_async(None)
49 -
50 - assert isinstance(tokenizer.get_lora_tokenizer(lora_request),
51 - PreTrainedTokenizerBase)
52 - assert tokenizer.get_lora_tokenizer(
53 - lora_request) != tokenizer.get_lora_tokenizer(None)
54 - assert tokenizer.get_lora_tokenizer(
55 - lora_request) == await tokenizer.get_lora_tokenizer_async(lora_request)
56 -
57 -
58 -def test_get_lora_tokenizer(sql_lora_files, tmpdir):
59 - lora_request = None
60 - tokenizer = get_lora_tokenizer(lora_request)
61 - assert not tokenizer
62 -
63 - lora_request = LoRARequest("1", 1, sql_lora_files)
64 - tokenizer = get_lora_tokenizer(lora_request)
65 - assert tokenizer.get_added_vocab()
66 -
67 - lora_request = LoRARequest("1", 1, str(tmpdir))
68 - tokenizer = get_lora_tokenizer(lora_request)
69 - assert not tokenizer
tests/lora/test_tokenizer_group.py new
+53
@@ -0,0 +1,53 @@
1 +import pytest
2 +from transformers import AutoTokenizer, PreTrainedTokenizerBase
3 +from vllm.lora.request import LoRARequest
4 +from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
5 +from vllm.transformers_utils.tokenizer import get_lora_tokenizer
6 +from ..conftest import get_tokenizer_pool_config
7 +
8 +
9 +@pytest.mark.asyncio
10 +@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
11 +async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
12 + reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
13 + tokenizer_group = get_tokenizer_group(
14 + get_tokenizer_pool_config(tokenizer_group_type),
15 + tokenizer_id="gpt2",
16 + enable_lora=True,
17 + max_num_seqs=1,
18 + max_input_length=None,
19 + )
20 + lora_request = LoRARequest("1", 1, sql_lora_files)
21 + assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
22 + request_id="request_id", prompt="prompt", lora_request=lora_request)
23 + assert reference_tokenizer.encode(
24 + "prompt") == await tokenizer_group.encode_async(
25 + request_id="request_id",
26 + prompt="prompt",
27 + lora_request=lora_request)
28 + assert isinstance(tokenizer_group.get_lora_tokenizer(None),
29 + PreTrainedTokenizerBase)
30 + assert tokenizer_group.get_lora_tokenizer(
31 + None) == await tokenizer_group.get_lora_tokenizer_async(None)
32 +
33 + assert isinstance(tokenizer_group.get_lora_tokenizer(lora_request),
34 + PreTrainedTokenizerBase)
35 + assert tokenizer_group.get_lora_tokenizer(
36 + lora_request) != tokenizer_group.get_lora_tokenizer(None)
37 + assert tokenizer_group.get_lora_tokenizer(
38 + lora_request) == await tokenizer_group.get_lora_tokenizer_async(
39 + lora_request)
40 +
41 +
42 +def test_get_lora_tokenizer(sql_lora_files, tmpdir):
43 + lora_request = None
44 + tokenizer = get_lora_tokenizer(lora_request)
45 + assert not tokenizer
46 +
47 + lora_request = LoRARequest("1", 1, sql_lora_files)
48 + tokenizer = get_lora_tokenizer(lora_request)
49 + assert tokenizer.get_added_vocab()
50 +
51 + lora_request = LoRARequest("1", 1, str(tmpdir))
52 + tokenizer = get_lora_tokenizer(lora_request)
53 + assert not tokenizer
tests/tokenization/__init__.py
tests/tokenization/test_cached_tokenizer.py new
+20
@@ -0,0 +1,20 @@
1 +from copy import deepcopy
2 +from vllm.transformers_utils.tokenizer import get_cached_tokenizer
3 +from transformers import AutoTokenizer
4 +
5 +
6 +def test_cached_tokenizer():
7 + reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
8 + reference_tokenizer.add_special_tokens({"cls_token": "<CLS>"})
9 + reference_tokenizer.add_special_tokens(
10 + {"additional_special_tokens": ["<SEP>"]})
11 + cached_tokenizer = get_cached_tokenizer(deepcopy(reference_tokenizer))
12 +
13 + assert reference_tokenizer.encode("prompt") == cached_tokenizer.encode(
14 + "prompt")
15 + assert set(reference_tokenizer.all_special_ids) == set(
16 + cached_tokenizer.all_special_ids)
17 + assert set(reference_tokenizer.all_special_tokens) == set(
18 + cached_tokenizer.all_special_tokens)
19 + assert set(reference_tokenizer.all_special_tokens_extended) == set(
20 + cached_tokenizer.all_special_tokens_extended)
tests/tokenization/test_detokenize.py renamed
tests/tokenization/test_tokenizer_group.py new
+100
@@ -0,0 +1,100 @@
1 +import os
2 +import pytest
3 +import asyncio
4 +from unittest.mock import patch
5 +
6 +from transformers import AutoTokenizer, PreTrainedTokenizerBase
7 +from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
8 +from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
9 + RayTokenizerGroupPool)
10 +from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
11 + TokenizerGroup)
12 +from ..conftest import get_tokenizer_pool_config
13 +
14 +
15 +@pytest.mark.asyncio
16 +@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
17 +async def test_tokenizer_group(tokenizer_group_type):
18 + reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
19 + tokenizer_group = get_tokenizer_group(
20 + get_tokenizer_pool_config(tokenizer_group_type),
21 + tokenizer_id="gpt2",
22 + enable_lora=False,
23 + max_num_seqs=1,
24 + max_input_length=None,
25 + )
26 + assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
27 + request_id="request_id", prompt="prompt", lora_request=None)
28 + assert reference_tokenizer.encode(
29 + "prompt") == await tokenizer_group.encode_async(
30 + request_id="request_id", prompt="prompt", lora_request=None)
31 + assert isinstance(tokenizer_group.get_lora_tokenizer(None),
32 + PreTrainedTokenizerBase)
33 + assert tokenizer_group.get_lora_tokenizer(
34 + None) == await tokenizer_group.get_lora_tokenizer_async(None)
35 +
36 +
37 +@pytest.mark.asyncio
38 +@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
39 +async def test_tokenizer_group_pool(tokenizer_group_type):
40 + reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
41 + tokenizer_group_pool = get_tokenizer_group(
42 + get_tokenizer_pool_config(tokenizer_group_type),
43 + tokenizer_id="gpt2",
44 + enable_lora=False,
45 + max_num_seqs=1,
46 + max_input_length=None,
47 + )
48 + # Send multiple requests to the tokenizer group pool
49 + # (more than the pool size)
50 + # and check that all requests are processed correctly.
51 + num_requests = tokenizer_group_pool.pool_size * 5
52 + requests = [
53 + tokenizer_group_pool.encode_async(request_id=str(i),
54 + prompt=f"prompt {i}",
55 + lora_request=None)
56 + for i in range(num_requests)
57 + ]
58 + results = await asyncio.gather(*requests)
59 + expected_results = [
60 + reference_tokenizer.encode(f"prompt {i}") for i in range(num_requests)
61 + ]
62 + assert results == expected_results
63 +
64 +
65 +@pytest.mark.asyncio
66 +@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
67 +async def test_tokenizer_group_ray_pool_env_var_propagation(
68 + tokenizer_group_type):
69 + """Test that env vars from caller process are propagated to
70 + tokenizer Ray actors."""
71 + env_var = "MY_ENV_VAR"
72 +
73 + class EnvVarCheckerTokenizerGroup(TokenizerGroup):
74 +
75 + def ping(self):
76 + assert os.environ.get(env_var) == "1"
77 + return super().ping()
78 +
79 + class EnvVarCheckerRayTokenizerGroupPool(RayTokenizerGroupPool):
80 + _worker_cls = EnvVarCheckerTokenizerGroup
81 +
82 + tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
83 + tokenizer_pool = EnvVarCheckerRayTokenizerGroupPool.from_config(
84 + tokenizer_pool_config,
85 + tokenizer_id="gpt2",
86 + enable_lora=False,
87 + max_num_seqs=1,
88 + max_input_length=None)
89 + with pytest.raises(AssertionError):
90 + tokenizer_pool.ping()
91 +
92 + with patch.dict(os.environ, {env_var: "1"}):
93 + tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
94 + tokenizer_pool = EnvVarCheckerRayTokenizerGroupPool.from_config(
95 + tokenizer_pool_config,
96 + tokenizer_id="gpt2",
97 + enable_lora=False,
98 + max_num_seqs=1,
99 + max_input_length=None)
100 + tokenizer_pool.ping()
vllm/config.py
+57
@@ -3,6 +3,7 @@ from dataclasses import dataclass
3 import os
4 from packaging.version import Version
5
6 +import json
7 import torch
8 from transformers import PretrainedConfig
9
@@ -389,6 +390,58 @@ class CacheConfig:
390 logger.warning("Possibly too large swap space. " + msg)
391
392
393 +@dataclass
394 +class TokenizerPoolConfig:
395 + """Configuration for the tokenizer pool.
396 +
397 + Args:
398 + pool_size: Number of tokenizer workers in the pool.
399 + pool_type: Type of the pool.
400 + extra_config: Additional config for the pool.
401 + The way the config will be used depends on the
402 + pool type.
403 + """
404 + pool_size: int
405 + pool_type: str
406 + extra_config: dict
407 +
408 + def __post_init__(self):
409 + if self.pool_type not in ("ray", ):
410 + raise ValueError(f"Unknown pool type: {self.pool_type}")
411 + if not isinstance(self.extra_config, dict):
412 + raise ValueError("extra_config must be a dictionary.")
413 +
414 + @classmethod
415 + def create_config(
416 + cls, tokenizer_pool_size: int, tokenizer_pool_type: str,
417 + tokenizer_pool_extra_config: Optional[Union[str, dict]]
418 + ) -> Optional["TokenizerPoolConfig"]:
419 + """Create a TokenizerPoolConfig from the given parameters.
420 +
421 + If tokenizer_pool_size is 0, return None.
422 +
423 + Args:
424 + tokenizer_pool_size: Number of tokenizer workers in the pool.
425 + tokenizer_pool_type: Type of the pool.
426 + tokenizer_pool_extra_config: Additional config for the pool.
427 + The way the config will be used depends on the
428 + pool type. This can be a JSON string (will be parsed).
429 + """
430 + if tokenizer_pool_size:
431 + if isinstance(tokenizer_pool_extra_config, str):
432 + tokenizer_pool_extra_config_parsed = json.loads(
433 + tokenizer_pool_extra_config)
434 + else:
435 + tokenizer_pool_extra_config_parsed = (
436 + tokenizer_pool_extra_config or {})
437 + tokenizer_pool_config = cls(tokenizer_pool_size,
438 + tokenizer_pool_type,
439 + tokenizer_pool_extra_config_parsed)
440 + else:
441 + tokenizer_pool_config = None
442 + return tokenizer_pool_config
443 +
444 +
445 class ParallelConfig:
446 """Configuration for the distributed execution.
447
@@ -403,6 +456,8 @@ class ParallelConfig:
456 parallel and large models.
457 disable_custom_all_reduce: Disable the custom all-reduce kernel and
458 fall back to NCCL.
459 + tokenizer_pool_config: Config for the tokenizer pool.
460 + If None, will use synchronous tokenization.
461 ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
462 https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
463 """
@@ -414,6 +469,7 @@ class ParallelConfig:
469 worker_use_ray: bool,
470 max_parallel_loading_workers: Optional[int] = None,
471 disable_custom_all_reduce: bool = False,
472 + tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
473 ray_workers_use_nsight: bool = False,
474 placement_group: Optional["PlacementGroup"] = None,
475 ) -> None:
@@ -430,6 +486,7 @@ class ParallelConfig:
486 self.worker_use_ray = worker_use_ray
487 self.max_parallel_loading_workers = max_parallel_loading_workers
488 self.disable_custom_all_reduce = disable_custom_all_reduce
489 + self.tokenizer_pool_config = tokenizer_pool_config
490 self.ray_workers_use_nsight = ray_workers_use_nsight
491 self.placement_group = placement_group
492
vllm/engine/arg_utils.py
+34 -9
@@ -4,7 +4,8 @@ from dataclasses import dataclass
4 from typing import Optional, Tuple
5
6 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
7 - ParallelConfig, SchedulerConfig, LoRAConfig)
7 + ParallelConfig, SchedulerConfig, LoRAConfig,
8 + TokenizerPoolConfig)
9
10
11 @dataclass
@@ -40,6 +41,9 @@ class EngineArgs:
41 enforce_eager: bool = False
42 max_context_len_to_capture: int = 8192
43 disable_custom_all_reduce: bool = False
44 + tokenizer_pool_size: int = 0
45 + tokenizer_pool_type: str = "ray"
46 + tokenizer_pool_extra_config: Optional[dict] = None
47 enable_lora: bool = False
48 max_loras: int = 1
49 max_lora_rank: int = 16
@@ -249,6 +253,25 @@ class EngineArgs:
253 action='store_true',
254 default=EngineArgs.disable_custom_all_reduce,
255 help='See ParallelConfig')
256 + parser.add_argument('--tokenizer-pool-size',
257 + type=int,
258 + default=EngineArgs.tokenizer_pool_size,
259 + help='Size of tokenizer pool to use for '
260 + 'asynchronous tokenization. If 0, will '
261 + 'use synchronous tokenization.')
262 + parser.add_argument('--tokenizer-pool-type',
263 + type=str,
264 + default=EngineArgs.tokenizer_pool_type,
265 + help='Type of tokenizer pool to use for '
266 + 'asynchronous tokenization. Ignored '
267 + 'if tokenizer_pool_size is 0.')
268 + parser.add_argument('--tokenizer-pool-extra-config',
269 + type=str,
270 + default=EngineArgs.tokenizer_pool_extra_config,
271 + help='Extra config for tokenizer pool. '
272 + 'This should be a JSON string that will be '
273 + 'parsed into a dictionary. Ignored if '
274 + 'tokenizer_pool_size is 0.')
275 # LoRA related configs
276 parser.add_argument('--enable-lora',
277 action='store_true',
@@ -312,14 +335,16 @@ class EngineArgs:
335 cache_config = CacheConfig(self.block_size,
336 self.gpu_memory_utilization,
337 self.swap_space, self.kv_cache_dtype,
315 - model_config.get_sliding_window(),
316 - self.enable_prefix_caching)
317 - parallel_config = ParallelConfig(self.pipeline_parallel_size,
318 - self.tensor_parallel_size,
319 - self.worker_use_ray,
320 - self.max_parallel_loading_workers,
321 - self.disable_custom_all_reduce,
322 - self.ray_workers_use_nsight)
338 + model_config.get_sliding_window())
339 + parallel_config = ParallelConfig(
340 + self.pipeline_parallel_size, self.tensor_parallel_size,
341 + self.worker_use_ray, self.max_parallel_loading_workers,
342 + self.disable_custom_all_reduce,
343 + TokenizerPoolConfig.create_config(
344 + self.tokenizer_pool_size,
345 + self.tokenizer_pool_type,
346 + self.tokenizer_pool_extra_config,
347 + ), self.ray_workers_use_nsight)
348 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
349 self.max_num_seqs,
350 model_config.max_model_len,
vllm/engine/llm_engine.py
+11 -4
@@ -17,8 +17,9 @@ from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18 from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
19 SequenceGroupOutput, SequenceOutput, SequenceStatus)
20 -from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
21 - TokenizerGroup)
20 +from vllm.transformers_utils.tokenizer import detokenize_incrementally
21 +from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
22 + get_tokenizer_group)
23 from vllm.utils import Counter
24
25 logger = init_logger(__name__)
@@ -102,6 +103,10 @@ class LLMEngine:
103 parallel_config, scheduler_config,
104 device_config, lora_config)
105
106 + # Ping the tokenizer to ensure liveness if it runs in a
107 + # different process.
108 + self.tokenizer.ping()
109 +
110 # Create the scheduler.
111 # NOTE: the cache_config here have been updated with the numbers of
112 # GPU and CPU blocks, which are profiled in the distributed executor.
@@ -152,6 +157,7 @@ class LLMEngine:
157
158 def _init_tokenizer(self, **tokenizer_init_kwargs):
159 init_kwargs = dict(
160 + tokenizer_id=self.model_config.tokenizer,
161 enable_lora=bool(self.lora_config),
162 max_num_seqs=self.scheduler_config.max_num_seqs,
163 max_input_length=None,
@@ -159,8 +165,9 @@ class LLMEngine:
165 trust_remote_code=self.model_config.trust_remote_code,
166 revision=self.model_config.tokenizer_revision)
167 init_kwargs.update(tokenizer_init_kwargs)
162 - self.tokenizer: TokenizerGroup = TokenizerGroup(
163 - self.model_config.tokenizer, **init_kwargs)
168 +
169 + self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
170 + self.parallel_config.tokenizer_pool_config, **init_kwargs)
171
172 def _verify_args(self) -> None:
173 self.model_config.verify_with_parallel_config(self.parallel_config)
vllm/transformers_utils/tokenizer.py
+38 -61
@@ -5,12 +5,48 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 -from vllm.utils import make_async, LRUCache
8 +from vllm.utils import make_async
9 from vllm.transformers_utils.tokenizers import *
10
11 logger = init_logger(__name__)
12
13
14 +def get_cached_tokenizer(
15 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
16 +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
17 + """Get tokenizer with cached properties.
18 +
19 + This will patch the tokenizer object in place.
20 +
21 + By default, transformers will recompute multiple tokenizer properties
22 + each time they are called, leading to a significant slowdown. This
23 + function caches these properties for faster access."""
24 +
25 + tokenizer_all_special_ids = set(tokenizer.all_special_ids)
26 + tokenizer_all_special_tokens_extended = (
27 + tokenizer.all_special_tokens_extended)
28 + tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
29 +
30 + class CachedTokenizer(tokenizer.__class__):
31 +
32 + @property
33 + def all_special_ids(self):
34 + return tokenizer_all_special_ids
35 +
36 + @property
37 + def all_special_tokens(self):
38 + return tokenizer_all_special_tokens
39 +
40 + @property
41 + def all_special_tokens_extended(self):
42 + return tokenizer_all_special_tokens_extended
43 +
44 + CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
45 +
46 + tokenizer.__class__ = CachedTokenizer
47 + return tokenizer
48 +
49 +
50 def get_tokenizer(
51 tokenizer_name: str,
52 *args,
@@ -64,7 +100,7 @@ def get_tokenizer(
100 logger.warning(
101 "Using a slow tokenizer. This might cause a significant "
102 "slowdown. Consider using a fast tokenizer instead.")
67 - return tokenizer
103 + return get_cached_tokenizer(tokenizer)
104
105
106 def get_lora_tokenizer(lora_request: LoRARequest, *args,
@@ -88,65 +124,6 @@ def get_lora_tokenizer(lora_request: LoRARequest, *args,
124 get_lora_tokenizer_async = make_async(get_lora_tokenizer)
125
126
91 -class TokenizerGroup:
92 - """A group of tokenizers that can be used for LoRA adapters."""
93 -
94 - def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
95 - max_input_length: Optional[int], **tokenizer_config):
96 - self.tokenizer_id = tokenizer_id
97 - self.tokenizer_config = tokenizer_config
98 - self.enable_lora = enable_lora
99 - self.max_input_length = max_input_length
100 - self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
101 - if enable_lora:
102 - self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
103 - else:
104 - self.lora_tokenizers = None
105 -
106 - def encode(self,
107 - prompt: str,
108 - request_id: Optional[str] = None,
109 - lora_request: Optional[LoRARequest] = None) -> List[int]:
110 - tokenizer = self.get_lora_tokenizer(lora_request)
111 - return tokenizer.encode(prompt)
112 -
113 - async def encode_async(
114 - self,
115 - prompt: str,
116 - request_id: Optional[str] = None,
117 - lora_request: Optional[LoRARequest] = None) -> List[int]:
118 - tokenizer = await self.get_lora_tokenizer_async(lora_request)
119 - return tokenizer.encode(prompt)
120 -
121 - def get_lora_tokenizer(
122 - self,
123 - lora_request: Optional[LoRARequest] = None
124 - ) -> "PreTrainedTokenizer":
125 - if not lora_request or not self.enable_lora:
126 - return self.tokenizer
127 - if lora_request.lora_int_id not in self.lora_tokenizers:
128 - tokenizer = (get_lora_tokenizer(
129 - lora_request, **self.tokenizer_config) or self.tokenizer)
130 - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
131 - return tokenizer
132 - else:
133 - return self.lora_tokenizers.get(lora_request.lora_int_id)
134 -
135 - async def get_lora_tokenizer_async(
136 - self,
137 - lora_request: Optional[LoRARequest] = None
138 - ) -> "PreTrainedTokenizer":
139 - if not lora_request or not self.enable_lora:
140 - return self.tokenizer
141 - if lora_request.lora_int_id not in self.lora_tokenizers:
142 - tokenizer = (await get_lora_tokenizer_async(
143 - lora_request, **self.tokenizer_config) or self.tokenizer)
144 - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
145 - return tokenizer
146 - else:
147 - return self.lora_tokenizers.get(lora_request.lora_int_id)
148 -
149 -
127 def _convert_tokens_to_string_with_added_encoders(
128 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
129 output_tokens: List[str],
vllm/transformers_utils/tokenizer_group/__init__.py new
+32
@@ -0,0 +1,32 @@
1 +from typing import Optional
2 +from vllm.config import TokenizerPoolConfig
3 +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
4 + BaseTokenizerGroup)
5 +from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
6 + TokenizerGroup)
7 +from vllm.engine.ray_utils import ray
8 +
9 +if ray:
10 + from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
11 + RayTokenizerGroupPool)
12 +else:
13 + RayTokenizerGroupPool = None
14 +
15 +
16 +def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
17 + **init_kwargs) -> BaseTokenizerGroup:
18 + if tokenizer_pool_config is None:
19 + return TokenizerGroup(**init_kwargs)
20 + if tokenizer_pool_config.pool_type == "ray":
21 + if RayTokenizerGroupPool is None:
22 + raise ImportError(
23 + "RayTokenizerGroupPool is not available. Please install "
24 + "the ray package to use the Ray tokenizer group pool.")
25 + return RayTokenizerGroupPool.from_config(tokenizer_pool_config,
26 + **init_kwargs)
27 + else:
28 + raise ValueError(
29 + f"Unknown pool type: {tokenizer_pool_config.pool_type}")
30 +
31 +
32 +__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py new
+48
@@ -0,0 +1,48 @@
1 +from abc import ABC, abstractmethod
2 +from typing import List, Optional
3 +
4 +from transformers import PreTrainedTokenizer
5 +
6 +from vllm.lora.request import LoRARequest
7 +
8 +
9 +class BaseTokenizerGroup(ABC):
10 + """A group of tokenizers that can be used for LoRA adapters."""
11 +
12 + @abstractmethod
13 + def ping(self) -> bool:
14 + """Check if the tokenizer group is alive."""
15 + pass
16 +
17 + @abstractmethod
18 + def get_max_input_len(self,
19 + lora_request: Optional[LoRARequest] = None
20 + ) -> Optional[int]:
21 + """Get the maximum input length for the LoRA request."""
22 + pass
23 +
24 + @abstractmethod
25 + def encode(self, prompt: str, request_id: Optional[str],
26 + lora_request: Optional[LoRARequest]) -> List[int]:
27 + """Encode a prompt using the tokenizer group."""
28 + pass
29 +
30 + @abstractmethod
31 + async def encode_async(self, prompt: str, request_id: Optional[str],
32 + lora_request: Optional[LoRARequest]) -> List[int]:
33 + """Encode a prompt using the tokenizer group."""
34 + pass
35 +
36 + @abstractmethod
37 + def get_lora_tokenizer(
38 + self,
39 + lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
40 + """Get a tokenizer for a LoRA request."""
41 + pass
42 +
43 + @abstractmethod
44 + async def get_lora_tokenizer_async(
45 + self,
46 + lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
47 + """Get a tokenizer for a LoRA request."""
48 + pass
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py new
+166
@@ -0,0 +1,166 @@
1 +import asyncio
2 +import os
3 +from typing import List, Optional
4 +
5 +from transformers import PreTrainedTokenizer
6 +
7 +from vllm.config import TokenizerPoolConfig
8 +from vllm.lora.request import LoRARequest
9 +from vllm.engine.ray_utils import ray
10 +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
11 + BaseTokenizerGroup)
12 +from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
13 + TokenizerGroup)
14 +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
15 +
16 +
17 +class RayTokenizerGroupPool(BaseTokenizerGroup):
18 + """A Ray-based pool of TokenizerGroups for async tokenization."""
19 +
20 + # Class to use for workers making up the pool.
21 + _worker_cls = TokenizerGroup
22 +
23 + @classmethod
24 + def from_config(cls, tokenizer_pool_config: TokenizerPoolConfig,
25 + **init_kwargs) -> "RayTokenizerGroupPool":
26 + ray_actor_options = (tokenizer_pool_config.extra_config or {
27 + "num_cpus": 0
28 + })
29 + ray_actor_options.setdefault(
30 + "scheduling_strategy",
31 + NodeAffinitySchedulingStrategy(
32 + node_id=ray.get_runtime_context().get_node_id(), soft=True))
33 +
34 + # Carry over the env vars to the actors.
35 + # This is necessary for API keys and such.
36 + ray_actor_options.setdefault("runtime_env", {})
37 + _carry_over_env_vars_to_runtime_env(ray_actor_options["runtime_env"])
38 +
39 + init_kwargs["num_actors"] = tokenizer_pool_config.pool_size
40 + init_kwargs["ray_actor_options"] = ray_actor_options
41 +
42 + return cls(**init_kwargs)
43 +
44 + def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
45 + max_input_length: Optional[int], num_actors: int,
46 + ray_actor_options: dict, **tokenizer_config):
47 + # Store a local copy of the TokenizerGroup for quick access
48 + # to underlying HF tokenizers.
49 + self._local_tokenizer_group = self._worker_cls(
50 + tokenizer_id=tokenizer_id,
51 + enable_lora=enable_lora,
52 + max_num_seqs=max_num_seqs,
53 + max_input_length=max_input_length,
54 + )
55 +
56 + ray_tokenizer_group_cls = ray.remote(
57 + self._worker_cls).options(**ray_actor_options)
58 + self.tokenizer_actors = [
59 + ray_tokenizer_group_cls.remote(tokenizer_id, enable_lora,
60 + max_num_seqs, max_input_length,
61 + **tokenizer_config)
62 + for _ in range(num_actors)
63 + ]
64 + self._idle_actors: Optional[asyncio.Queue] = None
65 +
66 + @property
67 + def pool_size(self) -> int:
68 + return len(self.tokenizer_actors)
69 +
70 + def ping(self):
71 + return ray.get(
72 + [actor.ping.remote() for actor in self.tokenizer_actors])
73 +
74 + def _ensure_queue_initialized(self):
75 + if self._idle_actors is None:
76 + self._idle_actors = asyncio.Queue()
77 + for actor in self.tokenizer_actors:
78 + self._idle_actors.put_nowait(actor)
79 +
80 + def encode(self,
81 + prompt: str,
82 + request_id: Optional[str] = None,
83 + lora_request: Optional[LoRARequest] = None) -> List[int]:
84 + """Encode a prompt using the tokenizer group.
85 +
86 + We pick an idle actor and use it to encode the prompt.
87 + The actor is then put back in the queue for future use.
88 + This is blocking.
89 + """
90 + self._ensure_queue_initialized()
91 +
92 + if self._idle_actors.empty():
93 + raise RuntimeError("No idle actors available.")
94 + actor = self._idle_actors.get_nowait()
95 + try:
96 + ret = ray.get(
97 + actor.encode.remote(request_id=request_id,
98 + prompt=prompt,
99 + lora_request=lora_request))
100 + finally:
101 + # Put the actor back in the queue.
102 + # This is done in a finally block to ensure that the actor is
103 + # always put back in the queue, even if an exception/cancellation
104 + # is raised.
105 + self._idle_actors.put_nowait(actor)
106 + return ret
107 +
108 + async def encode_async(
109 + self,
110 + prompt: str,
111 + request_id: Optional[str] = None,
112 + lora_request: Optional[LoRARequest] = None) -> List[int]:
113 + """Encode a prompt using the tokenizer group.
114 +
115 + We pick an idle actor and use it to encode the prompt.
116 + If there are no idle actors, we wait until one becomes
117 + available.
118 + The actor is then put back in the queue for future use.
119 + This is non-blocking.
120 + """
121 + self._ensure_queue_initialized()
122 +
123 + actor = await self._idle_actors.get()
124 + try:
125 + ret = await actor.encode.remote(request_id=request_id,
126 + prompt=prompt,
127 + lora_request=lora_request)
128 + finally:
129 + # Put the actor back in the queue.
130 + # This is done in a finally block to ensure that the actor is
131 + # always put back in the queue, even if an exception/cancellation
132 + # is raised.
133 + self._idle_actors.put_nowait(actor)
134 + return ret
135 +
136 + def get_max_input_len(self,
137 + lora_request: Optional[LoRARequest] = None
138 + ) -> Optional[int]:
139 + """Get the maximum input length for the LoRA request."""
140 + return self._local_tokenizer_group.get_max_input_len(lora_request)
141 +
142 + def get_lora_tokenizer(
143 + self,
144 + lora_request: Optional[LoRARequest] = None
145 + ) -> "PreTrainedTokenizer":
146 + return self._local_tokenizer_group.get_lora_tokenizer(lora_request)
147 +
148 + async def get_lora_tokenizer_async(
149 + self,
150 + lora_request: Optional[LoRARequest] = None
151 + ) -> "PreTrainedTokenizer":
152 + return await self._local_tokenizer_group.get_lora_tokenizer_async(
153 + lora_request)
154 +
155 +
156 +def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None:
157 + """Copy over all current process environment variables to the runtime_env.
158 +
159 + The variables in runtime_env will take precedence over the current process
160 + environment variables.
161 +
162 + runtime_env will be modified in place."""
163 + env_vars = os.environ.copy()
164 + runtime_env.setdefault("env_vars", {})
165 + env_vars.update(runtime_env["env_vars"])
166 + runtime_env["env_vars"] = env_vars
vllm/transformers_utils/tokenizer_group/tokenizer_group.py new
+80
@@ -0,0 +1,80 @@
1 +from typing import List, Optional
2 +
3 +from transformers import PreTrainedTokenizer
4 +
5 +from vllm.lora.request import LoRARequest
6 +from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
7 + get_lora_tokenizer_async)
8 +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
9 + BaseTokenizerGroup)
10 +from vllm.utils import LRUCache
11 +from vllm.transformers_utils.tokenizer import get_tokenizer
12 +
13 +
14 +class TokenizerGroup(BaseTokenizerGroup):
15 + """A group of tokenizers that can be used for LoRA adapters."""
16 +
17 + def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
18 + max_input_length: Optional[int], **tokenizer_config):
19 + self.tokenizer_id = tokenizer_id
20 + self.tokenizer_config = tokenizer_config
21 + self.enable_lora = enable_lora
22 + self.max_input_length = max_input_length
23 + self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24 + if enable_lora:
25 + self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
26 + else:
27 + self.lora_tokenizers = None
28 +
29 + def ping(self) -> bool:
30 + """Check if the tokenizer group is alive."""
31 + return True
32 +
33 + def get_max_input_len(self,
34 + lora_request: Optional[LoRARequest] = None
35 + ) -> Optional[int]:
36 + """Get the maximum input length for the LoRA request."""
37 + return self.max_input_length
38 +
39 + def encode(self,
40 + prompt: str,
41 + request_id: Optional[str] = None,
42 + lora_request: Optional[LoRARequest] = None) -> List[int]:
43 + tokenizer = self.get_lora_tokenizer(lora_request)
44 + return tokenizer.encode(prompt)
45 +
46 + async def encode_async(
47 + self,
48 + prompt: str,
49 + request_id: Optional[str] = None,
50 + lora_request: Optional[LoRARequest] = None) -> List[int]:
51 + tokenizer = await self.get_lora_tokenizer_async(lora_request)
52 + return tokenizer.encode(prompt)
53 +
54 + def get_lora_tokenizer(
55 + self,
56 + lora_request: Optional[LoRARequest] = None
57 + ) -> "PreTrainedTokenizer":
58 + if not lora_request or not self.enable_lora:
59 + return self.tokenizer
60 + if lora_request.lora_int_id not in self.lora_tokenizers:
61 + tokenizer = (get_lora_tokenizer(
62 + lora_request, **self.tokenizer_config) or self.tokenizer)
63 + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
64 + return tokenizer
65 + else:
66 + return self.lora_tokenizers.get(lora_request.lora_int_id)
67 +
68 + async def get_lora_tokenizer_async(
69 + self,
70 + lora_request: Optional[LoRARequest] = None
71 + ) -> "PreTrainedTokenizer":
72 + if not lora_request or not self.enable_lora:
73 + return self.tokenizer
74 + if lora_request.lora_int_id not in self.lora_tokenizers:
75 + tokenizer = (await get_lora_tokenizer_async(
76 + lora_request, **self.tokenizer_config) or self.tokenizer)
77 + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
78 + return tokenizer
79 + else:
80 + return self.lora_tokenizers.get(lora_request.lora_int_id)