A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Asynchronous tokenization (#2879)
Antoni Baum committed
Mar 15, 2024 at 16:37 UTC
fb96c1e98c05ffa35dd48416f68e88edb2f9eb34
17 files changed
+658
-153
.buildkite/test-pipeline.yaml
+1
-1
@@ -28,7 +28,7 @@ steps:
28
num_gpus: 2 # only support 1 or 2 for now.
29
30
- label: Engine Test
31
- command: pytest -v -s engine test_sequence.py
31
+ command: pytest -v -s engine tokenization test_sequence.py
32
33
- label: Entrypoints Test
34
command: pytest -v -s entrypoints
tests/async_engine/test_api_server.py
+7
-9
@@ -25,23 +25,21 @@ def _query_server_long(prompt: str) -> dict:
25
26
27
@pytest.fixture
28
-def api_server():
28
+def api_server(tokenizer_pool_size: int):
29
script_path = Path(__file__).parent.joinpath(
30
"api_server_async_engine.py").absolute()
31
uvicorn_process = subprocess.Popen([
32
- sys.executable,
33
- "-u",
34
- str(script_path),
35
- "--model",
36
- "facebook/opt-125m",
37
- "--host",
38
- "127.0.0.1",
32
+ sys.executable, "-u",
33
+ str(script_path), "--model", "facebook/opt-125m", "--host",
34
+ "127.0.0.1", "--tokenizer-pool-size",
35
+ str(tokenizer_pool_size)
36
])
37
yield
38
uvicorn_process.terminate()
39
40
44
-def test_api_server(api_server):
41
+@pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
42
+def test_api_server(api_server, tokenizer_pool_size: int):
43
"""
44
Run the API server and test it.
45
tests/conftest.py
+11
@@ -7,6 +7,7 @@ from transformers import AutoModelForCausalLM
7
8
from vllm import LLM, SamplingParams
9
from vllm.transformers_utils.tokenizer import get_tokenizer
10
+from vllm.config import TokenizerPoolConfig
11
12
_TEST_DIR = os.path.dirname(__file__)
13
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
@@ -258,3 +259,13 @@ class VllmRunner:
259
@pytest.fixture
260
def vllm_runner():
261
return VllmRunner
262
+
263
+
264
+def get_tokenizer_pool_config(tokenizer_group_type):
265
+ if tokenizer_group_type is None:
266
+ return None
267
+ if tokenizer_group_type == "ray":
268
+ return TokenizerPoolConfig(pool_size=1,
269
+ pool_type="ray",
270
+ extra_config={})
271
+ raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
tests/lora/test_tokenizer.py
deleted
-69
@@ -1,69 +0,0 @@
1
-import pytest
2
-from transformers import AutoTokenizer, PreTrainedTokenizerBase
3
-
4
-from vllm.lora.request import LoRARequest
5
-from vllm.transformers_utils.tokenizer import TokenizerGroup, get_lora_tokenizer
6
-
7
-
8
-@pytest.mark.asyncio
9
-async def test_transformers_tokenizer():
10
- reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
11
- tokenizer = TokenizerGroup(
12
- tokenizer_id="gpt2",
13
- enable_lora=False,
14
- max_num_seqs=1,
15
- max_input_length=None,
16
- )
17
- assert reference_tokenizer.encode("prompt") == tokenizer.encode(
18
- request_id="request_id", prompt="prompt", lora_request=None)
19
- assert reference_tokenizer.encode(
20
- "prompt") == await tokenizer.encode_async(request_id="request_id",
21
- prompt="prompt",
22
- lora_request=None)
23
- assert isinstance(tokenizer.get_lora_tokenizer(None),
24
- PreTrainedTokenizerBase)
25
- assert tokenizer.get_lora_tokenizer(
26
- None) == await tokenizer.get_lora_tokenizer_async(None)
27
-
28
-
29
-@pytest.mark.asyncio
30
-async def test_transformers_tokenizer_lora(sql_lora_files):
31
- reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
32
- tokenizer = TokenizerGroup(
33
- tokenizer_id="gpt2",
34
- enable_lora=True,
35
- max_num_seqs=1,
36
- max_input_length=None,
37
- )
38
- lora_request = LoRARequest("1", 1, sql_lora_files)
39
- assert reference_tokenizer.encode("prompt") == tokenizer.encode(
40
- request_id="request_id", prompt="prompt", lora_request=lora_request)
41
- assert reference_tokenizer.encode(
42
- "prompt") == await tokenizer.encode_async(request_id="request_id",
43
- prompt="prompt",
44
- lora_request=lora_request)
45
- assert isinstance(tokenizer.get_lora_tokenizer(None),
46
- PreTrainedTokenizerBase)
47
- assert tokenizer.get_lora_tokenizer(
48
- None) == await tokenizer.get_lora_tokenizer_async(None)
49
-
50
- assert isinstance(tokenizer.get_lora_tokenizer(lora_request),
51
- PreTrainedTokenizerBase)
52
- assert tokenizer.get_lora_tokenizer(
53
- lora_request) != tokenizer.get_lora_tokenizer(None)
54
- assert tokenizer.get_lora_tokenizer(
55
- lora_request) == await tokenizer.get_lora_tokenizer_async(lora_request)
56
-
57
-
58
-def test_get_lora_tokenizer(sql_lora_files, tmpdir):
59
- lora_request = None
60
- tokenizer = get_lora_tokenizer(lora_request)
61
- assert not tokenizer
62
-
63
- lora_request = LoRARequest("1", 1, sql_lora_files)
64
- tokenizer = get_lora_tokenizer(lora_request)
65
- assert tokenizer.get_added_vocab()
66
-
67
- lora_request = LoRARequest("1", 1, str(tmpdir))
68
- tokenizer = get_lora_tokenizer(lora_request)
69
- assert not tokenizer
tests/lora/test_tokenizer_group.py
new
+53
@@ -0,0 +1,53 @@
1
+import pytest
2
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
3
+from vllm.lora.request import LoRARequest
4
+from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
5
+from vllm.transformers_utils.tokenizer import get_lora_tokenizer
6
+from ..conftest import get_tokenizer_pool_config
7
+
8
+
9
+@pytest.mark.asyncio
10
+@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
11
+async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
12
+ reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
13
+ tokenizer_group = get_tokenizer_group(
14
+ get_tokenizer_pool_config(tokenizer_group_type),
15
+ tokenizer_id="gpt2",
16
+ enable_lora=True,
17
+ max_num_seqs=1,
18
+ max_input_length=None,
19
+ )
20
+ lora_request = LoRARequest("1", 1, sql_lora_files)
21
+ assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
22
+ request_id="request_id", prompt="prompt", lora_request=lora_request)
23
+ assert reference_tokenizer.encode(
24
+ "prompt") == await tokenizer_group.encode_async(
25
+ request_id="request_id",
26
+ prompt="prompt",
27
+ lora_request=lora_request)
28
+ assert isinstance(tokenizer_group.get_lora_tokenizer(None),
29
+ PreTrainedTokenizerBase)
30
+ assert tokenizer_group.get_lora_tokenizer(
31
+ None) == await tokenizer_group.get_lora_tokenizer_async(None)
32
+
33
+ assert isinstance(tokenizer_group.get_lora_tokenizer(lora_request),
34
+ PreTrainedTokenizerBase)
35
+ assert tokenizer_group.get_lora_tokenizer(
36
+ lora_request) != tokenizer_group.get_lora_tokenizer(None)
37
+ assert tokenizer_group.get_lora_tokenizer(
38
+ lora_request) == await tokenizer_group.get_lora_tokenizer_async(
39
+ lora_request)
40
+
41
+
42
+def test_get_lora_tokenizer(sql_lora_files, tmpdir):
43
+ lora_request = None
44
+ tokenizer = get_lora_tokenizer(lora_request)
45
+ assert not tokenizer
46
+
47
+ lora_request = LoRARequest("1", 1, sql_lora_files)
48
+ tokenizer = get_lora_tokenizer(lora_request)
49
+ assert tokenizer.get_added_vocab()
50
+
51
+ lora_request = LoRARequest("1", 1, str(tmpdir))
52
+ tokenizer = get_lora_tokenizer(lora_request)
53
+ assert not tokenizer
tests/tokenization/__init__.py
tests/tokenization/test_cached_tokenizer.py
new
+20
@@ -0,0 +1,20 @@
1
+from copy import deepcopy
2
+from vllm.transformers_utils.tokenizer import get_cached_tokenizer
3
+from transformers import AutoTokenizer
4
+
5
+
6
+def test_cached_tokenizer():
7
+ reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
8
+ reference_tokenizer.add_special_tokens({"cls_token": "<CLS>"})
9
+ reference_tokenizer.add_special_tokens(
10
+ {"additional_special_tokens": ["<SEP>"]})
11
+ cached_tokenizer = get_cached_tokenizer(deepcopy(reference_tokenizer))
12
+
13
+ assert reference_tokenizer.encode("prompt") == cached_tokenizer.encode(
14
+ "prompt")
15
+ assert set(reference_tokenizer.all_special_ids) == set(
16
+ cached_tokenizer.all_special_ids)
17
+ assert set(reference_tokenizer.all_special_tokens) == set(
18
+ cached_tokenizer.all_special_tokens)
19
+ assert set(reference_tokenizer.all_special_tokens_extended) == set(
20
+ cached_tokenizer.all_special_tokens_extended)
tests/tokenization/test_detokenize.py
renamed
tests/tokenization/test_tokenizer_group.py
new
+100
@@ -0,0 +1,100 @@
1
+import os
2
+import pytest
3
+import asyncio
4
+from unittest.mock import patch
5
+
6
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
7
+from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
8
+from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
9
+ RayTokenizerGroupPool)
10
+from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
11
+ TokenizerGroup)
12
+from ..conftest import get_tokenizer_pool_config
13
+
14
+
15
+@pytest.mark.asyncio
16
+@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
17
+async def test_tokenizer_group(tokenizer_group_type):
18
+ reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
19
+ tokenizer_group = get_tokenizer_group(
20
+ get_tokenizer_pool_config(tokenizer_group_type),
21
+ tokenizer_id="gpt2",
22
+ enable_lora=False,
23
+ max_num_seqs=1,
24
+ max_input_length=None,
25
+ )
26
+ assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
27
+ request_id="request_id", prompt="prompt", lora_request=None)
28
+ assert reference_tokenizer.encode(
29
+ "prompt") == await tokenizer_group.encode_async(
30
+ request_id="request_id", prompt="prompt", lora_request=None)
31
+ assert isinstance(tokenizer_group.get_lora_tokenizer(None),
32
+ PreTrainedTokenizerBase)
33
+ assert tokenizer_group.get_lora_tokenizer(
34
+ None) == await tokenizer_group.get_lora_tokenizer_async(None)
35
+
36
+
37
+@pytest.mark.asyncio
38
+@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
39
+async def test_tokenizer_group_pool(tokenizer_group_type):
40
+ reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
41
+ tokenizer_group_pool = get_tokenizer_group(
42
+ get_tokenizer_pool_config(tokenizer_group_type),
43
+ tokenizer_id="gpt2",
44
+ enable_lora=False,
45
+ max_num_seqs=1,
46
+ max_input_length=None,
47
+ )
48
+ # Send multiple requests to the tokenizer group pool
49
+ # (more than the pool size)
50
+ # and check that all requests are processed correctly.
51
+ num_requests = tokenizer_group_pool.pool_size * 5
52
+ requests = [
53
+ tokenizer_group_pool.encode_async(request_id=str(i),
54
+ prompt=f"prompt {i}",
55
+ lora_request=None)
56
+ for i in range(num_requests)
57
+ ]
58
+ results = await asyncio.gather(*requests)
59
+ expected_results = [
60
+ reference_tokenizer.encode(f"prompt {i}") for i in range(num_requests)
61
+ ]
62
+ assert results == expected_results
63
+
64
+
65
+@pytest.mark.asyncio
66
+@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
67
+async def test_tokenizer_group_ray_pool_env_var_propagation(
68
+ tokenizer_group_type):
69
+ """Test that env vars from caller process are propagated to
70
+ tokenizer Ray actors."""
71
+ env_var = "MY_ENV_VAR"
72
+
73
+ class EnvVarCheckerTokenizerGroup(TokenizerGroup):
74
+
75
+ def ping(self):
76
+ assert os.environ.get(env_var) == "1"
77
+ return super().ping()
78
+
79
+ class EnvVarCheckerRayTokenizerGroupPool(RayTokenizerGroupPool):
80
+ _worker_cls = EnvVarCheckerTokenizerGroup
81
+
82
+ tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
83
+ tokenizer_pool = EnvVarCheckerRayTokenizerGroupPool.from_config(
84
+ tokenizer_pool_config,
85
+ tokenizer_id="gpt2",
86
+ enable_lora=False,
87
+ max_num_seqs=1,
88
+ max_input_length=None)
89
+ with pytest.raises(AssertionError):
90
+ tokenizer_pool.ping()
91
+
92
+ with patch.dict(os.environ, {env_var: "1"}):
93
+ tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
94
+ tokenizer_pool = EnvVarCheckerRayTokenizerGroupPool.from_config(
95
+ tokenizer_pool_config,
96
+ tokenizer_id="gpt2",
97
+ enable_lora=False,
98
+ max_num_seqs=1,
99
+ max_input_length=None)
100
+ tokenizer_pool.ping()
vllm/config.py
+57
@@ -3,6 +3,7 @@ from dataclasses import dataclass
3
import os
4
from packaging.version import Version
5
6
+import json
7
import torch
8
from transformers import PretrainedConfig
9
@@ -389,6 +390,58 @@ class CacheConfig:
390
logger.warning("Possibly too large swap space. " + msg)
391
392
393
+@dataclass
394
+class TokenizerPoolConfig:
395
+ """Configuration for the tokenizer pool.
396
+
397
+ Args:
398
+ pool_size: Number of tokenizer workers in the pool.
399
+ pool_type: Type of the pool.
400
+ extra_config: Additional config for the pool.
401
+ The way the config will be used depends on the
402
+ pool type.
403
+ """
404
+ pool_size: int
405
+ pool_type: str
406
+ extra_config: dict
407
+
408
+ def __post_init__(self):
409
+ if self.pool_type not in ("ray", ):
410
+ raise ValueError(f"Unknown pool type: {self.pool_type}")
411
+ if not isinstance(self.extra_config, dict):
412
+ raise ValueError("extra_config must be a dictionary.")
413
+
414
+ @classmethod
415
+ def create_config(
416
+ cls, tokenizer_pool_size: int, tokenizer_pool_type: str,
417
+ tokenizer_pool_extra_config: Optional[Union[str, dict]]
418
+ ) -> Optional["TokenizerPoolConfig"]:
419
+ """Create a TokenizerPoolConfig from the given parameters.
420
+
421
+ If tokenizer_pool_size is 0, return None.
422
+
423
+ Args:
424
+ tokenizer_pool_size: Number of tokenizer workers in the pool.
425
+ tokenizer_pool_type: Type of the pool.
426
+ tokenizer_pool_extra_config: Additional config for the pool.
427
+ The way the config will be used depends on the
428
+ pool type. This can be a JSON string (will be parsed).
429
+ """
430
+ if tokenizer_pool_size:
431
+ if isinstance(tokenizer_pool_extra_config, str):
432
+ tokenizer_pool_extra_config_parsed = json.loads(
433
+ tokenizer_pool_extra_config)
434
+ else:
435
+ tokenizer_pool_extra_config_parsed = (
436
+ tokenizer_pool_extra_config or {})
437
+ tokenizer_pool_config = cls(tokenizer_pool_size,
438
+ tokenizer_pool_type,
439
+ tokenizer_pool_extra_config_parsed)
440
+ else:
441
+ tokenizer_pool_config = None
442
+ return tokenizer_pool_config
443
+
444
+
445
class ParallelConfig:
446
"""Configuration for the distributed execution.
447
@@ -403,6 +456,8 @@ class ParallelConfig:
456
parallel and large models.
457
disable_custom_all_reduce: Disable the custom all-reduce kernel and
458
fall back to NCCL.
459
+ tokenizer_pool_config: Config for the tokenizer pool.
460
+ If None, will use synchronous tokenization.
461
ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
462
https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
463
"""
@@ -414,6 +469,7 @@ class ParallelConfig:
469
worker_use_ray: bool,
470
max_parallel_loading_workers: Optional[int] = None,
471
disable_custom_all_reduce: bool = False,
472
+ tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
473
ray_workers_use_nsight: bool = False,
474
placement_group: Optional["PlacementGroup"] = None,
475
) -> None:
@@ -430,6 +486,7 @@ class ParallelConfig:
486
self.worker_use_ray = worker_use_ray
487
self.max_parallel_loading_workers = max_parallel_loading_workers
488
self.disable_custom_all_reduce = disable_custom_all_reduce
489
+ self.tokenizer_pool_config = tokenizer_pool_config
490
self.ray_workers_use_nsight = ray_workers_use_nsight
491
self.placement_group = placement_group
492
vllm/engine/arg_utils.py
+34
-9
@@ -4,7 +4,8 @@ from dataclasses import dataclass
4
from typing import Optional, Tuple
5
6
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
7
- ParallelConfig, SchedulerConfig, LoRAConfig)
7
+ ParallelConfig, SchedulerConfig, LoRAConfig,
8
+ TokenizerPoolConfig)
9
10
11
@dataclass
@@ -40,6 +41,9 @@ class EngineArgs:
41
enforce_eager: bool = False
42
max_context_len_to_capture: int = 8192
43
disable_custom_all_reduce: bool = False
44
+ tokenizer_pool_size: int = 0
45
+ tokenizer_pool_type: str = "ray"
46
+ tokenizer_pool_extra_config: Optional[dict] = None
47
enable_lora: bool = False
48
max_loras: int = 1
49
max_lora_rank: int = 16
@@ -249,6 +253,25 @@ class EngineArgs:
253
action='store_true',
254
default=EngineArgs.disable_custom_all_reduce,
255
help='See ParallelConfig')
256
+ parser.add_argument('--tokenizer-pool-size',
257
+ type=int,
258
+ default=EngineArgs.tokenizer_pool_size,
259
+ help='Size of tokenizer pool to use for '
260
+ 'asynchronous tokenization. If 0, will '
261
+ 'use synchronous tokenization.')
262
+ parser.add_argument('--tokenizer-pool-type',
263
+ type=str,
264
+ default=EngineArgs.tokenizer_pool_type,
265
+ help='Type of tokenizer pool to use for '
266
+ 'asynchronous tokenization. Ignored '
267
+ 'if tokenizer_pool_size is 0.')
268
+ parser.add_argument('--tokenizer-pool-extra-config',
269
+ type=str,
270
+ default=EngineArgs.tokenizer_pool_extra_config,
271
+ help='Extra config for tokenizer pool. '
272
+ 'This should be a JSON string that will be '
273
+ 'parsed into a dictionary. Ignored if '
274
+ 'tokenizer_pool_size is 0.')
275
# LoRA related configs
276
parser.add_argument('--enable-lora',
277
action='store_true',
@@ -312,14 +335,16 @@ class EngineArgs:
335
cache_config = CacheConfig(self.block_size,
336
self.gpu_memory_utilization,
337
self.swap_space, self.kv_cache_dtype,
315
- model_config.get_sliding_window(),
316
- self.enable_prefix_caching)
317
- parallel_config = ParallelConfig(self.pipeline_parallel_size,
318
- self.tensor_parallel_size,
319
- self.worker_use_ray,
320
- self.max_parallel_loading_workers,
321
- self.disable_custom_all_reduce,
322
- self.ray_workers_use_nsight)
338
+ model_config.get_sliding_window())
339
+ parallel_config = ParallelConfig(
340
+ self.pipeline_parallel_size, self.tensor_parallel_size,
341
+ self.worker_use_ray, self.max_parallel_loading_workers,
342
+ self.disable_custom_all_reduce,
343
+ TokenizerPoolConfig.create_config(
344
+ self.tokenizer_pool_size,
345
+ self.tokenizer_pool_type,
346
+ self.tokenizer_pool_extra_config,
347
+ ), self.ray_workers_use_nsight)
348
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
349
self.max_num_seqs,
350
model_config.max_model_len,
vllm/engine/llm_engine.py
+11
-4
@@ -17,8 +17,9 @@ from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
19
SequenceGroupOutput, SequenceOutput, SequenceStatus)
20
-from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
21
- TokenizerGroup)
20
+from vllm.transformers_utils.tokenizer import detokenize_incrementally
21
+from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
22
+ get_tokenizer_group)
23
from vllm.utils import Counter
24
25
logger = init_logger(__name__)
@@ -102,6 +103,10 @@ class LLMEngine:
103
parallel_config, scheduler_config,
104
device_config, lora_config)
105
106
+ # Ping the tokenizer to ensure liveness if it runs in a
107
+ # different process.
108
+ self.tokenizer.ping()
109
+
110
# Create the scheduler.
111
# NOTE: the cache_config here have been updated with the numbers of
112
# GPU and CPU blocks, which are profiled in the distributed executor.
@@ -152,6 +157,7 @@ class LLMEngine:
157
158
def _init_tokenizer(self, **tokenizer_init_kwargs):
159
init_kwargs = dict(
160
+ tokenizer_id=self.model_config.tokenizer,
161
enable_lora=bool(self.lora_config),
162
max_num_seqs=self.scheduler_config.max_num_seqs,
163
max_input_length=None,
@@ -159,8 +165,9 @@ class LLMEngine:
165
trust_remote_code=self.model_config.trust_remote_code,
166
revision=self.model_config.tokenizer_revision)
167
init_kwargs.update(tokenizer_init_kwargs)
162
- self.tokenizer: TokenizerGroup = TokenizerGroup(
163
- self.model_config.tokenizer, **init_kwargs)
168
+
169
+ self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
170
+ self.parallel_config.tokenizer_pool_config, **init_kwargs)
171
172
def _verify_args(self) -> None:
173
self.model_config.verify_with_parallel_config(self.parallel_config)
vllm/transformers_utils/tokenizer.py
+38
-61
@@ -5,12 +5,48 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
-from vllm.utils import make_async, LRUCache
8
+from vllm.utils import make_async
9
from vllm.transformers_utils.tokenizers import *
10
11
logger = init_logger(__name__)
12
13
14
+def get_cached_tokenizer(
15
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
16
+) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
17
+ """Get tokenizer with cached properties.
18
+
19
+ This will patch the tokenizer object in place.
20
+
21
+ By default, transformers will recompute multiple tokenizer properties
22
+ each time they are called, leading to a significant slowdown. This
23
+ function caches these properties for faster access."""
24
+
25
+ tokenizer_all_special_ids = set(tokenizer.all_special_ids)
26
+ tokenizer_all_special_tokens_extended = (
27
+ tokenizer.all_special_tokens_extended)
28
+ tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
29
+
30
+ class CachedTokenizer(tokenizer.__class__):
31
+
32
+ @property
33
+ def all_special_ids(self):
34
+ return tokenizer_all_special_ids
35
+
36
+ @property
37
+ def all_special_tokens(self):
38
+ return tokenizer_all_special_tokens
39
+
40
+ @property
41
+ def all_special_tokens_extended(self):
42
+ return tokenizer_all_special_tokens_extended
43
+
44
+ CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
45
+
46
+ tokenizer.__class__ = CachedTokenizer
47
+ return tokenizer
48
+
49
+
50
def get_tokenizer(
51
tokenizer_name: str,
52
*args,
@@ -64,7 +100,7 @@ def get_tokenizer(
100
logger.warning(
101
"Using a slow tokenizer. This might cause a significant "
102
"slowdown. Consider using a fast tokenizer instead.")
67
- return tokenizer
103
+ return get_cached_tokenizer(tokenizer)
104
105
106
def get_lora_tokenizer(lora_request: LoRARequest, *args,
@@ -88,65 +124,6 @@ def get_lora_tokenizer(lora_request: LoRARequest, *args,
124
get_lora_tokenizer_async = make_async(get_lora_tokenizer)
125
126
91
-class TokenizerGroup:
92
- """A group of tokenizers that can be used for LoRA adapters."""
93
-
94
- def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
95
- max_input_length: Optional[int], **tokenizer_config):
96
- self.tokenizer_id = tokenizer_id
97
- self.tokenizer_config = tokenizer_config
98
- self.enable_lora = enable_lora
99
- self.max_input_length = max_input_length
100
- self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
101
- if enable_lora:
102
- self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
103
- else:
104
- self.lora_tokenizers = None
105
-
106
- def encode(self,
107
- prompt: str,
108
- request_id: Optional[str] = None,
109
- lora_request: Optional[LoRARequest] = None) -> List[int]:
110
- tokenizer = self.get_lora_tokenizer(lora_request)
111
- return tokenizer.encode(prompt)
112
-
113
- async def encode_async(
114
- self,
115
- prompt: str,
116
- request_id: Optional[str] = None,
117
- lora_request: Optional[LoRARequest] = None) -> List[int]:
118
- tokenizer = await self.get_lora_tokenizer_async(lora_request)
119
- return tokenizer.encode(prompt)
120
-
121
- def get_lora_tokenizer(
122
- self,
123
- lora_request: Optional[LoRARequest] = None
124
- ) -> "PreTrainedTokenizer":
125
- if not lora_request or not self.enable_lora:
126
- return self.tokenizer
127
- if lora_request.lora_int_id not in self.lora_tokenizers:
128
- tokenizer = (get_lora_tokenizer(
129
- lora_request, **self.tokenizer_config) or self.tokenizer)
130
- self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
131
- return tokenizer
132
- else:
133
- return self.lora_tokenizers.get(lora_request.lora_int_id)
134
-
135
- async def get_lora_tokenizer_async(
136
- self,
137
- lora_request: Optional[LoRARequest] = None
138
- ) -> "PreTrainedTokenizer":
139
- if not lora_request or not self.enable_lora:
140
- return self.tokenizer
141
- if lora_request.lora_int_id not in self.lora_tokenizers:
142
- tokenizer = (await get_lora_tokenizer_async(
143
- lora_request, **self.tokenizer_config) or self.tokenizer)
144
- self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
145
- return tokenizer
146
- else:
147
- return self.lora_tokenizers.get(lora_request.lora_int_id)
148
-
149
-
127
def _convert_tokens_to_string_with_added_encoders(
128
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
129
output_tokens: List[str],
vllm/transformers_utils/tokenizer_group/__init__.py
new
+32
@@ -0,0 +1,32 @@
1
+from typing import Optional
2
+from vllm.config import TokenizerPoolConfig
3
+from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
4
+ BaseTokenizerGroup)
5
+from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
6
+ TokenizerGroup)
7
+from vllm.engine.ray_utils import ray
8
+
9
+if ray:
10
+ from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
11
+ RayTokenizerGroupPool)
12
+else:
13
+ RayTokenizerGroupPool = None
14
+
15
+
16
+def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
17
+ **init_kwargs) -> BaseTokenizerGroup:
18
+ if tokenizer_pool_config is None:
19
+ return TokenizerGroup(**init_kwargs)
20
+ if tokenizer_pool_config.pool_type == "ray":
21
+ if RayTokenizerGroupPool is None:
22
+ raise ImportError(
23
+ "RayTokenizerGroupPool is not available. Please install "
24
+ "the ray package to use the Ray tokenizer group pool.")
25
+ return RayTokenizerGroupPool.from_config(tokenizer_pool_config,
26
+ **init_kwargs)
27
+ else:
28
+ raise ValueError(
29
+ f"Unknown pool type: {tokenizer_pool_config.pool_type}")
30
+
31
+
32
+__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
new
+48
@@ -0,0 +1,48 @@
1
+from abc import ABC, abstractmethod
2
+from typing import List, Optional
3
+
4
+from transformers import PreTrainedTokenizer
5
+
6
+from vllm.lora.request import LoRARequest
7
+
8
+
9
+class BaseTokenizerGroup(ABC):
10
+ """A group of tokenizers that can be used for LoRA adapters."""
11
+
12
+ @abstractmethod
13
+ def ping(self) -> bool:
14
+ """Check if the tokenizer group is alive."""
15
+ pass
16
+
17
+ @abstractmethod
18
+ def get_max_input_len(self,
19
+ lora_request: Optional[LoRARequest] = None
20
+ ) -> Optional[int]:
21
+ """Get the maximum input length for the LoRA request."""
22
+ pass
23
+
24
+ @abstractmethod
25
+ def encode(self, prompt: str, request_id: Optional[str],
26
+ lora_request: Optional[LoRARequest]) -> List[int]:
27
+ """Encode a prompt using the tokenizer group."""
28
+ pass
29
+
30
+ @abstractmethod
31
+ async def encode_async(self, prompt: str, request_id: Optional[str],
32
+ lora_request: Optional[LoRARequest]) -> List[int]:
33
+ """Encode a prompt using the tokenizer group."""
34
+ pass
35
+
36
+ @abstractmethod
37
+ def get_lora_tokenizer(
38
+ self,
39
+ lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
40
+ """Get a tokenizer for a LoRA request."""
41
+ pass
42
+
43
+ @abstractmethod
44
+ async def get_lora_tokenizer_async(
45
+ self,
46
+ lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
47
+ """Get a tokenizer for a LoRA request."""
48
+ pass
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
new
+166
@@ -0,0 +1,166 @@
1
+import asyncio
2
+import os
3
+from typing import List, Optional
4
+
5
+from transformers import PreTrainedTokenizer
6
+
7
+from vllm.config import TokenizerPoolConfig
8
+from vllm.lora.request import LoRARequest
9
+from vllm.engine.ray_utils import ray
10
+from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
11
+ BaseTokenizerGroup)
12
+from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
13
+ TokenizerGroup)
14
+from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
15
+
16
+
17
+class RayTokenizerGroupPool(BaseTokenizerGroup):
18
+ """A Ray-based pool of TokenizerGroups for async tokenization."""
19
+
20
+ # Class to use for workers making up the pool.
21
+ _worker_cls = TokenizerGroup
22
+
23
+ @classmethod
24
+ def from_config(cls, tokenizer_pool_config: TokenizerPoolConfig,
25
+ **init_kwargs) -> "RayTokenizerGroupPool":
26
+ ray_actor_options = (tokenizer_pool_config.extra_config or {
27
+ "num_cpus": 0
28
+ })
29
+ ray_actor_options.setdefault(
30
+ "scheduling_strategy",
31
+ NodeAffinitySchedulingStrategy(
32
+ node_id=ray.get_runtime_context().get_node_id(), soft=True))
33
+
34
+ # Carry over the env vars to the actors.
35
+ # This is necessary for API keys and such.
36
+ ray_actor_options.setdefault("runtime_env", {})
37
+ _carry_over_env_vars_to_runtime_env(ray_actor_options["runtime_env"])
38
+
39
+ init_kwargs["num_actors"] = tokenizer_pool_config.pool_size
40
+ init_kwargs["ray_actor_options"] = ray_actor_options
41
+
42
+ return cls(**init_kwargs)
43
+
44
+ def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
45
+ max_input_length: Optional[int], num_actors: int,
46
+ ray_actor_options: dict, **tokenizer_config):
47
+ # Store a local copy of the TokenizerGroup for quick access
48
+ # to underlying HF tokenizers.
49
+ self._local_tokenizer_group = self._worker_cls(
50
+ tokenizer_id=tokenizer_id,
51
+ enable_lora=enable_lora,
52
+ max_num_seqs=max_num_seqs,
53
+ max_input_length=max_input_length,
54
+ )
55
+
56
+ ray_tokenizer_group_cls = ray.remote(
57
+ self._worker_cls).options(**ray_actor_options)
58
+ self.tokenizer_actors = [
59
+ ray_tokenizer_group_cls.remote(tokenizer_id, enable_lora,
60
+ max_num_seqs, max_input_length,
61
+ **tokenizer_config)
62
+ for _ in range(num_actors)
63
+ ]
64
+ self._idle_actors: Optional[asyncio.Queue] = None
65
+
66
+ @property
67
+ def pool_size(self) -> int:
68
+ return len(self.tokenizer_actors)
69
+
70
+ def ping(self):
71
+ return ray.get(
72
+ [actor.ping.remote() for actor in self.tokenizer_actors])
73
+
74
+ def _ensure_queue_initialized(self):
75
+ if self._idle_actors is None:
76
+ self._idle_actors = asyncio.Queue()
77
+ for actor in self.tokenizer_actors:
78
+ self._idle_actors.put_nowait(actor)
79
+
80
+ def encode(self,
81
+ prompt: str,
82
+ request_id: Optional[str] = None,
83
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
84
+ """Encode a prompt using the tokenizer group.
85
+
86
+ We pick an idle actor and use it to encode the prompt.
87
+ The actor is then put back in the queue for future use.
88
+ This is blocking.
89
+ """
90
+ self._ensure_queue_initialized()
91
+
92
+ if self._idle_actors.empty():
93
+ raise RuntimeError("No idle actors available.")
94
+ actor = self._idle_actors.get_nowait()
95
+ try:
96
+ ret = ray.get(
97
+ actor.encode.remote(request_id=request_id,
98
+ prompt=prompt,
99
+ lora_request=lora_request))
100
+ finally:
101
+ # Put the actor back in the queue.
102
+ # This is done in a finally block to ensure that the actor is
103
+ # always put back in the queue, even if an exception/cancellation
104
+ # is raised.
105
+ self._idle_actors.put_nowait(actor)
106
+ return ret
107
+
108
+ async def encode_async(
109
+ self,
110
+ prompt: str,
111
+ request_id: Optional[str] = None,
112
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
113
+ """Encode a prompt using the tokenizer group.
114
+
115
+ We pick an idle actor and use it to encode the prompt.
116
+ If there are no idle actors, we wait until one becomes
117
+ available.
118
+ The actor is then put back in the queue for future use.
119
+ This is non-blocking.
120
+ """
121
+ self._ensure_queue_initialized()
122
+
123
+ actor = await self._idle_actors.get()
124
+ try:
125
+ ret = await actor.encode.remote(request_id=request_id,
126
+ prompt=prompt,
127
+ lora_request=lora_request)
128
+ finally:
129
+ # Put the actor back in the queue.
130
+ # This is done in a finally block to ensure that the actor is
131
+ # always put back in the queue, even if an exception/cancellation
132
+ # is raised.
133
+ self._idle_actors.put_nowait(actor)
134
+ return ret
135
+
136
+ def get_max_input_len(self,
137
+ lora_request: Optional[LoRARequest] = None
138
+ ) -> Optional[int]:
139
+ """Get the maximum input length for the LoRA request."""
140
+ return self._local_tokenizer_group.get_max_input_len(lora_request)
141
+
142
+ def get_lora_tokenizer(
143
+ self,
144
+ lora_request: Optional[LoRARequest] = None
145
+ ) -> "PreTrainedTokenizer":
146
+ return self._local_tokenizer_group.get_lora_tokenizer(lora_request)
147
+
148
+ async def get_lora_tokenizer_async(
149
+ self,
150
+ lora_request: Optional[LoRARequest] = None
151
+ ) -> "PreTrainedTokenizer":
152
+ return await self._local_tokenizer_group.get_lora_tokenizer_async(
153
+ lora_request)
154
+
155
+
156
+def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None:
157
+ """Copy over all current process environment variables to the runtime_env.
158
+
159
+ The variables in runtime_env will take precedence over the current process
160
+ environment variables.
161
+
162
+ runtime_env will be modified in place."""
163
+ env_vars = os.environ.copy()
164
+ runtime_env.setdefault("env_vars", {})
165
+ env_vars.update(runtime_env["env_vars"])
166
+ runtime_env["env_vars"] = env_vars
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
new
+80
@@ -0,0 +1,80 @@
1
+from typing import List, Optional
2
+
3
+from transformers import PreTrainedTokenizer
4
+
5
+from vllm.lora.request import LoRARequest
6
+from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
7
+ get_lora_tokenizer_async)
8
+from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
9
+ BaseTokenizerGroup)
10
+from vllm.utils import LRUCache
11
+from vllm.transformers_utils.tokenizer import get_tokenizer
12
+
13
+
14
+class TokenizerGroup(BaseTokenizerGroup):
15
+ """A group of tokenizers that can be used for LoRA adapters."""
16
+
17
+ def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
18
+ max_input_length: Optional[int], **tokenizer_config):
19
+ self.tokenizer_id = tokenizer_id
20
+ self.tokenizer_config = tokenizer_config
21
+ self.enable_lora = enable_lora
22
+ self.max_input_length = max_input_length
23
+ self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24
+ if enable_lora:
25
+ self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
26
+ else:
27
+ self.lora_tokenizers = None
28
+
29
+ def ping(self) -> bool:
30
+ """Check if the tokenizer group is alive."""
31
+ return True
32
+
33
+ def get_max_input_len(self,
34
+ lora_request: Optional[LoRARequest] = None
35
+ ) -> Optional[int]:
36
+ """Get the maximum input length for the LoRA request."""
37
+ return self.max_input_length
38
+
39
+ def encode(self,
40
+ prompt: str,
41
+ request_id: Optional[str] = None,
42
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
43
+ tokenizer = self.get_lora_tokenizer(lora_request)
44
+ return tokenizer.encode(prompt)
45
+
46
+ async def encode_async(
47
+ self,
48
+ prompt: str,
49
+ request_id: Optional[str] = None,
50
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
51
+ tokenizer = await self.get_lora_tokenizer_async(lora_request)
52
+ return tokenizer.encode(prompt)
53
+
54
+ def get_lora_tokenizer(
55
+ self,
56
+ lora_request: Optional[LoRARequest] = None
57
+ ) -> "PreTrainedTokenizer":
58
+ if not lora_request or not self.enable_lora:
59
+ return self.tokenizer
60
+ if lora_request.lora_int_id not in self.lora_tokenizers:
61
+ tokenizer = (get_lora_tokenizer(
62
+ lora_request, **self.tokenizer_config) or self.tokenizer)
63
+ self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
64
+ return tokenizer
65
+ else:
66
+ return self.lora_tokenizers.get(lora_request.lora_int_id)
67
+
68
+ async def get_lora_tokenizer_async(
69
+ self,
70
+ lora_request: Optional[LoRARequest] = None
71
+ ) -> "PreTrainedTokenizer":
72
+ if not lora_request or not self.enable_lora:
73
+ return self.tokenizer
74
+ if lora_request.lora_int_id not in self.lora_tokenizers:
75
+ tokenizer = (await get_lora_tokenizer_async(
76
+ lora_request, **self.tokenizer_config) or self.tokenizer)
77
+ self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
78
+ return tokenizer
79
+ else:
80
+ return self.lora_tokenizers.get(lora_request.lora_int_id)