A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Tokenizer] Add an option to specify tokenizer (#284)
Woosuk Kwon committed
Jun 28, 2023 at 09:46 UTC
4338cc475029dcd37a291a867d52419122648e72
10 files changed
+61
-60
benchmarks/benchmark_latency.py
+2
@@ -17,6 +17,7 @@ def main(args: argparse.Namespace):
17
# the engine will automatically process the request in multiple batches.
18
llm = LLM(
19
model=args.model,
20
+ tokenizer=args.tokenizer,
21
tensor_parallel_size=args.tensor_parallel_size,
22
max_num_seqs=args.batch_size,
23
max_num_batched_tokens=args.batch_size * args.input_len,
@@ -63,6 +64,7 @@ if __name__ == '__main__':
64
description='Benchmark the latency of processing a single batch of '
65
'requests till completion.')
66
parser.add_argument('--model', type=str, default='facebook/opt-125m')
67
+ parser.add_argument('--tokenizer', type=str, default=None)
68
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
69
parser.add_argument('--input-len', type=int, default=32)
70
parser.add_argument('--output-len', type=int, default=128)
benchmarks/benchmark_serving.py
+2
-9
@@ -24,20 +24,13 @@ from typing import AsyncGenerator, List, Tuple
24
25
import aiohttp
26
import numpy as np
27
-from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
27
+from transformers import PreTrainedTokenizerBase
28
+from vllm.transformers_utils.tokenizer import get_tokenizer
29
30
# (prompt len, output len, latency)
31
REQUEST_LATENCY: List[Tuple[int, int, float]] = []
32
33
33
-def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
34
- config = AutoConfig.from_pretrained(model_name)
35
- if config.model_type == "llama":
36
- # A workaround for potential protobuf errors.
37
- model_name = "hf-internal-testing/llama-tokenizer"
38
- return AutoTokenizer.from_pretrained(model_name)
39
-
40
-
34
def sample_requests(
35
dataset_path: str,
36
num_requests: int,
benchmarks/benchmark_throughput.py
+15
-20
@@ -6,23 +6,11 @@ import time
6
from typing import List, Tuple
7
8
import torch
9
-from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
10
- PreTrainedTokenizerBase)
9
+from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
10
from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
14
-
15
-
16
-def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
17
- config = AutoConfig.from_pretrained(model_name)
18
- if config.model_type == "llama":
19
- # A workaround for potential protobuf errors.
20
- model_name = "hf-internal-testing/llama-tokenizer"
21
- tokenizer = AutoTokenizer.from_pretrained(model_name)
22
- # To enable padding in the HF backend.
23
- tokenizer.pad_token = tokenizer.eos_token
24
- return tokenizer
25
- return AutoTokenizer.from_pretrained(model_name)
13
+from vllm.transformers_utils.tokenizer import get_tokenizer
14
15
16
def sample_requests(
@@ -74,6 +62,7 @@ def sample_requests(
62
def run_vllm(
63
requests: List[Tuple[str, int, int]],
64
model: str,
65
+ tokenizer: str,
66
tensor_parallel_size: int,
67
seed: int,
68
n: int,
@@ -81,6 +70,7 @@ def run_vllm(
70
) -> float:
71
llm = LLM(
72
model=model,
73
+ tokenizer=tokenizer,
74
tensor_parallel_size=tensor_parallel_size,
75
seed=seed,
76
)
@@ -118,9 +108,10 @@ def run_hf(
108
max_batch_size: int,
109
) -> float:
110
assert not use_beam_search
121
- tokenizer = get_tokenizer(model)
122
- llm = AutoModelForCausalLM.from_pretrained(
123
- model, torch_dtype=torch.float16)
111
+ llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
112
+ if llm.config.model_type == "llama":
113
+ # To enable padding in the HF backend.
114
+ tokenizer.pad_token = tokenizer.eos_token
115
llm = llm.cuda()
116
117
pbar = tqdm(total=len(requests))
@@ -170,13 +161,13 @@ def main(args: argparse.Namespace):
161
random.seed(args.seed)
162
163
# Sample the requests.
173
- tokenizer = get_tokenizer(args.model)
164
+ tokenizer = get_tokenizer(args.tokenizer)
165
requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
166
167
if args.backend == "vllm":
168
elapsed_time = run_vllm(
178
- requests, args.model, args.tensor_parallel_size, args.seed, args.n,
179
- args.use_beam_search)
169
+ requests, args.model, args.tokenizer, args.tensor_parallel_size,
170
+ args.seed, args.n, args.use_beam_search)
171
elif args.backend == "hf":
172
assert args.tensor_parallel_size == 1
173
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -198,6 +189,7 @@ if __name__ == "__main__":
189
parser.add_argument("--dataset", type=str, required=True,
190
help="Path to the dataset.")
191
parser.add_argument("--model", type=str, default="facebook/opt-125m")
192
+ parser.add_argument("--tokenizer", type=str, default=None)
193
parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
194
parser.add_argument("--n", type=int, default=1,
195
help="Number of generated sequences per prompt.")
@@ -208,11 +200,14 @@ if __name__ == "__main__":
200
parser.add_argument("--hf-max-batch-size", type=int, default=None,
201
help="Maximum batch size for HF backend.")
202
args = parser.parse_args()
203
+
204
if args.backend == "vllm":
205
if args.hf_max_batch_size is not None:
206
raise ValueError("HF max batch size is only for HF backend.")
207
elif args.backend == "hf":
208
if args.hf_max_batch_size is None:
209
raise ValueError("HF max batch size is required for HF backend.")
210
+ if args.tokenizer is None:
211
+ args.tokenizer = args.model
212
213
main(args)
vllm/config.py
+3
@@ -16,6 +16,7 @@ class ModelConfig:
16
17
Args:
18
model: Name or path of the huggingface model to use.
19
+ tokenizer: Name or path of the huggingface tokenizer to use.
20
download_dir: Directory to download and load the weights, default to the
21
default cache directory of huggingface.
22
use_np_weights: Save a numpy copy of model weights for faster loading.
@@ -30,6 +31,7 @@ class ModelConfig:
31
def __init__(
32
self,
33
model: str,
34
+ tokenizer: Optional[str],
35
download_dir: Optional[str],
36
use_np_weights: bool,
37
use_dummy_weights: bool,
@@ -37,6 +39,7 @@ class ModelConfig:
39
seed: int,
40
) -> None:
41
self.model = model
42
+ self.tokenizer = tokenizer
43
self.download_dir = download_dir
44
self.use_np_weights = use_np_weights
45
self.use_dummy_weights = use_dummy_weights
vllm/engine/arg_utils.py
+6
-1
@@ -11,6 +11,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
11
class EngineArgs:
12
"""Arguments for vLLM engine."""
13
model: str
14
+ tokenizer: Optional[str] = None
15
download_dir: Optional[str] = None
16
use_np_weights: bool = False
17
use_dummy_weights: bool = False
@@ -27,6 +28,8 @@ class EngineArgs:
28
disable_log_stats: bool = False
29
30
def __post_init__(self):
31
+ if self.tokenizer is None:
32
+ self.tokenizer = self.model
33
self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
34
35
@staticmethod
@@ -37,6 +40,8 @@ class EngineArgs:
40
# Model arguments
41
parser.add_argument('--model', type=str, default='facebook/opt-125m',
42
help='name or path of the huggingface model to use')
43
+ parser.add_argument('--tokenizer', type=str, default=EngineArgs.tokenizer,
44
+ help='name or path of the huggingface tokenizer to use')
45
parser.add_argument('--download-dir', type=str,
46
default=EngineArgs.download_dir,
47
help='directory to download and load the weights, '
@@ -104,7 +109,7 @@ class EngineArgs:
109
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
110
# Initialize the configs.
111
model_config = ModelConfig(
107
- self.model, self.download_dir, self.use_np_weights,
112
+ self.model, self.tokenizer, self.download_dir, self.use_np_weights,
113
self.use_dummy_weights, self.dtype, self.seed)
114
cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
115
self.swap_space)
vllm/engine/llm_engine.py
+4
-2
@@ -6,11 +6,12 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
6
from vllm.core.scheduler import Scheduler
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.ray_utils import DeviceID, initialize_cluster, ray
9
-from vllm.engine.tokenizer_utils import detokenize_incrementally, get_tokenizer
9
from vllm.logger import init_logger
10
from vllm.outputs import RequestOutput
11
from vllm.sampling_params import SamplingParams
12
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
13
+from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
14
+ get_tokenizer)
15
from vllm.utils import Counter
16
from vllm.worker.worker import Worker
17
@@ -59,6 +60,7 @@ class LLMEngine:
60
logger.info(
61
"Initializing an LLM engine with config: "
62
f"model={model_config.model!r}, "
63
+ f"tokenizer={model_config.tokenizer!r}, "
64
f"dtype={model_config.dtype}, "
65
f"use_dummy_weights={model_config.use_dummy_weights}, "
66
f"download_dir={model_config.download_dir!r}, "
@@ -75,7 +77,7 @@ class LLMEngine:
77
self.log_stats = log_stats
78
self._verify_args()
79
78
- self.tokenizer = get_tokenizer(model_config.model)
80
+ self.tokenizer = get_tokenizer(model_config.tokenizer)
81
self.seq_counter = Counter()
82
83
# Create the parallel GPU workers.
vllm/entrypoints/llm.py
+3
@@ -25,6 +25,7 @@ class LLM:
25
26
Args:
27
model: The name or path of a HuggingFace Transformers model.
28
+ tokenizer: The name or path of a HuggingFace Transformers tokenizer.
29
tensor_parallel_size: The number of GPUs to use for distributed
30
execution with tensor parallelism.
31
dtype: The data type for the model weights and activations. Currently,
@@ -38,6 +39,7 @@ class LLM:
39
def __init__(
40
self,
41
model: str,
42
+ tokenizer: Optional[str] = None,
43
tensor_parallel_size: int = 1,
44
dtype: str = "auto",
45
seed: int = 0,
@@ -47,6 +49,7 @@ class LLM:
49
kwargs["disable_log_stats"] = True
50
engine_args = EngineArgs(
51
model=model,
52
+ tokenizer=tokenizer,
53
tensor_parallel_size=tensor_parallel_size,
54
dtype=dtype,
55
seed=seed,
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -15,7 +15,6 @@ import uvicorn
15
16
from vllm.engine.arg_utils import AsyncEngineArgs
17
from vllm.engine.async_llm_engine import AsyncLLMEngine
18
-from vllm.engine.tokenizer_utils import get_tokenizer
18
from vllm.entrypoints.openai.protocol import (
19
CompletionRequest, CompletionResponse, CompletionResponseChoice,
20
CompletionResponseStreamChoice, CompletionStreamResponse, ErrorResponse,
@@ -23,6 +22,7 @@ from vllm.entrypoints.openai.protocol import (
22
from vllm.logger import init_logger
23
from vllm.outputs import RequestOutput
24
from vllm.sampling_params import SamplingParams
25
+from vllm.transformers_utils.tokenizer import get_tokenizer
26
from vllm.utils import random_uuid
27
28
TIMEOUT_KEEP_ALIVE = 5 # seconds
vllm/transformers_utils/__init__.py
vllm/transformers_utils/tokenizer.py
renamed
+25
-27
@@ -1,46 +1,44 @@
1
from typing import List, Tuple, Union
2
3
-from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
3
+from transformers import (AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
5
6
from vllm.logger import init_logger
7
8
logger = init_logger(__name__)
9
10
-_MODEL_TYPES_WITH_SLOW_TOKENIZER = []
10
+# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file.
11
+_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
12
13
14
def get_tokenizer(
14
- model_name: str,
15
+ tokenizer_name: str,
16
*args,
17
**kwargs,
18
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
19
"""Gets a tokenizer for the given model name via Huggingface."""
19
- config = AutoConfig.from_pretrained(model_name)
20
- if "open_llama" in model_name:
21
- kwargs["use_fast"] = False
20
+ if "llama" in tokenizer_name.lower() and kwargs.get("use_fast", True):
21
logger.info(
23
- "OpenLLaMA models do not support the fast tokenizer. "
24
- "Using the slow tokenizer instead.")
25
- elif config.model_type == "llama" and kwargs.get("use_fast", True):
26
- # LLaMA fast tokenizer causes protobuf errors in some environments.
27
- # However, we found that the below LLaMA fast tokenizer works well in
28
- # most environments.
29
- model_name = "hf-internal-testing/llama-tokenizer"
30
- logger.info(
31
- f"Using the LLaMA fast tokenizer in '{model_name}' to avoid "
32
- "potential protobuf errors.")
33
- elif config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
34
- if kwargs.get("use_fast", False) == True:
35
- raise ValueError(
36
- f"Cannot use the fast tokenizer for {config.model_type} due to "
37
- "bugs in the fast tokenizer.")
38
- logger.info(
39
- f"Using the slow tokenizer for {config.model_type} due to bugs in "
40
- "the fast tokenizer. This could potentially lead to performance "
41
- "degradation.")
42
- kwargs["use_fast"] = False
43
- return AutoTokenizer.from_pretrained(model_name, *args, **kwargs)
22
+ "For some LLaMA-based models, initializing the fast tokenizer may "
23
+ "take a long time. To eliminate the initialization time, consider "
24
+ f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
25
+ "tokenizer.")
26
+ try:
27
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, *args,
28
+ **kwargs)
29
+ except TypeError as e:
30
+ # The LLaMA tokenizer causes a protobuf error in some environments.
31
+ err_msg = (
32
+ "Failed to load the tokenizer. If you are using a LLaMA-based "
33
+ f"model, use '{_FAST_LLAMA_TOKENIZER}' instead of the original "
34
+ "tokenizer.")
35
+ raise RuntimeError(err_msg) from e
36
+
37
+ if not isinstance(tokenizer, PreTrainedTokenizerFast):
38
+ logger.warning(
39
+ "Using a slow tokenizer. This might cause a significant "
40
+ "slowdown. Consider using a fast tokenizer instead.")
41
+ return tokenizer
42
43
44
def detokenize_incrementally(