@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4338cc4750
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Tokenizer] Add an option to specify tokenizer (#284)

Woosuk Kwon committed Jun 28, 2023 at 09:46 UTC 4338cc475029dcd37a291a867d52419122648e72
10 files changed +61 -60
benchmarks/benchmark_latency.py
+2
@@ -17,6 +17,7 @@ def main(args: argparse.Namespace):
17 # the engine will automatically process the request in multiple batches.
18 llm = LLM(
19 model=args.model,
20 + tokenizer=args.tokenizer,
21 tensor_parallel_size=args.tensor_parallel_size,
22 max_num_seqs=args.batch_size,
23 max_num_batched_tokens=args.batch_size * args.input_len,
@@ -63,6 +64,7 @@ if __name__ == '__main__':
64 description='Benchmark the latency of processing a single batch of '
65 'requests till completion.')
66 parser.add_argument('--model', type=str, default='facebook/opt-125m')
67 + parser.add_argument('--tokenizer', type=str, default=None)
68 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
69 parser.add_argument('--input-len', type=int, default=32)
70 parser.add_argument('--output-len', type=int, default=128)
benchmarks/benchmark_serving.py
+2 -9
@@ -24,20 +24,13 @@ from typing import AsyncGenerator, List, Tuple
24
25 import aiohttp
26 import numpy as np
27 -from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
27 +from transformers import PreTrainedTokenizerBase
28 +from vllm.transformers_utils.tokenizer import get_tokenizer
29
30 # (prompt len, output len, latency)
31 REQUEST_LATENCY: List[Tuple[int, int, float]] = []
32
33
33 -def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
34 - config = AutoConfig.from_pretrained(model_name)
35 - if config.model_type == "llama":
36 - # A workaround for potential protobuf errors.
37 - model_name = "hf-internal-testing/llama-tokenizer"
38 - return AutoTokenizer.from_pretrained(model_name)
39 -
40 -
34 def sample_requests(
35 dataset_path: str,
36 num_requests: int,
benchmarks/benchmark_throughput.py
+15 -20
@@ -6,23 +6,11 @@ import time
6 from typing import List, Tuple
7
8 import torch
9 -from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
10 - PreTrainedTokenizerBase)
9 +from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
10 from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
14 -
15 -
16 -def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
17 - config = AutoConfig.from_pretrained(model_name)
18 - if config.model_type == "llama":
19 - # A workaround for potential protobuf errors.
20 - model_name = "hf-internal-testing/llama-tokenizer"
21 - tokenizer = AutoTokenizer.from_pretrained(model_name)
22 - # To enable padding in the HF backend.
23 - tokenizer.pad_token = tokenizer.eos_token
24 - return tokenizer
25 - return AutoTokenizer.from_pretrained(model_name)
13 +from vllm.transformers_utils.tokenizer import get_tokenizer
14
15
16 def sample_requests(
@@ -74,6 +62,7 @@ def sample_requests(
62 def run_vllm(
63 requests: List[Tuple[str, int, int]],
64 model: str,
65 + tokenizer: str,
66 tensor_parallel_size: int,
67 seed: int,
68 n: int,
@@ -81,6 +70,7 @@ def run_vllm(
70 ) -> float:
71 llm = LLM(
72 model=model,
73 + tokenizer=tokenizer,
74 tensor_parallel_size=tensor_parallel_size,
75 seed=seed,
76 )
@@ -118,9 +108,10 @@ def run_hf(
108 max_batch_size: int,
109 ) -> float:
110 assert not use_beam_search
121 - tokenizer = get_tokenizer(model)
122 - llm = AutoModelForCausalLM.from_pretrained(
123 - model, torch_dtype=torch.float16)
111 + llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
112 + if llm.config.model_type == "llama":
113 + # To enable padding in the HF backend.
114 + tokenizer.pad_token = tokenizer.eos_token
115 llm = llm.cuda()
116
117 pbar = tqdm(total=len(requests))
@@ -170,13 +161,13 @@ def main(args: argparse.Namespace):
161 random.seed(args.seed)
162
163 # Sample the requests.
173 - tokenizer = get_tokenizer(args.model)
164 + tokenizer = get_tokenizer(args.tokenizer)
165 requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
166
167 if args.backend == "vllm":
168 elapsed_time = run_vllm(
178 - requests, args.model, args.tensor_parallel_size, args.seed, args.n,
179 - args.use_beam_search)
169 + requests, args.model, args.tokenizer, args.tensor_parallel_size,
170 + args.seed, args.n, args.use_beam_search)
171 elif args.backend == "hf":
172 assert args.tensor_parallel_size == 1
173 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -198,6 +189,7 @@ if __name__ == "__main__":
189 parser.add_argument("--dataset", type=str, required=True,
190 help="Path to the dataset.")
191 parser.add_argument("--model", type=str, default="facebook/opt-125m")
192 + parser.add_argument("--tokenizer", type=str, default=None)
193 parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
194 parser.add_argument("--n", type=int, default=1,
195 help="Number of generated sequences per prompt.")
@@ -208,11 +200,14 @@ if __name__ == "__main__":
200 parser.add_argument("--hf-max-batch-size", type=int, default=None,
201 help="Maximum batch size for HF backend.")
202 args = parser.parse_args()
203 +
204 if args.backend == "vllm":
205 if args.hf_max_batch_size is not None:
206 raise ValueError("HF max batch size is only for HF backend.")
207 elif args.backend == "hf":
208 if args.hf_max_batch_size is None:
209 raise ValueError("HF max batch size is required for HF backend.")
210 + if args.tokenizer is None:
211 + args.tokenizer = args.model
212
213 main(args)
vllm/config.py
+3
@@ -16,6 +16,7 @@ class ModelConfig:
16
17 Args:
18 model: Name or path of the huggingface model to use.
19 + tokenizer: Name or path of the huggingface tokenizer to use.
20 download_dir: Directory to download and load the weights, default to the
21 default cache directory of huggingface.
22 use_np_weights: Save a numpy copy of model weights for faster loading.
@@ -30,6 +31,7 @@ class ModelConfig:
31 def __init__(
32 self,
33 model: str,
34 + tokenizer: Optional[str],
35 download_dir: Optional[str],
36 use_np_weights: bool,
37 use_dummy_weights: bool,
@@ -37,6 +39,7 @@ class ModelConfig:
39 seed: int,
40 ) -> None:
41 self.model = model
42 + self.tokenizer = tokenizer
43 self.download_dir = download_dir
44 self.use_np_weights = use_np_weights
45 self.use_dummy_weights = use_dummy_weights
vllm/engine/arg_utils.py
+6 -1
@@ -11,6 +11,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
11 class EngineArgs:
12 """Arguments for vLLM engine."""
13 model: str
14 + tokenizer: Optional[str] = None
15 download_dir: Optional[str] = None
16 use_np_weights: bool = False
17 use_dummy_weights: bool = False
@@ -27,6 +28,8 @@ class EngineArgs:
28 disable_log_stats: bool = False
29
30 def __post_init__(self):
31 + if self.tokenizer is None:
32 + self.tokenizer = self.model
33 self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
34
35 @staticmethod
@@ -37,6 +40,8 @@ class EngineArgs:
40 # Model arguments
41 parser.add_argument('--model', type=str, default='facebook/opt-125m',
42 help='name or path of the huggingface model to use')
43 + parser.add_argument('--tokenizer', type=str, default=EngineArgs.tokenizer,
44 + help='name or path of the huggingface tokenizer to use')
45 parser.add_argument('--download-dir', type=str,
46 default=EngineArgs.download_dir,
47 help='directory to download and load the weights, '
@@ -104,7 +109,7 @@ class EngineArgs:
109 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
110 # Initialize the configs.
111 model_config = ModelConfig(
107 - self.model, self.download_dir, self.use_np_weights,
112 + self.model, self.tokenizer, self.download_dir, self.use_np_weights,
113 self.use_dummy_weights, self.dtype, self.seed)
114 cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
115 self.swap_space)
vllm/engine/llm_engine.py
+4 -2
@@ -6,11 +6,12 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
6 from vllm.core.scheduler import Scheduler
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.ray_utils import DeviceID, initialize_cluster, ray
9 -from vllm.engine.tokenizer_utils import detokenize_incrementally, get_tokenizer
9 from vllm.logger import init_logger
10 from vllm.outputs import RequestOutput
11 from vllm.sampling_params import SamplingParams
12 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
13 +from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
14 + get_tokenizer)
15 from vllm.utils import Counter
16 from vllm.worker.worker import Worker
17
@@ -59,6 +60,7 @@ class LLMEngine:
60 logger.info(
61 "Initializing an LLM engine with config: "
62 f"model={model_config.model!r}, "
63 + f"tokenizer={model_config.tokenizer!r}, "
64 f"dtype={model_config.dtype}, "
65 f"use_dummy_weights={model_config.use_dummy_weights}, "
66 f"download_dir={model_config.download_dir!r}, "
@@ -75,7 +77,7 @@ class LLMEngine:
77 self.log_stats = log_stats
78 self._verify_args()
79
78 - self.tokenizer = get_tokenizer(model_config.model)
80 + self.tokenizer = get_tokenizer(model_config.tokenizer)
81 self.seq_counter = Counter()
82
83 # Create the parallel GPU workers.
vllm/entrypoints/llm.py
+3
@@ -25,6 +25,7 @@ class LLM:
25
26 Args:
27 model: The name or path of a HuggingFace Transformers model.
28 + tokenizer: The name or path of a HuggingFace Transformers tokenizer.
29 tensor_parallel_size: The number of GPUs to use for distributed
30 execution with tensor parallelism.
31 dtype: The data type for the model weights and activations. Currently,
@@ -38,6 +39,7 @@ class LLM:
39 def __init__(
40 self,
41 model: str,
42 + tokenizer: Optional[str] = None,
43 tensor_parallel_size: int = 1,
44 dtype: str = "auto",
45 seed: int = 0,
@@ -47,6 +49,7 @@ class LLM:
49 kwargs["disable_log_stats"] = True
50 engine_args = EngineArgs(
51 model=model,
52 + tokenizer=tokenizer,
53 tensor_parallel_size=tensor_parallel_size,
54 dtype=dtype,
55 seed=seed,
vllm/entrypoints/openai/api_server.py
+1 -1
@@ -15,7 +15,6 @@ import uvicorn
15
16 from vllm.engine.arg_utils import AsyncEngineArgs
17 from vllm.engine.async_llm_engine import AsyncLLMEngine
18 -from vllm.engine.tokenizer_utils import get_tokenizer
18 from vllm.entrypoints.openai.protocol import (
19 CompletionRequest, CompletionResponse, CompletionResponseChoice,
20 CompletionResponseStreamChoice, CompletionStreamResponse, ErrorResponse,
@@ -23,6 +22,7 @@ from vllm.entrypoints.openai.protocol import (
22 from vllm.logger import init_logger
23 from vllm.outputs import RequestOutput
24 from vllm.sampling_params import SamplingParams
25 +from vllm.transformers_utils.tokenizer import get_tokenizer
26 from vllm.utils import random_uuid
27
28 TIMEOUT_KEEP_ALIVE = 5 # seconds
vllm/transformers_utils/__init__.py
vllm/transformers_utils/tokenizer.py renamed
+25 -27
@@ -1,46 +1,44 @@
1 from typing import List, Tuple, Union
2
3 -from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
3 +from transformers import (AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
5
6 from vllm.logger import init_logger
7
8 logger = init_logger(__name__)
9
10 -_MODEL_TYPES_WITH_SLOW_TOKENIZER = []
10 +# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file.
11 +_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
12
13
14 def get_tokenizer(
14 - model_name: str,
15 + tokenizer_name: str,
16 *args,
17 **kwargs,
18 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
19 """Gets a tokenizer for the given model name via Huggingface."""
19 - config = AutoConfig.from_pretrained(model_name)
20 - if "open_llama" in model_name:
21 - kwargs["use_fast"] = False
20 + if "llama" in tokenizer_name.lower() and kwargs.get("use_fast", True):
21 logger.info(
23 - "OpenLLaMA models do not support the fast tokenizer. "
24 - "Using the slow tokenizer instead.")
25 - elif config.model_type == "llama" and kwargs.get("use_fast", True):
26 - # LLaMA fast tokenizer causes protobuf errors in some environments.
27 - # However, we found that the below LLaMA fast tokenizer works well in
28 - # most environments.
29 - model_name = "hf-internal-testing/llama-tokenizer"
30 - logger.info(
31 - f"Using the LLaMA fast tokenizer in '{model_name}' to avoid "
32 - "potential protobuf errors.")
33 - elif config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
34 - if kwargs.get("use_fast", False) == True:
35 - raise ValueError(
36 - f"Cannot use the fast tokenizer for {config.model_type} due to "
37 - "bugs in the fast tokenizer.")
38 - logger.info(
39 - f"Using the slow tokenizer for {config.model_type} due to bugs in "
40 - "the fast tokenizer. This could potentially lead to performance "
41 - "degradation.")
42 - kwargs["use_fast"] = False
43 - return AutoTokenizer.from_pretrained(model_name, *args, **kwargs)
22 + "For some LLaMA-based models, initializing the fast tokenizer may "
23 + "take a long time. To eliminate the initialization time, consider "
24 + f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
25 + "tokenizer.")
26 + try:
27 + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, *args,
28 + **kwargs)
29 + except TypeError as e:
30 + # The LLaMA tokenizer causes a protobuf error in some environments.
31 + err_msg = (
32 + "Failed to load the tokenizer. If you are using a LLaMA-based "
33 + f"model, use '{_FAST_LLAMA_TOKENIZER}' instead of the original "
34 + "tokenizer.")
35 + raise RuntimeError(err_msg) from e
36 +
37 + if not isinstance(tokenizer, PreTrainedTokenizerFast):
38 + logger.warning(
39 + "Using a slow tokenizer. This might cause a significant "
40 + "slowdown. Consider using a fast tokenizer instead.")
41 + return tokenizer
42
43
44 def detokenize_incrementally(