A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Tokenizer] Add tokenizer mode (#298)
Woosuk Kwon committed
Jun 28, 2023 at 14:19 UTC
998d9d15095e7a69629f9e131c8b59bfdd1c6314
6 files changed
+38
-5
vllm/config.py
+14
-1
@@ -17,6 +17,8 @@ class ModelConfig:
17
Args:
18
model: Name or path of the huggingface model to use.
19
tokenizer: Name or path of the huggingface tokenizer to use.
20
+ tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
21
+ available, and "slow" will always use the slow tokenizer.
22
download_dir: Directory to download and load the weights, default to the
23
default cache directory of huggingface.
24
use_np_weights: Save a numpy copy of model weights for faster loading.
@@ -31,7 +33,8 @@ class ModelConfig:
33
def __init__(
34
self,
35
model: str,
34
- tokenizer: Optional[str],
36
+ tokenizer: str,
37
+ tokenizer_mode: str,
38
download_dir: Optional[str],
39
use_np_weights: bool,
40
use_dummy_weights: bool,
@@ -40,6 +43,7 @@ class ModelConfig:
43
) -> None:
44
self.model = model
45
self.tokenizer = tokenizer
46
+ self.tokenizer_mode = tokenizer_mode
47
self.download_dir = download_dir
48
self.use_np_weights = use_np_weights
49
self.use_dummy_weights = use_dummy_weights
@@ -47,6 +51,15 @@ class ModelConfig:
51
52
self.hf_config: PretrainedConfig = AutoConfig.from_pretrained(model)
53
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
54
+ self._verify_tokenizer_mode()
55
+
56
+ def _verify_tokenizer_mode(self) -> None:
57
+ tokenizer_mode = self.tokenizer_mode.lower()
58
+ if tokenizer_mode not in ["auto", "slow"]:
59
+ raise ValueError(
60
+ f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
61
+ "either 'auto' or 'slow'.")
62
+ self.tokenizer_mode = tokenizer_mode
63
64
def verify_with_parallel_config(
65
self,
vllm/engine/arg_utils.py
+9
-2
@@ -12,6 +12,7 @@ class EngineArgs:
12
"""Arguments for vLLM engine."""
13
model: str
14
tokenizer: Optional[str] = None
15
+ tokenizer_mode: str = "auto"
16
download_dir: Optional[str] = None
17
use_np_weights: bool = False
18
use_dummy_weights: bool = False
@@ -42,6 +43,12 @@ class EngineArgs:
43
help='name or path of the huggingface model to use')
44
parser.add_argument('--tokenizer', type=str, default=EngineArgs.tokenizer,
45
help='name or path of the huggingface tokenizer to use')
46
+ parser.add_argument('--tokenizer-mode', type=str,
47
+ default=EngineArgs.tokenizer_mode,
48
+ choices=['auto', 'slow'],
49
+ help='tokenizer mode. "auto" will use the fast '
50
+ 'tokenizer if available, and "slow" will '
51
+ 'always use the slow tokenizer.')
52
parser.add_argument('--download-dir', type=str,
53
default=EngineArgs.download_dir,
54
help='directory to download and load the weights, '
@@ -109,8 +116,8 @@ class EngineArgs:
116
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
117
# Initialize the configs.
118
model_config = ModelConfig(
112
- self.model, self.tokenizer, self.download_dir, self.use_np_weights,
113
- self.use_dummy_weights, self.dtype, self.seed)
119
+ self.model, self.tokenizer, self.tokenizer_mode, self.download_dir,
120
+ self.use_np_weights, self.use_dummy_weights, self.dtype, self.seed)
121
cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
122
self.swap_space)
123
parallel_config = ParallelConfig(self.pipeline_parallel_size,
vllm/engine/llm_engine.py
+3
-1
@@ -61,6 +61,7 @@ class LLMEngine:
61
"Initializing an LLM engine with config: "
62
f"model={model_config.model!r}, "
63
f"tokenizer={model_config.tokenizer!r}, "
64
+ f"tokenizer_mode={model_config.tokenizer_mode}, "
65
f"dtype={model_config.dtype}, "
66
f"use_dummy_weights={model_config.use_dummy_weights}, "
67
f"download_dir={model_config.download_dir!r}, "
@@ -77,7 +78,8 @@ class LLMEngine:
78
self.log_stats = log_stats
79
self._verify_args()
80
80
- self.tokenizer = get_tokenizer(model_config.tokenizer)
81
+ self.tokenizer = get_tokenizer(model_config.tokenizer,
82
+ model_config.tokenizer_mode)
83
self.seq_counter = Counter()
84
85
# Create the parallel GPU workers.
vllm/entrypoints/llm.py
+4
@@ -26,6 +26,8 @@ class LLM:
26
Args:
27
model: The name or path of a HuggingFace Transformers model.
28
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
29
+ tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
30
+ if available, and "slow" will always use the slow tokenizer.
31
tensor_parallel_size: The number of GPUs to use for distributed
32
execution with tensor parallelism.
33
dtype: The data type for the model weights and activations. Currently,
@@ -40,6 +42,7 @@ class LLM:
42
self,
43
model: str,
44
tokenizer: Optional[str] = None,
45
+ tokenizer_mode: str = "auto",
46
tensor_parallel_size: int = 1,
47
dtype: str = "auto",
48
seed: int = 0,
@@ -50,6 +53,7 @@ class LLM:
53
engine_args = EngineArgs(
54
model=model,
55
tokenizer=tokenizer,
56
+ tokenizer_mode=tokenizer_mode,
57
tensor_parallel_size=tensor_parallel_size,
58
dtype=dtype,
59
seed=seed,
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -313,7 +313,7 @@ if __name__ == "__main__":
313
engine = AsyncLLMEngine.from_engine_args(engine_args)
314
315
# A separate tokenizer to map token IDs to strings.
316
- tokenizer = get_tokenizer(args.model)
316
+ tokenizer = get_tokenizer(engine_args.tokenizer, engine_args.tokenizer_mode)
317
318
uvicorn.run(app, host=args.host, port=args.port, log_level="info",
319
timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
vllm/transformers_utils/tokenizer.py
+7
@@ -13,10 +13,17 @@ _FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
13
14
def get_tokenizer(
15
tokenizer_name: str,
16
+ tokenizer_mode: str = "auto",
17
*args,
18
**kwargs,
19
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
20
"""Gets a tokenizer for the given model name via Huggingface."""
21
+ if tokenizer_mode == "slow":
22
+ if kwargs.get("use_fast", False):
23
+ raise ValueError(
24
+ "Cannot use the fast tokenizer in slow tokenizer mode.")
25
+ kwargs["use_fast"] = False
26
+
27
if "llama" in tokenizer_name.lower() and kwargs.get("use_fast", True):
28
logger.info(
29
"For some LLaMA-based models, initializing the fast tokenizer may "