@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 998d9d1509
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Tokenizer] Add tokenizer mode (#298)

Woosuk Kwon committed Jun 28, 2023 at 14:19 UTC 998d9d15095e7a69629f9e131c8b59bfdd1c6314
6 files changed +38 -5
vllm/config.py
+14 -1
@@ -17,6 +17,8 @@ class ModelConfig:
17 Args:
18 model: Name or path of the huggingface model to use.
19 tokenizer: Name or path of the huggingface tokenizer to use.
20 + tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
21 + available, and "slow" will always use the slow tokenizer.
22 download_dir: Directory to download and load the weights, default to the
23 default cache directory of huggingface.
24 use_np_weights: Save a numpy copy of model weights for faster loading.
@@ -31,7 +33,8 @@ class ModelConfig:
33 def __init__(
34 self,
35 model: str,
34 - tokenizer: Optional[str],
36 + tokenizer: str,
37 + tokenizer_mode: str,
38 download_dir: Optional[str],
39 use_np_weights: bool,
40 use_dummy_weights: bool,
@@ -40,6 +43,7 @@ class ModelConfig:
43 ) -> None:
44 self.model = model
45 self.tokenizer = tokenizer
46 + self.tokenizer_mode = tokenizer_mode
47 self.download_dir = download_dir
48 self.use_np_weights = use_np_weights
49 self.use_dummy_weights = use_dummy_weights
@@ -47,6 +51,15 @@ class ModelConfig:
51
52 self.hf_config: PretrainedConfig = AutoConfig.from_pretrained(model)
53 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
54 + self._verify_tokenizer_mode()
55 +
56 + def _verify_tokenizer_mode(self) -> None:
57 + tokenizer_mode = self.tokenizer_mode.lower()
58 + if tokenizer_mode not in ["auto", "slow"]:
59 + raise ValueError(
60 + f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
61 + "either 'auto' or 'slow'.")
62 + self.tokenizer_mode = tokenizer_mode
63
64 def verify_with_parallel_config(
65 self,
vllm/engine/arg_utils.py
+9 -2
@@ -12,6 +12,7 @@ class EngineArgs:
12 """Arguments for vLLM engine."""
13 model: str
14 tokenizer: Optional[str] = None
15 + tokenizer_mode: str = "auto"
16 download_dir: Optional[str] = None
17 use_np_weights: bool = False
18 use_dummy_weights: bool = False
@@ -42,6 +43,12 @@ class EngineArgs:
43 help='name or path of the huggingface model to use')
44 parser.add_argument('--tokenizer', type=str, default=EngineArgs.tokenizer,
45 help='name or path of the huggingface tokenizer to use')
46 + parser.add_argument('--tokenizer-mode', type=str,
47 + default=EngineArgs.tokenizer_mode,
48 + choices=['auto', 'slow'],
49 + help='tokenizer mode. "auto" will use the fast '
50 + 'tokenizer if available, and "slow" will '
51 + 'always use the slow tokenizer.')
52 parser.add_argument('--download-dir', type=str,
53 default=EngineArgs.download_dir,
54 help='directory to download and load the weights, '
@@ -109,8 +116,8 @@ class EngineArgs:
116 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
117 # Initialize the configs.
118 model_config = ModelConfig(
112 - self.model, self.tokenizer, self.download_dir, self.use_np_weights,
113 - self.use_dummy_weights, self.dtype, self.seed)
119 + self.model, self.tokenizer, self.tokenizer_mode, self.download_dir,
120 + self.use_np_weights, self.use_dummy_weights, self.dtype, self.seed)
121 cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
122 self.swap_space)
123 parallel_config = ParallelConfig(self.pipeline_parallel_size,
vllm/engine/llm_engine.py
+3 -1
@@ -61,6 +61,7 @@ class LLMEngine:
61 "Initializing an LLM engine with config: "
62 f"model={model_config.model!r}, "
63 f"tokenizer={model_config.tokenizer!r}, "
64 + f"tokenizer_mode={model_config.tokenizer_mode}, "
65 f"dtype={model_config.dtype}, "
66 f"use_dummy_weights={model_config.use_dummy_weights}, "
67 f"download_dir={model_config.download_dir!r}, "
@@ -77,7 +78,8 @@ class LLMEngine:
78 self.log_stats = log_stats
79 self._verify_args()
80
80 - self.tokenizer = get_tokenizer(model_config.tokenizer)
81 + self.tokenizer = get_tokenizer(model_config.tokenizer,
82 + model_config.tokenizer_mode)
83 self.seq_counter = Counter()
84
85 # Create the parallel GPU workers.
vllm/entrypoints/llm.py
+4
@@ -26,6 +26,8 @@ class LLM:
26 Args:
27 model: The name or path of a HuggingFace Transformers model.
28 tokenizer: The name or path of a HuggingFace Transformers tokenizer.
29 + tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
30 + if available, and "slow" will always use the slow tokenizer.
31 tensor_parallel_size: The number of GPUs to use for distributed
32 execution with tensor parallelism.
33 dtype: The data type for the model weights and activations. Currently,
@@ -40,6 +42,7 @@ class LLM:
42 self,
43 model: str,
44 tokenizer: Optional[str] = None,
45 + tokenizer_mode: str = "auto",
46 tensor_parallel_size: int = 1,
47 dtype: str = "auto",
48 seed: int = 0,
@@ -50,6 +53,7 @@ class LLM:
53 engine_args = EngineArgs(
54 model=model,
55 tokenizer=tokenizer,
56 + tokenizer_mode=tokenizer_mode,
57 tensor_parallel_size=tensor_parallel_size,
58 dtype=dtype,
59 seed=seed,
vllm/entrypoints/openai/api_server.py
+1 -1
@@ -313,7 +313,7 @@ if __name__ == "__main__":
313 engine = AsyncLLMEngine.from_engine_args(engine_args)
314
315 # A separate tokenizer to map token IDs to strings.
316 - tokenizer = get_tokenizer(args.model)
316 + tokenizer = get_tokenizer(engine_args.tokenizer, engine_args.tokenizer_mode)
317
318 uvicorn.run(app, host=args.host, port=args.port, log_level="info",
319 timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
vllm/transformers_utils/tokenizer.py
+7
@@ -13,10 +13,17 @@ _FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
13
14 def get_tokenizer(
15 tokenizer_name: str,
16 + tokenizer_mode: str = "auto",
17 *args,
18 **kwargs,
19 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
20 """Gets a tokenizer for the given model name via Huggingface."""
21 + if tokenizer_mode == "slow":
22 + if kwargs.get("use_fast", False):
23 + raise ValueError(
24 + "Cannot use the fast tokenizer in slow tokenizer mode.")
25 + kwargs["use_fast"] = False
26 +
27 if "llama" in tokenizer_name.lower() and kwargs.get("use_fast", True):
28 logger.info(
29 "For some LLaMA-based models, initializing the fast tokenizer may "