@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a37d815b83
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Make initialization of tokenizer and detokenizer optional (#3748)

Co-authored-by: Yun Ding <yunding@nvidia.com> Co-authored-by: Roger Wang <ywang@roblox.com>

GeauxEric committed Apr 21, 2024 at 15:06 UTC a37d815b83849b5a96a182929dd6f3bd35f68fb8
6 files changed +68 -12
tests/engine/test_skip_tokenizer_init.py new
+23
@@ -0,0 +1,23 @@
1 +import pytest
2 +
3 +from vllm.entrypoints.llm import LLM
4 +from vllm.sampling_params import SamplingParams
5 +
6 +
7 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
8 +def test_skip_tokenizer_initialization(model: str):
9 + # This test checks if the flag skip_tokenizer_init skips the initialization
10 + # of tokenizer and detokenizer. The generated output is expected to contain
11 + # token ids.
12 + llm = LLM(model=model, skip_tokenizer_init=True)
13 + sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
14 + with pytest.raises(ValueError) as err:
15 + llm.generate("abc", sampling_params)
16 + assert "prompts must be None if" in str(err.value)
17 + outputs = llm.generate(prompt_token_ids=[[1, 2, 3]],
18 + sampling_params=sampling_params)
19 + assert len(outputs) > 0
20 + completions = outputs[0].outputs
21 + assert len(completions) > 0
22 + assert completions[0].text == ""
23 + assert completions[0].token_ids
vllm/config.py
+6 -1
@@ -66,6 +66,8 @@ class ModelConfig:
66 max_context_len_to_capture: Maximum context len covered by CUDA graphs.
67 When a sequence has context length larger than this, we fall back
68 to eager mode.
69 + skip_tokenizer_init: If true, skip initialization of tokenizer and
70 + detokenizer.
71 """
72
73 def __init__(
@@ -85,6 +87,7 @@ class ModelConfig:
87 enforce_eager: bool = False,
88 max_context_len_to_capture: Optional[int] = None,
89 max_logprobs: int = 5,
90 + skip_tokenizer_init: bool = False,
91 ) -> None:
92 self.model = model
93 self.tokenizer = tokenizer
@@ -99,6 +102,7 @@ class ModelConfig:
102 self.enforce_eager = enforce_eager
103 self.max_context_len_to_capture = max_context_len_to_capture
104 self.max_logprobs = max_logprobs
105 + self.skip_tokenizer_init = skip_tokenizer_init
106
107 self.hf_config = get_config(self.model, trust_remote_code, revision,
108 code_revision)
@@ -106,7 +110,8 @@ class ModelConfig:
110 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
111 self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
112 max_model_len)
109 - self._verify_tokenizer_mode()
113 + if not self.skip_tokenizer_init:
114 + self._verify_tokenizer_mode()
115 self._verify_quantization()
116 self._verify_cuda_graph()
117
vllm/engine/arg_utils.py
+6 -1
@@ -16,6 +16,7 @@ class EngineArgs:
16 """Arguments for vLLM engine."""
17 model: str
18 tokenizer: Optional[str] = None
19 + skip_tokenizer_init: bool = False
20 tokenizer_mode: str = 'auto'
21 trust_remote_code: bool = False
22 download_dir: Optional[str] = None
@@ -93,6 +94,10 @@ class EngineArgs:
94 type=str,
95 default=EngineArgs.tokenizer,
96 help='Name or path of the huggingface tokenizer to use.')
97 + parser.add_argument(
98 + '--skip-tokenizer-init',
99 + action='store_true',
100 + help='Skip initialization of tokenizer and detokenizer')
101 parser.add_argument(
102 '--revision',
103 type=str,
@@ -453,7 +458,7 @@ class EngineArgs:
458 self.code_revision, self.tokenizer_revision, self.max_model_len,
459 self.quantization, self.quantization_param_path,
460 self.enforce_eager, self.max_context_len_to_capture,
456 - self.max_logprobs)
461 + self.max_logprobs, self.skip_tokenizer_init)
462 cache_config = CacheConfig(self.block_size,
463 self.gpu_memory_utilization,
464 self.swap_space, self.kv_cache_dtype,
vllm/engine/llm_engine.py
+21 -8
@@ -100,6 +100,7 @@ class LLMEngine:
100 f"model={model_config.model!r}, "
101 f"speculative_config={speculative_config!r}, "
102 f"tokenizer={model_config.tokenizer!r}, "
103 + f"skip_tokenizer_init={model_config.skip_tokenizer_init}, "
104 f"tokenizer_mode={model_config.tokenizer_mode}, "
105 f"revision={model_config.revision}, "
106 f"tokenizer_revision={model_config.tokenizer_revision}, "
@@ -132,8 +133,14 @@ class LLMEngine:
133 self.decoding_config = decoding_config or DecodingConfig()
134 self.log_stats = log_stats
135
135 - self._init_tokenizer()
136 - self.detokenizer = Detokenizer(self.tokenizer)
136 + if not self.model_config.skip_tokenizer_init:
137 + self.tokenizer: BaseTokenizerGroup
138 + self._init_tokenizer()
139 + self.detokenizer = Detokenizer(self.tokenizer)
140 + else:
141 + self.detokenizer = None
142 + self.tokenizer = None
143 +
144 self.seq_counter = Counter()
145 self.generation_config_fields = _load_generation_config_dict(
146 model_config)
@@ -187,9 +194,10 @@ class LLMEngine:
194 parallel_config.disable_custom_all_reduce,
195 })
196
190 - # Ping the tokenizer to ensure liveness if it runs in a
191 - # different process.
192 - self.tokenizer.ping()
197 + if self.tokenizer:
198 + # Ping the tokenizer to ensure liveness if it runs in a
199 + # different process.
200 + self.tokenizer.ping()
201
202 # Create the scheduler.
203 # NOTE: the cache_config here have been updated with the numbers of
@@ -296,7 +304,7 @@ class LLMEngine:
304 trust_remote_code=self.model_config.trust_remote_code,
305 revision=self.model_config.tokenizer_revision)
306 init_kwargs.update(tokenizer_init_kwargs)
299 - self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
307 + self.tokenizer = get_tokenizer_group(
308 self.parallel_config.tokenizer_pool_config, **init_kwargs)
309
310 def _verify_args(self) -> None:
@@ -393,8 +401,13 @@ class LLMEngine:
401 # Create the sequences.
402 block_size = self.cache_config.block_size
403 seq_id = next(self.seq_counter)
396 - eos_token_id = self.tokenizer.get_lora_tokenizer(
397 - lora_request).eos_token_id
404 + eos_token_id = None
405 + if self.tokenizer:
406 + eos_token_id = self.tokenizer.get_lora_tokenizer(
407 + lora_request).eos_token_id
408 + else:
409 + logger.warning("Use None for EOS token id because tokenizer is "
410 + "not initialized")
411 seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
412 eos_token_id, lora_request)
413
vllm/engine/output_processor/single_step.py
+3 -2
@@ -59,7 +59,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
59
60 # Process prompt logprobs
61 prompt_logprobs = outputs.prompt_logprobs
62 - if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
62 + if prompt_logprobs is not None and \
63 + seq_group.sampling_params.detokenize and self.detokenizer:
64 self.detokenizer.decode_prompt_logprobs_inplace(
65 seq_group, prompt_logprobs)
66 seq_group.prompt_logprobs = prompt_logprobs
@@ -105,7 +106,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
106 child_seqs.append((parent, parent))
107
108 for seq, _ in child_seqs:
108 - if seq_group.sampling_params.detokenize:
109 + if seq_group.sampling_params.detokenize and self.detokenizer:
110 new_char_count = self.detokenizer.decode_sequence_inplace(
111 seq, seq_group.sampling_params)
112 else:
vllm/entrypoints/llm.py
+9
@@ -32,6 +32,9 @@ class LLM:
32 tokenizer: The name or path of a HuggingFace Transformers tokenizer.
33 tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
34 if available, and "slow" will always use the slow tokenizer.
35 + skip_tokenizer_init: If true, skip initialization of tokenizer and
36 + detokenizer. Expect valid prompt_token_ids and None for prompt
37 + from the input.
38 trust_remote_code: Trust remote code (e.g., from HuggingFace) when
39 downloading the model and tokenizer.
40 tensor_parallel_size: The number of GPUs to use for distributed
@@ -76,6 +79,7 @@ class LLM:
79 model: str,
80 tokenizer: Optional[str] = None,
81 tokenizer_mode: str = "auto",
82 + skip_tokenizer_init: bool = False,
83 trust_remote_code: bool = False,
84 tensor_parallel_size: int = 1,
85 dtype: str = "auto",
@@ -96,6 +100,7 @@ class LLM:
100 model=model,
101 tokenizer=tokenizer,
102 tokenizer_mode=tokenizer_mode,
103 + skip_tokenizer_init=skip_tokenizer_init,
104 trust_remote_code=trust_remote_code,
105 tensor_parallel_size=tensor_parallel_size,
106 dtype=dtype,
@@ -160,6 +165,10 @@ class LLM:
165 if prompts is None and prompt_token_ids is None:
166 raise ValueError("Either prompts or prompt_token_ids must be "
167 "provided.")
168 + if self.llm_engine.model_config.skip_tokenizer_init \
169 + and prompts is not None:
170 + raise ValueError("prompts must be None if skip_tokenizer_init "
171 + "is True")
172 if isinstance(prompts, str):
173 # Convert a single prompt to a list.
174 prompts = [prompts]