A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Make initialization of tokenizer and detokenizer optional (#3748)
Co-authored-by: Yun Ding <yunding@nvidia.com> Co-authored-by: Roger Wang <ywang@roblox.com>
GeauxEric committed
Apr 21, 2024 at 15:06 UTC
a37d815b83849b5a96a182929dd6f3bd35f68fb8
6 files changed
+68
-12
tests/engine/test_skip_tokenizer_init.py
new
+23
@@ -0,0 +1,23 @@
1
+import pytest
2
+
3
+from vllm.entrypoints.llm import LLM
4
+from vllm.sampling_params import SamplingParams
5
+
6
+
7
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
8
+def test_skip_tokenizer_initialization(model: str):
9
+ # This test checks if the flag skip_tokenizer_init skips the initialization
10
+ # of tokenizer and detokenizer. The generated output is expected to contain
11
+ # token ids.
12
+ llm = LLM(model=model, skip_tokenizer_init=True)
13
+ sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
14
+ with pytest.raises(ValueError) as err:
15
+ llm.generate("abc", sampling_params)
16
+ assert "prompts must be None if" in str(err.value)
17
+ outputs = llm.generate(prompt_token_ids=[[1, 2, 3]],
18
+ sampling_params=sampling_params)
19
+ assert len(outputs) > 0
20
+ completions = outputs[0].outputs
21
+ assert len(completions) > 0
22
+ assert completions[0].text == ""
23
+ assert completions[0].token_ids
vllm/config.py
+6
-1
@@ -66,6 +66,8 @@ class ModelConfig:
66
max_context_len_to_capture: Maximum context len covered by CUDA graphs.
67
When a sequence has context length larger than this, we fall back
68
to eager mode.
69
+ skip_tokenizer_init: If true, skip initialization of tokenizer and
70
+ detokenizer.
71
"""
72
73
def __init__(
@@ -85,6 +87,7 @@ class ModelConfig:
87
enforce_eager: bool = False,
88
max_context_len_to_capture: Optional[int] = None,
89
max_logprobs: int = 5,
90
+ skip_tokenizer_init: bool = False,
91
) -> None:
92
self.model = model
93
self.tokenizer = tokenizer
@@ -99,6 +102,7 @@ class ModelConfig:
102
self.enforce_eager = enforce_eager
103
self.max_context_len_to_capture = max_context_len_to_capture
104
self.max_logprobs = max_logprobs
105
+ self.skip_tokenizer_init = skip_tokenizer_init
106
107
self.hf_config = get_config(self.model, trust_remote_code, revision,
108
code_revision)
@@ -106,7 +110,8 @@ class ModelConfig:
110
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
111
self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
112
max_model_len)
109
- self._verify_tokenizer_mode()
113
+ if not self.skip_tokenizer_init:
114
+ self._verify_tokenizer_mode()
115
self._verify_quantization()
116
self._verify_cuda_graph()
117
vllm/engine/arg_utils.py
+6
-1
@@ -16,6 +16,7 @@ class EngineArgs:
16
"""Arguments for vLLM engine."""
17
model: str
18
tokenizer: Optional[str] = None
19
+ skip_tokenizer_init: bool = False
20
tokenizer_mode: str = 'auto'
21
trust_remote_code: bool = False
22
download_dir: Optional[str] = None
@@ -93,6 +94,10 @@ class EngineArgs:
94
type=str,
95
default=EngineArgs.tokenizer,
96
help='Name or path of the huggingface tokenizer to use.')
97
+ parser.add_argument(
98
+ '--skip-tokenizer-init',
99
+ action='store_true',
100
+ help='Skip initialization of tokenizer and detokenizer')
101
parser.add_argument(
102
'--revision',
103
type=str,
@@ -453,7 +458,7 @@ class EngineArgs:
458
self.code_revision, self.tokenizer_revision, self.max_model_len,
459
self.quantization, self.quantization_param_path,
460
self.enforce_eager, self.max_context_len_to_capture,
456
- self.max_logprobs)
461
+ self.max_logprobs, self.skip_tokenizer_init)
462
cache_config = CacheConfig(self.block_size,
463
self.gpu_memory_utilization,
464
self.swap_space, self.kv_cache_dtype,
vllm/engine/llm_engine.py
+21
-8
@@ -100,6 +100,7 @@ class LLMEngine:
100
f"model={model_config.model!r}, "
101
f"speculative_config={speculative_config!r}, "
102
f"tokenizer={model_config.tokenizer!r}, "
103
+ f"skip_tokenizer_init={model_config.skip_tokenizer_init}, "
104
f"tokenizer_mode={model_config.tokenizer_mode}, "
105
f"revision={model_config.revision}, "
106
f"tokenizer_revision={model_config.tokenizer_revision}, "
@@ -132,8 +133,14 @@ class LLMEngine:
133
self.decoding_config = decoding_config or DecodingConfig()
134
self.log_stats = log_stats
135
135
- self._init_tokenizer()
136
- self.detokenizer = Detokenizer(self.tokenizer)
136
+ if not self.model_config.skip_tokenizer_init:
137
+ self.tokenizer: BaseTokenizerGroup
138
+ self._init_tokenizer()
139
+ self.detokenizer = Detokenizer(self.tokenizer)
140
+ else:
141
+ self.detokenizer = None
142
+ self.tokenizer = None
143
+
144
self.seq_counter = Counter()
145
self.generation_config_fields = _load_generation_config_dict(
146
model_config)
@@ -187,9 +194,10 @@ class LLMEngine:
194
parallel_config.disable_custom_all_reduce,
195
})
196
190
- # Ping the tokenizer to ensure liveness if it runs in a
191
- # different process.
192
- self.tokenizer.ping()
197
+ if self.tokenizer:
198
+ # Ping the tokenizer to ensure liveness if it runs in a
199
+ # different process.
200
+ self.tokenizer.ping()
201
202
# Create the scheduler.
203
# NOTE: the cache_config here have been updated with the numbers of
@@ -296,7 +304,7 @@ class LLMEngine:
304
trust_remote_code=self.model_config.trust_remote_code,
305
revision=self.model_config.tokenizer_revision)
306
init_kwargs.update(tokenizer_init_kwargs)
299
- self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
307
+ self.tokenizer = get_tokenizer_group(
308
self.parallel_config.tokenizer_pool_config, **init_kwargs)
309
310
def _verify_args(self) -> None:
@@ -393,8 +401,13 @@ class LLMEngine:
401
# Create the sequences.
402
block_size = self.cache_config.block_size
403
seq_id = next(self.seq_counter)
396
- eos_token_id = self.tokenizer.get_lora_tokenizer(
397
- lora_request).eos_token_id
404
+ eos_token_id = None
405
+ if self.tokenizer:
406
+ eos_token_id = self.tokenizer.get_lora_tokenizer(
407
+ lora_request).eos_token_id
408
+ else:
409
+ logger.warning("Use None for EOS token id because tokenizer is "
410
+ "not initialized")
411
seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
412
eos_token_id, lora_request)
413
vllm/engine/output_processor/single_step.py
+3
-2
@@ -59,7 +59,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
59
60
# Process prompt logprobs
61
prompt_logprobs = outputs.prompt_logprobs
62
- if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
62
+ if prompt_logprobs is not None and \
63
+ seq_group.sampling_params.detokenize and self.detokenizer:
64
self.detokenizer.decode_prompt_logprobs_inplace(
65
seq_group, prompt_logprobs)
66
seq_group.prompt_logprobs = prompt_logprobs
@@ -105,7 +106,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
106
child_seqs.append((parent, parent))
107
108
for seq, _ in child_seqs:
108
- if seq_group.sampling_params.detokenize:
109
+ if seq_group.sampling_params.detokenize and self.detokenizer:
110
new_char_count = self.detokenizer.decode_sequence_inplace(
111
seq, seq_group.sampling_params)
112
else:
vllm/entrypoints/llm.py
+9
@@ -32,6 +32,9 @@ class LLM:
32
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
33
tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
34
if available, and "slow" will always use the slow tokenizer.
35
+ skip_tokenizer_init: If true, skip initialization of tokenizer and
36
+ detokenizer. Expect valid prompt_token_ids and None for prompt
37
+ from the input.
38
trust_remote_code: Trust remote code (e.g., from HuggingFace) when
39
downloading the model and tokenizer.
40
tensor_parallel_size: The number of GPUs to use for distributed
@@ -76,6 +79,7 @@ class LLM:
79
model: str,
80
tokenizer: Optional[str] = None,
81
tokenizer_mode: str = "auto",
82
+ skip_tokenizer_init: bool = False,
83
trust_remote_code: bool = False,
84
tensor_parallel_size: int = 1,
85
dtype: str = "auto",
@@ -96,6 +100,7 @@ class LLM:
100
model=model,
101
tokenizer=tokenizer,
102
tokenizer_mode=tokenizer_mode,
103
+ skip_tokenizer_init=skip_tokenizer_init,
104
trust_remote_code=trust_remote_code,
105
tensor_parallel_size=tensor_parallel_size,
106
dtype=dtype,
@@ -160,6 +165,10 @@ class LLM:
165
if prompts is None and prompt_token_ids is None:
166
raise ValueError("Either prompts or prompt_token_ids must be "
167
"provided.")
168
+ if self.llm_engine.model_config.skip_tokenizer_init \
169
+ and prompts is not None:
170
+ raise ValueError("prompts must be None if skip_tokenizer_init "
171
+ "is True")
172
if isinstance(prompts, str):
173
# Convert a single prompt to a list.
174
prompts = [prompts]