A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Consolidate ModelConfig code related to HF config (#10104)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 7, 2024 at 14:00 UTC
db7db4aab9fd23e818d89ca9037099d30c071a5a
10 files changed
+68
-43
docs/source/serving/compatibility_matrix.rst
+1
-1
@@ -359,7 +359,7 @@ Feature x Hardware
359
- ✅
360
- ✅
361
- ✅
362
- - `✗ <https://github.com/vllm-project/vllm/blob/a84e598e2125960d3b4f716b78863f24ac562947/vllm/worker/cpu_model_runner.py#L125>`__
362
+ - ✅
363
- ✗
364
* - :abbr:`logP (Logprobs)`
365
- ✅
tests/test_config.py
+38
@@ -165,3 +165,41 @@ def test_rope_customization():
165
assert getattr(longchat_model_config.hf_config, "rope_scaling",
166
None) == TEST_ROPE_SCALING
167
assert longchat_model_config.max_model_len == 4096
168
+
169
+
170
+@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
171
+ ("facebook/opt-125m", False),
172
+ ("facebook/bart-base", True),
173
+ ("meta-llama/Llama-3.2-1B", False),
174
+ ("meta-llama/Llama-3.2-11B-Vision", True),
175
+])
176
+def test_is_encoder_decoder(model_id, is_encoder_decoder):
177
+ config = ModelConfig(
178
+ model_id,
179
+ task="auto",
180
+ tokenizer=model_id,
181
+ tokenizer_mode="auto",
182
+ trust_remote_code=False,
183
+ dtype="float16",
184
+ seed=0,
185
+ )
186
+
187
+ assert config.is_encoder_decoder == is_encoder_decoder
188
+
189
+
190
+@pytest.mark.parametrize(("model_id", "uses_mrope"), [
191
+ ("facebook/opt-125m", False),
192
+ ("Qwen/Qwen2-VL-2B-Instruct", True),
193
+])
194
+def test_uses_mrope(model_id, uses_mrope):
195
+ config = ModelConfig(
196
+ model_id,
197
+ task="auto",
198
+ tokenizer=model_id,
199
+ tokenizer_mode="auto",
200
+ trust_remote_code=False,
201
+ dtype="float16",
202
+ seed=0,
203
+ )
204
+
205
+ assert config.uses_mrope == uses_mrope
vllm/config.py
+8
-6
@@ -15,7 +15,8 @@ from vllm.platforms import current_platform
15
from vllm.tracing import is_otel_available, otel_import_error_traceback
16
from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
get_hf_image_processor_config,
18
- get_hf_text_config)
18
+ get_hf_text_config,
19
+ is_encoder_decoder, uses_mrope)
20
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
21
print_warning_once)
22
@@ -667,12 +668,13 @@ class ModelConfig:
668
return self.multimodal_config
669
670
@property
670
- def is_encoder_decoder_model(self) -> bool:
671
+ def is_encoder_decoder(self) -> bool:
672
"""Extract the HF encoder/decoder model flag."""
672
- return getattr(
673
- self.hf_config, "is_encoder_decoder",
674
- False) or (hasattr(self.hf_config, "text_config") and getattr(
675
- self.hf_config.text_config, "is_encoder_decoder", False))
673
+ return is_encoder_decoder(self.hf_config)
674
+
675
+ @property
676
+ def uses_mrope(self) -> bool:
677
+ return uses_mrope(self.hf_config)
678
679
@property
680
def is_multimodal_model(self) -> bool:
vllm/inputs/preprocess.py
+1
-1
@@ -580,4 +580,4 @@ class InputPreprocessor:
580
)
581
582
def is_encoder_decoder_model(self):
583
- return self.model_config.is_encoder_decoder_model
583
+ return self.model_config.is_encoder_decoder
vllm/transformers_utils/config.py
+9
@@ -129,6 +129,15 @@ def uses_mrope(config: PretrainedConfig) -> bool:
129
return "mrope_section" in rope_scaling
130
131
132
+def is_encoder_decoder(config: PretrainedConfig) -> bool:
133
+ """Detect if the model with this config is used as an encoder/decoder."""
134
+ text_config = getattr(config, "text_config", None)
135
+ if text_config is not None:
136
+ return is_encoder_decoder(text_config)
137
+
138
+ return getattr(config, "is_encoder_decoder", False)
139
+
140
+
141
def get_config(
142
model: Union[str, Path],
143
trust_remote_code: bool,
vllm/utils.py
-4
@@ -88,9 +88,6 @@ STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
88
"currently supported with encoder/"
89
"decoder models.")
90
91
-STR_NOT_IMPL_ENC_DEC_CPU = ("CPU is not currently supported with "
92
- "encoder/decoder models.")
93
-
91
# Efficiently import all enc/dec error strings
92
# rather than having to import all of the above
93
STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
@@ -105,7 +102,6 @@ STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
102
"STR_NOT_IMPL_ENC_DEC_SPEC_DEC": STR_NOT_IMPL_ENC_DEC_SPEC_DEC,
103
"STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
104
"STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
108
- "STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU
105
}
106
107
# Constants related to forcing the attention backend selection
vllm/worker/cpu_model_runner.py
+1
-8
@@ -18,7 +18,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
18
MultiModalInputs, MultiModalPlaceholderMap)
19
from vllm.sequence import (IntermediateTensors, SequenceData,
20
SequenceGroupMetadata)
21
-from vllm.transformers_utils.config import uses_mrope
21
from vllm.utils import make_tensor_with_pad
22
from vllm.worker.model_runner_base import (
23
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -163,7 +162,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
162
163
# special processing for mrope position deltas.
164
mrope_positions = None
166
- if self.runner.model_is_mrope:
165
+ if self.runner.model_config.uses_mrope:
166
image_grid_thw = mm_kwargs.get("image_grid_thw", None)
167
video_grid_thw = mm_kwargs.get("video_grid_thw", None)
168
assert image_grid_thw is not None or video_grid_thw is not None, (
@@ -446,12 +445,6 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
445
# Lazy initialization.
446
self.model: nn.Module # Set after init_Model
447
449
- @property
450
- def model_is_mrope(self) -> bool:
451
- """Detect if the model has "mrope" rope_scaling type.
452
- mrope requires keep "rope_deltas" between prompt and decoding phases."""
453
- return uses_mrope(self.model_config.hf_config)
454
-
448
def load_model(self) -> None:
449
self.model = get_model(vllm_config=self.vllm_config)
450
vllm/worker/cpu_worker.py
+1
-4
@@ -151,7 +151,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
151
self.local_omp_cpuid = omp_cpuids.split("|")[rank]
152
153
ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
154
- if self._is_encoder_decoder_model():
154
+ if self.model_config.is_encoder_decoder:
155
ModelRunnerClass = CPUEncoderDecoderModelRunner
156
self.model_runner: CPUModelRunner = ModelRunnerClass(
157
vllm_config=vllm_config,
@@ -188,9 +188,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
188
raise RuntimeError("Profiler is not enabled.")
189
self.profiler.stop()
190
191
- def _is_encoder_decoder_model(self):
192
- return self.model_config.is_encoder_decoder_model
193
-
191
def init_device(self) -> None:
192
if self.local_omp_cpuid != "all":
193
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
vllm/worker/model_runner.py
+8
-15
@@ -47,7 +47,6 @@ from vllm.prompt_adapter.worker_manager import (
47
LRUCacheWorkerPromptAdapterManager)
48
from vllm.sampling_params import SamplingParams
49
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50
-from vllm.transformers_utils.config import uses_mrope
50
from vllm.utils import (DeviceMemoryProfiler, GiB_bytes, PyObjectCache,
51
async_tensor_h2d, flatten_2d_lists,
52
is_pin_memory_available, supports_dynamo,
@@ -493,7 +492,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
492
context_len = seq_data.get_num_computed_tokens()
493
seq_len = min(seq_len, context_len + token_chunk_size)
494
elif self.runner.scheduler_config.is_multi_step or \
496
- self.runner.model_config.is_encoder_decoder_model:
495
+ self.runner.model_config.is_encoder_decoder:
496
context_len = seq_len - 1
497
else:
498
context_len = seq_data.get_num_computed_tokens()
@@ -666,7 +665,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
665
inter_data.multi_modal_placeholder_maps = placeholder_maps
666
667
# special processing for mrope position deltas.
669
- if self.runner.model_is_mrope:
668
+ if self.runner.model_config.uses_mrope:
669
image_grid_thw = mm_kwargs.get("image_grid_thw", None)
670
video_grid_thw = mm_kwargs.get("video_grid_thw", None)
671
assert image_grid_thw is not None or video_grid_thw is not None, (
@@ -711,7 +710,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
710
711
encoder_seq_len = 0
712
714
- if self.runner.model_config.is_encoder_decoder_model:
713
+ if self.runner.model_config.is_encoder_decoder:
714
encoder_seq_len = seq_group_metadata.encoder_seq_data.get_len()
715
716
inter_data = self.init_cached_inter_data(
@@ -837,7 +836,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
836
if not inter_data.is_prompt:
837
max_decode_seq_len = max(max_decode_seq_len,
838
max(inter_data.seq_lens))
840
- if self.runner.model_config.is_encoder_decoder_model:
839
+ if self.runner.model_config.is_encoder_decoder:
840
max_encoder_seq_len = max(max_encoder_seq_len,
841
inter_data.encoder_seq_len)
842
@@ -1375,12 +1374,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1374
raise RuntimeError("PromptAdapter is not enabled.")
1375
return self.prompt_adapter_manager.list_adapters()
1376
1378
- @property
1379
- def model_is_mrope(self) -> bool:
1380
- """Detect if the model has "mrope" rope_scaling type.
1381
- mrope requires keep "rope_deltas" between prompt and decoding phases."""
1382
- return uses_mrope(self.model_config.hf_config)
1383
-
1377
@torch.inference_mode()
1378
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
1379
"""Cuda graph capture a model.
@@ -1411,7 +1404,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1404
max_batch_size = self.max_batchsize_to_capture
1405
input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1406
input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1414
- if self.model_is_mrope:
1407
+ if self.model_config.uses_mrope:
1408
input_positions = torch.tile(input_positions, (3, 1))
1409
# Prepare dummy previous_hidden_states only if needed by the model.
1410
# This is used by draft models such as EAGLE.
@@ -1447,7 +1440,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1440
self.attn_state.graph_capture_get_metadata_for_batch(
1441
batch_size,
1442
is_encoder_decoder_model=self.model_config.
1450
- is_encoder_decoder_model))
1443
+ is_encoder_decoder))
1444
1445
if self.lora_config:
1446
lora_mapping = LoRAMapping(
@@ -1466,7 +1459,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1459
graph_runner = CUDAGraphRunner(
1460
self.model, self.attn_backend.get_name(),
1461
self.attn_state.graph_clone(batch_size),
1469
- self.model_config.is_encoder_decoder_model)
1462
+ self.model_config.is_encoder_decoder)
1463
1464
capture_inputs = {
1465
"input_ids":
@@ -1497,7 +1490,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1490
self.model.get_seqlen_agnostic_capture_inputs(
1491
batch_size)
1492
})
1500
- if self.model_config.is_encoder_decoder_model:
1493
+ if self.model_config.is_encoder_decoder:
1494
# add the additional inputs to capture for
1495
# encoder-decoder models.
1496
self._update_inputs_to_capture_for_enc_dec_model(
vllm/worker/worker.py
+1
-4
@@ -77,7 +77,7 @@ class Worker(LocalOrDistributedWorkerBase):
77
ModelRunnerClass = model_runner_cls
78
elif model_config.task == "embedding":
79
ModelRunnerClass = EmbeddingModelRunner
80
- elif self._is_encoder_decoder_model():
80
+ elif self.model_config.is_encoder_decoder:
81
ModelRunnerClass = EncoderDecoderModelRunner
82
self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
83
vllm_config=self.vllm_config,
@@ -119,9 +119,6 @@ class Worker(LocalOrDistributedWorkerBase):
119
raise RuntimeError("Profiler is not enabled.")
120
self.profiler.stop()
121
122
- def _is_encoder_decoder_model(self):
123
- return self.model_config.is_encoder_decoder_model
124
-
122
def init_device(self) -> None:
123
if self.device_config.device.type == "cuda":
124
# torch.distributed.all_reduce does not free the input tensor until