@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db7db4aab9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Consolidate ModelConfig code related to HF config (#10104)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 7, 2024 at 14:00 UTC db7db4aab9fd23e818d89ca9037099d30c071a5a
10 files changed +68 -43
docs/source/serving/compatibility_matrix.rst
+1 -1
@@ -359,7 +359,7 @@ Feature x Hardware
359 - ✅
360 - ✅
361 - ✅
362 - - `✗ <https://github.com/vllm-project/vllm/blob/a84e598e2125960d3b4f716b78863f24ac562947/vllm/worker/cpu_model_runner.py#L125>`__
362 + - ✅
363 - ✗
364 * - :abbr:`logP (Logprobs)`
365 - ✅
tests/test_config.py
+38
@@ -165,3 +165,41 @@ def test_rope_customization():
165 assert getattr(longchat_model_config.hf_config, "rope_scaling",
166 None) == TEST_ROPE_SCALING
167 assert longchat_model_config.max_model_len == 4096
168 +
169 +
170 +@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
171 + ("facebook/opt-125m", False),
172 + ("facebook/bart-base", True),
173 + ("meta-llama/Llama-3.2-1B", False),
174 + ("meta-llama/Llama-3.2-11B-Vision", True),
175 +])
176 +def test_is_encoder_decoder(model_id, is_encoder_decoder):
177 + config = ModelConfig(
178 + model_id,
179 + task="auto",
180 + tokenizer=model_id,
181 + tokenizer_mode="auto",
182 + trust_remote_code=False,
183 + dtype="float16",
184 + seed=0,
185 + )
186 +
187 + assert config.is_encoder_decoder == is_encoder_decoder
188 +
189 +
190 +@pytest.mark.parametrize(("model_id", "uses_mrope"), [
191 + ("facebook/opt-125m", False),
192 + ("Qwen/Qwen2-VL-2B-Instruct", True),
193 +])
194 +def test_uses_mrope(model_id, uses_mrope):
195 + config = ModelConfig(
196 + model_id,
197 + task="auto",
198 + tokenizer=model_id,
199 + tokenizer_mode="auto",
200 + trust_remote_code=False,
201 + dtype="float16",
202 + seed=0,
203 + )
204 +
205 + assert config.uses_mrope == uses_mrope
vllm/config.py
+8 -6
@@ -15,7 +15,8 @@ from vllm.platforms import current_platform
15 from vllm.tracing import is_otel_available, otel_import_error_traceback
16 from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 - get_hf_text_config)
18 + get_hf_text_config,
19 + is_encoder_decoder, uses_mrope)
20 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
21 print_warning_once)
22
@@ -667,12 +668,13 @@ class ModelConfig:
668 return self.multimodal_config
669
670 @property
670 - def is_encoder_decoder_model(self) -> bool:
671 + def is_encoder_decoder(self) -> bool:
672 """Extract the HF encoder/decoder model flag."""
672 - return getattr(
673 - self.hf_config, "is_encoder_decoder",
674 - False) or (hasattr(self.hf_config, "text_config") and getattr(
675 - self.hf_config.text_config, "is_encoder_decoder", False))
673 + return is_encoder_decoder(self.hf_config)
674 +
675 + @property
676 + def uses_mrope(self) -> bool:
677 + return uses_mrope(self.hf_config)
678
679 @property
680 def is_multimodal_model(self) -> bool:
vllm/inputs/preprocess.py
+1 -1
@@ -580,4 +580,4 @@ class InputPreprocessor:
580 )
581
582 def is_encoder_decoder_model(self):
583 - return self.model_config.is_encoder_decoder_model
583 + return self.model_config.is_encoder_decoder
vllm/transformers_utils/config.py
+9
@@ -129,6 +129,15 @@ def uses_mrope(config: PretrainedConfig) -> bool:
129 return "mrope_section" in rope_scaling
130
131
132 +def is_encoder_decoder(config: PretrainedConfig) -> bool:
133 + """Detect if the model with this config is used as an encoder/decoder."""
134 + text_config = getattr(config, "text_config", None)
135 + if text_config is not None:
136 + return is_encoder_decoder(text_config)
137 +
138 + return getattr(config, "is_encoder_decoder", False)
139 +
140 +
141 def get_config(
142 model: Union[str, Path],
143 trust_remote_code: bool,
vllm/utils.py
-4
@@ -88,9 +88,6 @@ STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
88 "currently supported with encoder/"
89 "decoder models.")
90
91 -STR_NOT_IMPL_ENC_DEC_CPU = ("CPU is not currently supported with "
92 - "encoder/decoder models.")
93 -
91 # Efficiently import all enc/dec error strings
92 # rather than having to import all of the above
93 STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
@@ -105,7 +102,6 @@ STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
102 "STR_NOT_IMPL_ENC_DEC_SPEC_DEC": STR_NOT_IMPL_ENC_DEC_SPEC_DEC,
103 "STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
104 "STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
108 - "STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU
105 }
106
107 # Constants related to forcing the attention backend selection
vllm/worker/cpu_model_runner.py
+1 -8
@@ -18,7 +18,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
18 MultiModalInputs, MultiModalPlaceholderMap)
19 from vllm.sequence import (IntermediateTensors, SequenceData,
20 SequenceGroupMetadata)
21 -from vllm.transformers_utils.config import uses_mrope
21 from vllm.utils import make_tensor_with_pad
22 from vllm.worker.model_runner_base import (
23 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -163,7 +162,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
162
163 # special processing for mrope position deltas.
164 mrope_positions = None
166 - if self.runner.model_is_mrope:
165 + if self.runner.model_config.uses_mrope:
166 image_grid_thw = mm_kwargs.get("image_grid_thw", None)
167 video_grid_thw = mm_kwargs.get("video_grid_thw", None)
168 assert image_grid_thw is not None or video_grid_thw is not None, (
@@ -446,12 +445,6 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
445 # Lazy initialization.
446 self.model: nn.Module # Set after init_Model
447
449 - @property
450 - def model_is_mrope(self) -> bool:
451 - """Detect if the model has "mrope" rope_scaling type.
452 - mrope requires keep "rope_deltas" between prompt and decoding phases."""
453 - return uses_mrope(self.model_config.hf_config)
454 -
448 def load_model(self) -> None:
449 self.model = get_model(vllm_config=self.vllm_config)
450
vllm/worker/cpu_worker.py
+1 -4
@@ -151,7 +151,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
151 self.local_omp_cpuid = omp_cpuids.split("|")[rank]
152
153 ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
154 - if self._is_encoder_decoder_model():
154 + if self.model_config.is_encoder_decoder:
155 ModelRunnerClass = CPUEncoderDecoderModelRunner
156 self.model_runner: CPUModelRunner = ModelRunnerClass(
157 vllm_config=vllm_config,
@@ -188,9 +188,6 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
188 raise RuntimeError("Profiler is not enabled.")
189 self.profiler.stop()
190
191 - def _is_encoder_decoder_model(self):
192 - return self.model_config.is_encoder_decoder_model
193 -
191 def init_device(self) -> None:
192 if self.local_omp_cpuid != "all":
193 ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
vllm/worker/model_runner.py
+8 -15
@@ -47,7 +47,6 @@ from vllm.prompt_adapter.worker_manager import (
47 LRUCacheWorkerPromptAdapterManager)
48 from vllm.sampling_params import SamplingParams
49 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50 -from vllm.transformers_utils.config import uses_mrope
50 from vllm.utils import (DeviceMemoryProfiler, GiB_bytes, PyObjectCache,
51 async_tensor_h2d, flatten_2d_lists,
52 is_pin_memory_available, supports_dynamo,
@@ -493,7 +492,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
492 context_len = seq_data.get_num_computed_tokens()
493 seq_len = min(seq_len, context_len + token_chunk_size)
494 elif self.runner.scheduler_config.is_multi_step or \
496 - self.runner.model_config.is_encoder_decoder_model:
495 + self.runner.model_config.is_encoder_decoder:
496 context_len = seq_len - 1
497 else:
498 context_len = seq_data.get_num_computed_tokens()
@@ -666,7 +665,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
665 inter_data.multi_modal_placeholder_maps = placeholder_maps
666
667 # special processing for mrope position deltas.
669 - if self.runner.model_is_mrope:
668 + if self.runner.model_config.uses_mrope:
669 image_grid_thw = mm_kwargs.get("image_grid_thw", None)
670 video_grid_thw = mm_kwargs.get("video_grid_thw", None)
671 assert image_grid_thw is not None or video_grid_thw is not None, (
@@ -711,7 +710,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
710
711 encoder_seq_len = 0
712
714 - if self.runner.model_config.is_encoder_decoder_model:
713 + if self.runner.model_config.is_encoder_decoder:
714 encoder_seq_len = seq_group_metadata.encoder_seq_data.get_len()
715
716 inter_data = self.init_cached_inter_data(
@@ -837,7 +836,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
836 if not inter_data.is_prompt:
837 max_decode_seq_len = max(max_decode_seq_len,
838 max(inter_data.seq_lens))
840 - if self.runner.model_config.is_encoder_decoder_model:
839 + if self.runner.model_config.is_encoder_decoder:
840 max_encoder_seq_len = max(max_encoder_seq_len,
841 inter_data.encoder_seq_len)
842
@@ -1375,12 +1374,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1374 raise RuntimeError("PromptAdapter is not enabled.")
1375 return self.prompt_adapter_manager.list_adapters()
1376
1378 - @property
1379 - def model_is_mrope(self) -> bool:
1380 - """Detect if the model has "mrope" rope_scaling type.
1381 - mrope requires keep "rope_deltas" between prompt and decoding phases."""
1382 - return uses_mrope(self.model_config.hf_config)
1383 -
1377 @torch.inference_mode()
1378 def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
1379 """Cuda graph capture a model.
@@ -1411,7 +1404,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1404 max_batch_size = self.max_batchsize_to_capture
1405 input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1406 input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1414 - if self.model_is_mrope:
1407 + if self.model_config.uses_mrope:
1408 input_positions = torch.tile(input_positions, (3, 1))
1409 # Prepare dummy previous_hidden_states only if needed by the model.
1410 # This is used by draft models such as EAGLE.
@@ -1447,7 +1440,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1440 self.attn_state.graph_capture_get_metadata_for_batch(
1441 batch_size,
1442 is_encoder_decoder_model=self.model_config.
1450 - is_encoder_decoder_model))
1443 + is_encoder_decoder))
1444
1445 if self.lora_config:
1446 lora_mapping = LoRAMapping(
@@ -1466,7 +1459,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1459 graph_runner = CUDAGraphRunner(
1460 self.model, self.attn_backend.get_name(),
1461 self.attn_state.graph_clone(batch_size),
1469 - self.model_config.is_encoder_decoder_model)
1462 + self.model_config.is_encoder_decoder)
1463
1464 capture_inputs = {
1465 "input_ids":
@@ -1497,7 +1490,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1490 self.model.get_seqlen_agnostic_capture_inputs(
1491 batch_size)
1492 })
1500 - if self.model_config.is_encoder_decoder_model:
1493 + if self.model_config.is_encoder_decoder:
1494 # add the additional inputs to capture for
1495 # encoder-decoder models.
1496 self._update_inputs_to_capture_for_enc_dec_model(
vllm/worker/worker.py
+1 -4
@@ -77,7 +77,7 @@ class Worker(LocalOrDistributedWorkerBase):
77 ModelRunnerClass = model_runner_cls
78 elif model_config.task == "embedding":
79 ModelRunnerClass = EmbeddingModelRunner
80 - elif self._is_encoder_decoder_model():
80 + elif self.model_config.is_encoder_decoder:
81 ModelRunnerClass = EncoderDecoderModelRunner
82 self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
83 vllm_config=self.vllm_config,
@@ -119,9 +119,6 @@ class Worker(LocalOrDistributedWorkerBase):
119 raise RuntimeError("Profiler is not enabled.")
120 self.profiler.stop()
121
122 - def _is_encoder_decoder_model(self):
123 - return self.model_config.is_encoder_decoder_model
124 -
122 def init_device(self) -> None:
123 if self.device_config.device.type == "cuda":
124 # torch.distributed.all_reduce does not free the input tensor until