@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 58170d6503
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Hardware][CPU] Add embedding models support for CPU backend (#10193)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 11, 2024 at 16:54 UTC 58170d65034f7a89edc56c716f1fcf05ff336aa5
9 files changed +185 -52
.buildkite/run-cpu-test-ppc64le.sh
+1 -2
@@ -25,8 +25,7 @@ function cpu_tests() {
25 decord einops librosa peft Pillow sentence-transformers soundfile \
26 transformers_stream_generator matplotlib datamodel_code_generator
27 pip install torchvision --index-url https://download.pytorch.org/whl/cpu
28 - # Embedding models are not supported for CPU yet
29 - # pytest -v -s tests/models/embedding/language
28 + pytest -v -s tests/models/embedding/language
29 pytest -v -s tests/models/encoder_decoder/language
30 pytest -v -s tests/models/decoder_only/language/test_models.py
31 pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
.buildkite/run-cpu-test.sh
+1 -2
@@ -32,8 +32,7 @@ function cpu_tests() {
32 decord einops librosa peft Pillow sentence-transformers soundfile \
33 transformers_stream_generator matplotlib datamodel_code_generator
34 pip install torchvision --index-url https://download.pytorch.org/whl/cpu
35 - # Embedding models are not supported for CPU yet
36 - # pytest -v -s tests/models/embedding/language
35 + pytest -v -s tests/models/embedding/language
36 pytest -v -s tests/models/encoder_decoder/language
37 pytest -v -s tests/models/decoder_only/language/test_models.py
38 pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
tests/models/embedding/language/test_embedding.py
+4 -3
@@ -4,6 +4,8 @@ Run `pytest tests/models/embedding/language/test_embedding.py`.
4 """
5 import pytest
6
7 +from vllm.utils import current_platform
8 +
9 from ..utils import check_embeddings_close
10
11 # Model, Guard
@@ -21,15 +23,14 @@ ENCODER_ONLY = [
23 @pytest.mark.parametrize("model", MODELS)
24 @pytest.mark.parametrize("dtype", ["half"])
25 def test_models(
24 - monkeypatch,
26 hf_runner,
27 vllm_runner,
28 example_prompts,
29 model,
30 dtype: str,
31 ) -> None:
31 - if model in ENCODER_ONLY:
32 - monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
32 + if model not in ENCODER_ONLY and current_platform.is_cpu():
33 + pytest.skip("Skip large embedding models test on CPU.")
34
35 # The example_prompts has ending "\n", for example:
36 # "Write a short story about a robot that dreams for the first time.\n"
vllm/attention/backends/torch_sdpa.py
+10 -4
@@ -158,7 +158,8 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
158 * Appropriate sequence lengths tensor for key & value
159 '''
160
161 - if attn_type == AttentionType.DECODER:
161 + if (attn_type == AttentionType.DECODER
162 + or attn_type == AttentionType.ENCODER_ONLY):
163 seq_lens_q = self.seq_lens
164 seq_lens_kv = self.seq_lens
165 elif attn_type == AttentionType.ENCODER:
@@ -189,7 +190,8 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
190 * Appropriate attention bias value given the attention type
191 '''
192
192 - if attn_type == AttentionType.DECODER:
193 + if (attn_type == AttentionType.DECODER
194 + or attn_type == AttentionType.ENCODER_ONLY):
195 return self.attn_bias
196 elif attn_type == AttentionType.ENCODER:
197 return self.encoder_attn_bias
@@ -215,7 +217,8 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
217 encoder/decoder cross-attention
218 '''
219
218 - if attn_type == AttentionType.DECODER:
220 + if (attn_type == AttentionType.DECODER
221 + or attn_type == AttentionType.ENCODER_ONLY):
222 self.attn_bias = attn_bias
223 elif attn_type == AttentionType.ENCODER:
224 self.encoder_attn_bias = attn_bias
@@ -252,7 +255,8 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
255 * Appropriate block tables (or None)
256 '''
257
255 - if attn_type == AttentionType.DECODER:
258 + if (attn_type == AttentionType.DECODER
259 + or attn_type == AttentionType.ENCODER_ONLY):
260 # Decoder self-attention
261 # Choose max_seq_len based on whether we are in prompt_run
262 return (self.seq_lens_tensor, self.max_decode_seq_len,
@@ -420,6 +424,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
424 "Torch SDPA backend doesn't support prefix decoding.")
425
426 if decode_meta := attn_metadata.decode_metadata:
427 + assert attn_type != AttentionType.ENCODER_ONLY, (
428 + "Encoder-only models should not have decode metadata.")
429 # Decoding run.
430 (
431 seq_lens_arg,
vllm/model_executor/models/bert.py
-6
@@ -5,7 +5,6 @@ from torch import nn
5 from transformers import BertConfig
6
7 from vllm.attention import Attention, AttentionMetadata, AttentionType
8 -from vllm.attention.backends.xformers import XFormersImpl
8 from vllm.config import CacheConfig, VllmConfig
9 from vllm.distributed import get_tensor_model_parallel_world_size
10 from vllm.model_executor.layers.activation import get_act_fn
@@ -218,11 +217,6 @@ class BertSelfAttention(nn.Module):
217 quant_config=quant_config,
218 prefix=f"{prefix}.attn")
219
221 - if not isinstance(self.attn.impl, XFormersImpl):
222 - raise ValueError(
223 - "Encoder-only models currently require XFORMERS attention "
224 - "backend. Set VLLM_ATTENTION_BACKEND=XFORMERS to use BERT.")
225 -
220 def forward(
221 self,
222 hidden_states: torch.Tensor,
vllm/worker/cpu_embedding_model_runner.py new
+122
@@ -0,0 +1,122 @@
1 +import dataclasses
2 +from typing import Any, Dict, List, Optional, Tuple, Type, Union
3 +
4 +import torch
5 +
6 +from vllm.model_executor.pooling_metadata import PoolingMetadata
7 +from vllm.multimodal import MultiModalKwargs
8 +from vllm.pooling_params import PoolingParams
9 +from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
10 + SequenceGroupMetadata)
11 +from vllm.worker.cpu_model_runner import (CPUModelRunnerBase, ModelInputForCPU,
12 + ModelInputForCPUBuilder)
13 +
14 +
15 +@dataclasses.dataclass(frozen=True)
16 +class ModelInputForCPUWithPoolingMetadata(ModelInputForCPU):
17 + """
18 + Used by the CPUEmbeddingModelRunner.
19 + """
20 + pooling_metadata: Optional["PoolingMetadata"] = None
21 +
22 +
23 +class CPUEmbeddingModelRunner(
24 + CPUModelRunnerBase[ModelInputForCPUWithPoolingMetadata]):
25 + _model_input_cls: Type[ModelInputForCPUWithPoolingMetadata] = (
26 + ModelInputForCPUWithPoolingMetadata)
27 + _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
28 +
29 + @torch.inference_mode()
30 + def execute_model(
31 + self,
32 + model_input: ModelInputForCPUWithPoolingMetadata,
33 + kv_caches: List[torch.Tensor],
34 + intermediate_tensors: Optional[IntermediateTensors] = None,
35 + num_steps: int = 1,
36 + ) -> Optional[Union[List[PoolerOutput], IntermediateTensors]]:
37 + if num_steps > 1:
38 + raise ValueError(
39 + "CPU worker does not support multi-step execution.")
40 +
41 + num_layers = self.model_config.get_num_layers(self.parallel_config)
42 + # use an empty tensor instead of `None`` to force Dynamo to pass
43 + # it by reference, rather by specializing on the value ``None``.
44 + # the `dtype` argument does not matter, and we use `float32` as
45 + # a placeholder (it has wide hardware support).
46 + kv_caches = [
47 + torch.tensor([], dtype=torch.float32, device=self.device)
48 + for _ in range(num_layers)
49 + ]
50 +
51 + model_executable = self.model
52 + execute_model_kwargs = {
53 + "input_ids":
54 + model_input.input_tokens,
55 + "positions":
56 + model_input.input_positions,
57 + "kv_caches":
58 + kv_caches,
59 + "attn_metadata":
60 + model_input.attn_metadata,
61 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
62 + device=self.device),
63 + "intermediate_tensors":
64 + intermediate_tensors,
65 + }
66 +
67 + hidden_states = model_executable(**execute_model_kwargs)
68 +
69 + return [
70 + self.model.pooler(hidden_states=hidden_states,
71 + pooling_metadata=model_input.pooling_metadata)
72 + ]
73 +
74 + def make_model_input_from_broadcasted_tensor_dict(
75 + self,
76 + tensor_dict: Dict[str,
77 + Any]) -> ModelInputForCPUWithPoolingMetadata:
78 + return ModelInputForCPUWithPoolingMetadata.from_broadcasted_tensor_dict(
79 + tensor_dict,
80 + attn_backend=self.attn_backend,
81 + )
82 +
83 + def prepare_model_input(
84 + self,
85 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
86 + virtual_engine: int = 0,
87 + finished_requests_ids: Optional[List[str]] = None
88 + ) -> ModelInputForCPUWithPoolingMetadata:
89 + assert seq_group_metadata_list is not None
90 + model_input = self._prepare_model_input_tensors(
91 + seq_group_metadata_list, finished_requests_ids)
92 + # Prepare PoolingMetadata.
93 + assert model_input.seq_lens is not None
94 + pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
95 + model_input.seq_lens)
96 +
97 + return dataclasses.replace(model_input,
98 + pooling_metadata=pooling_metadata)
99 +
100 + def _prepare_pooling(
101 + self,
102 + seq_group_metadata_list: List[SequenceGroupMetadata],
103 + prompt_lens: List[int],
104 + ) -> PoolingMetadata:
105 + """Prepare PoolingMetadata for the sequence group metadata list."""
106 + seq_groups: List[Tuple[List[int], PoolingParams]] = []
107 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
108 + seq_ids = list(seq_group_metadata.seq_data.keys())
109 + pooling_params = seq_group_metadata.pooling_params
110 + seq_groups.append((seq_ids, pooling_params))
111 +
112 + seq_data: Dict[int, SequenceData] = {}
113 + for seq_group_metadata in seq_group_metadata_list:
114 + seq_data.update(seq_group_metadata.seq_data)
115 +
116 + pooling_metadata = PoolingMetadata(
117 + seq_groups=seq_groups,
118 + seq_data=seq_data,
119 + prompt_lens=prompt_lens,
120 + )
121 +
122 + return pooling_metadata
vllm/worker/cpu_enc_dec_model_runner.py
+5 -6
@@ -8,7 +8,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput
8 from vllm.multimodal import MultiModalKwargs
9 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
10 from vllm.utils import make_tensor_with_pad
11 -from vllm.worker.cpu_model_runner import (CPUModelRunner,
11 +from vllm.worker.cpu_model_runner import (CPUModelRunnerBase,
12 ModelInputForCPUBuilder,
13 ModelInputForCPUWithSamplingMetadata)
14 from vllm.worker.model_runner_base import (
@@ -50,7 +50,8 @@ class EncoderDecoderModelInputForCPU(ModelInputForCPUWithSamplingMetadata):
50 super().from_broadcasted_tensor_dict(tensor_dict, attn_backend))
51
52
53 -class CPUEncoderDecoderModelRunner(CPUModelRunner):
53 +class CPUEncoderDecoderModelRunner(
54 + CPUModelRunnerBase[EncoderDecoderModelInputForCPU]):
55 _model_input_cls: Type[EncoderDecoderModelInputForCPU] = (
56 EncoderDecoderModelInputForCPU)
57 _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
@@ -87,10 +88,8 @@ class CPUEncoderDecoderModelRunner(CPUModelRunner):
88 virtual_engine: int = 0,
89 finished_requests_ids: Optional[List[str]] = None
90 ) -> EncoderDecoderModelInputForCPU:
90 - model_input = super().prepare_model_input(seq_group_metadata_list,
91 - virtual_engine,
92 - finished_requests_ids)
93 - model_input = cast(EncoderDecoderModelInputForCPU, model_input)
91 + model_input = self._prepare_model_input_tensors(
92 + seq_group_metadata_list, finished_requests_ids)
93 (
94 attn_metadata,
95 encoder_input_tokens_tensor,
vllm/worker/cpu_model_runner.py
+35 -22
@@ -2,7 +2,8 @@ import dataclasses
2 import weakref
3 from collections import defaultdict
4 from dataclasses import dataclass
5 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
5 +from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6 + TypeVar, Union)
7
8 import torch
9 from torch import nn
@@ -31,6 +32,7 @@ if TYPE_CHECKING:
32
33 logger = init_logger(__name__)
34
35 +TModelInputForCPU = TypeVar('TModelInputForCPU', bound="ModelInputForCPU")
36 _PAD_SLOT_ID = -1
37
38
@@ -60,10 +62,10 @@ class ModelInputForCPU(ModelRunnerInputBase):
62
63 @classmethod
64 def from_broadcasted_tensor_dict(
63 - cls: Type["ModelInputForCPU"],
65 + cls: Type[TModelInputForCPU],
66 tensor_dict: Dict[str, Any],
67 attn_backend: Optional["AttentionBackend"] = None
66 - ) -> "ModelInputForCPU":
68 + ) -> TModelInputForCPU:
69 if attn_backend is not None:
70 tensor_dict = _init_attn_metadata_from_tensor_dict(
71 attn_backend, tensor_dict)
@@ -255,11 +257,14 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
257 slot_mapping.append(_PAD_SLOT_ID)
258 continue
259
258 - block_number = block_table[i //
259 - self.block_size] # type: ignore
260 - block_offset = i % self.block_size # type: ignore
261 - slot = block_number * self.block_size + block_offset
262 - slot_mapping.append(slot)
260 + # For encoder-only models, the block_table is None,
261 + # and there is no need to initialize the slot_mapping.
262 + if block_table is not None:
263 + block_number = block_table[i //
264 + self.block_size] # type: ignore
265 + block_offset = i % self.block_size # type: ignore
266 + slot = block_number * self.block_size + block_offset
267 + slot_mapping.append(slot)
268
269 if any(input_mrope_positions):
270 input_positions = None # type: ignore
@@ -402,10 +407,12 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
407 )
408
409
405 -class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
406 - _model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
407 - ModelInputForCPUWithSamplingMetadata)
408 - _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
410 +class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
411 + """
412 + Helper class for shared methods between CPU model runners.
413 + """
414 + _model_input_cls: Type[TModelInputForCPU]
415 + _builder_cls: Type[ModelInputForCPUBuilder]
416
417 def __init__(
418 self,
@@ -448,20 +455,11 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
455 def load_model(self) -> None:
456 self.model = get_model(vllm_config=self.vllm_config)
457
451 - def make_model_input_from_broadcasted_tensor_dict(
452 - self,
453 - tensor_dict: Dict[str, Any],
454 - ) -> ModelInputForCPUWithSamplingMetadata:
455 - return ModelInputForCPUWithSamplingMetadata.from_broadcasted_tensor_dict( # noqa: E501
456 - tensor_dict,
457 - attn_backend=self.attn_backend,
458 - )
459 -
458 def _prepare_model_input_tensors(
459 self,
460 seq_group_metadata_list: List[SequenceGroupMetadata],
461 finished_requests_ids: Optional[List[str]] = None
464 - ) -> ModelInputForCPUWithSamplingMetadata:
462 + ) -> TModelInputForCPU:
463 """Helper method to prepare the model input based on a given sequence
464 group. Prepares metadata needed for the base model forward pass but not
465 metadata for possible additional steps, e.g., sampling.
@@ -473,6 +471,21 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
471
472 return builder.build() # type: ignore
473
474 +
475 +class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
476 + _model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
477 + ModelInputForCPUWithSamplingMetadata)
478 + _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
479 +
480 + def make_model_input_from_broadcasted_tensor_dict(
481 + self,
482 + tensor_dict: Dict[str, Any],
483 + ) -> ModelInputForCPUWithSamplingMetadata:
484 + return ModelInputForCPUWithSamplingMetadata.from_broadcasted_tensor_dict( # noqa: E501
485 + tensor_dict,
486 + attn_backend=self.attn_backend,
487 + )
488 +
489 def prepare_model_input(
490 self,
491 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/cpu_worker.py
+7 -7
@@ -14,8 +14,9 @@ from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
15 from vllm.sequence import ExecuteModelRequest
16 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17 +from vllm.worker.cpu_embedding_model_runner import CPUEmbeddingModelRunner
18 from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18 -from vllm.worker.cpu_model_runner import CPUModelRunner
19 +from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
20 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21 LoraNotSupportedWorkerBase, WorkerBase,
22 WorkerInput)
@@ -150,21 +151,20 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
151 else:
152 self.local_omp_cpuid = omp_cpuids.split("|")[rank]
153
153 - ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
154 + ModelRunnerClass: Type[CPUModelRunnerBase] = CPUModelRunner
155 if self.model_config.task == "embedding":
155 - raise NotImplementedError(
156 - "Embedding models are not supported for CPU backend")
157 - # ModelRunnerClass = CPUEmbeddingModelRunner
156 + ModelRunnerClass = CPUEmbeddingModelRunner
157 elif self.model_config.is_encoder_decoder:
158 ModelRunnerClass = CPUEncoderDecoderModelRunner
160 - self.model_runner: CPUModelRunner = ModelRunnerClass(
159 + self.model_runner: CPUModelRunnerBase = ModelRunnerClass(
160 vllm_config=vllm_config,
161 kv_cache_dtype=kv_cache_dtype,
162 is_driver_worker=is_driver_worker)
163 # Uninitialized cache engine. Will be initialized by
164 # initialize_cache.
165 self.cache_engine: List[CPUCacheEngine]
167 - self.cpu_cache: List[List[torch.Tensor]]
166 + # Initialize cpu_cache as embedding models don't initialize kv_caches
167 + self.cpu_cache: Optional[List[List[torch.Tensor]]] = None
168
169 # Torch profiler. Enabled and configured through env vars:
170 # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace