@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ddbe25502
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Hardware][CPU] using current_platform.is_cpu (#9536)

wangshuai09 committed Oct 22, 2024 at 15:50 UTC 3ddbe25502fb8c49e67096ba6e641ecdc3519757
17 files changed +60 -64
tests/conftest.py
+4 -2
@@ -32,9 +32,10 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
32 to_enc_dec_tuple_list, zip_enc_dec_prompts)
33 from vllm.logger import init_logger
34 from vllm.outputs import RequestOutput
35 +from vllm.platforms import current_platform
36 from vllm.sampling_params import BeamSearchParams
37 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
37 - identity, is_cpu)
38 + identity)
39
40 logger = init_logger(__name__)
41
@@ -236,7 +237,8 @@ class HfRunner:
237
238 def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
239 if device is None:
239 - return self.wrap_device(input, "cpu" if is_cpu() else "cuda")
240 + return self.wrap_device(
241 + input, "cpu" if current_platform.is_cpu() else "cuda")
242
243 if hasattr(input, "device") and input.device.type == device:
244 return input
tests/encoder_decoder/test_e2e_correctness.py
+3 -3
@@ -7,8 +7,8 @@ from typing import List, Optional, Tuple
7 import pytest
8 from transformers import AutoModelForSeq2SeqLM
9
10 +from vllm.platforms import current_platform
11 from vllm.sequence import SampleLogprobs
11 -from vllm.utils import is_cpu
12
13 from ..conftest import DecoderPromptType
14 from ..models.utils import check_logprobs_close
@@ -35,7 +35,7 @@ def vllm_to_hf_output(
35 @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
36 @pytest.mark.parametrize("enforce_eager", [True, False])
37 @pytest.mark.skipif(
38 - is_cpu(),
38 + current_platform.is_cpu(),
39 reason="CPU backend is not currently supported with encoder/decoder models"
40 )
41 def test_encoder_decoder_e2e(
@@ -50,7 +50,7 @@ def test_encoder_decoder_e2e(
50 enforce_eager: bool,
51 ) -> None:
52 '''
53 - End-to-End (E2E) test for the encoder-decoder framework.
53 + End-to-End (E2E) test for the encoder-decoder framework.
54 This test evaluates the encoder-decoder functionality using the BART
55 model. We compare the outputs of the Hugging Face and vLLM
56 implementations to ensure that both implementations produce consistent
tests/kernels/test_attention_selector.py
+2 -1
@@ -19,7 +19,8 @@ def test_env(name: str, device: str, monkeypatch):
19 override_backend_env_variable(monkeypatch, name)
20
21 if device == "cpu":
22 - with patch("vllm.attention.selector.is_cpu", return_value=True):
22 + with patch("vllm.attention.selector.current_platform.is_cpu",
23 + return_value=True):
24 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
25 False)
26 assert backend.name == "TORCH_SDPA"
tests/models/decoder_only/language/test_phimoe.py
+2 -2
@@ -5,7 +5,7 @@ Run `pytest tests/models/test_phimoe.py`.
5 import pytest
6 import torch
7
8 -from vllm.utils import is_cpu
8 +from vllm.platforms import current_platform
9
10 from ....utils import large_gpu_test
11 from ...utils import check_logprobs_close
@@ -70,7 +70,7 @@ def test_phimoe_routing_function():
70 assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
71
72
73 -@pytest.mark.skipif(condition=is_cpu(),
73 +@pytest.mark.skipif(condition=current_platform.is_cpu(),
74 reason="This test takes a lot time to run on CPU, "
75 "and vllm CI's disk space is not enough for this model.")
76 @large_gpu_test(min_gb=80)
tests/models/decoder_only/vision_language/test_fuyu.py
+3 -3
@@ -3,8 +3,8 @@ from typing import List, Optional, Tuple, Type
3 import pytest
4
5 from vllm.multimodal.utils import rescale_image_size
6 +from vllm.platforms import current_platform
7 from vllm.sequence import SampleLogprobs
7 -from vllm.utils import is_cpu
8
9 from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 from ...utils import check_logprobs_close
@@ -46,7 +46,7 @@ def run_test(
46
47 All the image fixtures for the test are from IMAGE_ASSETS.
48 For huggingface runner, we provide the PIL images as input.
49 - For vllm runner, we provide MultiModalDataDict objects
49 + For vllm runner, we provide MultiModalDataDict objects
50 and corresponding MultiModalConfig as input.
51 Note, the text input is also adjusted to abide by vllm contract.
52 The text output is sanitized to be able to compare with hf.
@@ -103,7 +103,7 @@ def run_test(
103
104
105 target_dtype = "half"
106 -if is_cpu():
106 +if current_platform.is_cpu():
107 target_dtype = "bfloat16"
108
109
tests/models/decoder_only/vision_language/test_internvl.py
+3 -3
@@ -7,7 +7,7 @@ from PIL.Image import Image
7 from transformers import AutoConfig
8
9 from vllm.multimodal.utils import rescale_image_size
10 -from vllm.utils import is_cpu
10 +from vllm.platforms import current_platform
11
12 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13 _ImageAssets)
@@ -78,7 +78,7 @@ def run_test(
78
79 All the image fixtures for the test are from IMAGE_ASSETS.
80 For huggingface runner, we provide the PIL images as input.
81 - For vllm runner, we provide MultiModalDataDict objects
81 + For vllm runner, we provide MultiModalDataDict objects
82 and corresponding MultiModalConfig as input.
83 Note, the text input is also adjusted to abide by vllm contract.
84 The text output is sanitized to be able to compare with hf.
@@ -244,7 +244,7 @@ def run_awq_test(
244
245
246 target_dtype = "half"
247 -if is_cpu():
247 +if current_platform.is_cpu():
248 target_dtype = "bfloat16"
249
250
tests/models/decoder_only/vision_language/test_phi3v.py
+3 -2
@@ -10,8 +10,9 @@ from vllm.inputs import InputContext, token_inputs
10 from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
11 from vllm.multimodal import MultiModalRegistry
12 from vllm.multimodal.utils import rescale_image_size
13 +from vllm.platforms import current_platform
14 from vllm.sequence import SampleLogprobs
14 -from vllm.utils import is_cpu, is_hip
15 +from vllm.utils import is_hip
16
17 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
18 _ImageAssets)
@@ -49,7 +50,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
50
51
52 target_dtype = "half"
52 -if is_cpu():
53 +if current_platform.is_cpu():
54 target_dtype = "bfloat16"
55
56 # ROCm Triton FA can run into shared memory issues with these models,
tests/models/utils.py
+4 -4
@@ -5,8 +5,8 @@ import torch
5
6 from vllm.config import ModelConfig, TaskOption
7 from vllm.inputs import InputContext
8 +from vllm.platforms import current_platform
9 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9 -from vllm.utils import is_cpu
10
11 TokensText = Tuple[List[int], str]
12
@@ -19,7 +19,7 @@ def check_outputs_equal(
19 name_1: str,
20 ):
21 """
22 - Compare the two sequences generated by different models,
22 + Compare the two sequences generated by different models,
23 which should be equal.
24 """
25 assert len(outputs_0_lst) == len(outputs_1_lst)
@@ -255,7 +255,7 @@ def build_model_context(model_name: str,
255 mm_processor_kwargs: Optional[Dict] = None,
256 limit_mm_per_prompt: Optional[Dict] = None):
257 """Creates an InputContext for a given model.
258 -
258 +
259 Args:
260 model_name: Name of the model being considered.
261 tokenizer_name: Name of the tokenizer being considered.
@@ -270,7 +270,7 @@ def build_model_context(model_name: str,
270 if tokenizer_name is None:
271 tokenizer_name = model_name
272 if dtype is None:
273 - dtype = "bfloat16" if is_cpu() else "half"
273 + dtype = "bfloat16" if current_platform.is_cpu() else "half"
274
275 model_config = ModelConfig(
276 model_name,
tests/worker/test_encoder_decoder_model_runner.py
+6 -5
@@ -5,8 +5,9 @@ import pytest
5 import torch
6
7 from vllm.engine.arg_utils import EngineArgs
8 +from vllm.platforms import current_platform
9 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
9 -from vllm.utils import is_cpu, make_tensor_with_pad
10 +from vllm.utils import make_tensor_with_pad
11 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12 from vllm.worker.model_runner import _get_graph_batch_size
13
@@ -31,7 +32,7 @@ def _create_model_runner(model: str, *args,
32 return model_runner
33
34
34 -@pytest.mark.skipif(condition=is_cpu(),
35 +@pytest.mark.skipif(condition=current_platform.is_cpu(),
36 reason="CPU backend is currently "
37 "unsupported for encoder/ "
38 "decoder models")
@@ -74,7 +75,7 @@ def test_empty_seq_group():
75 assert return_seq_lens is None
76
77
77 -@pytest.mark.skipif(condition=is_cpu(),
78 +@pytest.mark.skipif(condition=current_platform.is_cpu(),
79 reason="CPU backend is currently "
80 "unsupported for encoder/ "
81 "decoder models")
@@ -264,7 +265,7 @@ def test_prepare_prompt(batch_size):
265 assert torch.equal(actual, expected)
266
267
267 -@pytest.mark.skipif(condition=is_cpu(),
268 +@pytest.mark.skipif(condition=current_platform.is_cpu(),
269 reason="CPU backend is currently "
270 "unsupported for encoder/ "
271 "decoder models")
@@ -490,7 +491,7 @@ def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
491 def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
492 """
493 Tests that for encoder-decoder models with CUDA Graph capture and replay
493 - enabled, the tensors used during the decode phase are correctly padded
494 + enabled, the tensors used during the decode phase are correctly padded
495 for varying input batch sizes.
496 """
497 model_runner = _create_model_runner(
vllm/attention/backends/torch_sdpa.py
+4 -4
@@ -10,9 +10,9 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata, AttentionType)
11 from vllm.attention.backends.utils import CommonAttentionState
12 from vllm.attention.ops.paged_attn import PagedAttentionMetadata
13 -from vllm.utils import is_cpu
13 +from vllm.platforms import current_platform
14
15 -if is_cpu():
15 +if current_platform.is_cpu():
16 try:
17 from vllm.attention.ops.ipex_attn import PagedAttention
18 except ImportError:
@@ -234,10 +234,10 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
234 on the type of attention operation.
235
236 Decoder attn -> select entirely decoder self-attention-related fields
237 - Encoder/decoder cross-attn -> select encoder sequence lengths &
237 + Encoder/decoder cross-attn -> select encoder sequence lengths &
238 cross-attn block-tables fields
239 Encoder attn -> select encoder sequence lengths fields & no block tables
240 -
240 +
241 Arguments:
242
243 * attn_metadata: Attention metadata structure associated with attention
vllm/attention/ops/blocksparse_attention/interface.py
+10 -10
@@ -3,7 +3,7 @@ import math
3 import torch
4
5 from vllm.platforms import current_platform
6 -from vllm.utils import is_cpu, is_hip
6 +from vllm.utils import is_hip
7
8 from .utils import (dense_to_crow_col, get_head_sliding_step,
9 get_sparse_attn_mask)
@@ -32,7 +32,7 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
32 ):
33 super().__init__()
34 if use_spda is None:
35 - use_spda = is_hip() or is_cpu() or not \
35 + use_spda = is_hip() or current_platform.is_cpu() or not \
36 IS_COMPUTE_8_OR_ABOVE
37 device = device or (torch.cuda.current_device()
38 if current_platform.is_cuda_alike() else "cpu")
@@ -109,13 +109,13 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
109 q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
110 Support grouped attention, with `q[:, i*r:(i*r + r)]`
111 is correspondent to `k[:, i]`, where `r` is the q/k ratio.
112 - cu_seqlens_k: shape=(batch_size + 1,),
113 - indicating segment of samples,
112 + cu_seqlens_k: shape=(batch_size + 1,),
113 + indicating segment of samples,
114 e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i
115 cu_seqlens_q: shape=(batch_size + 1, ).
116 Default None: same as cu_seqlens_k for prefilling or
117 [0, 1, .., batch_size] for decoding.
118 - The only case you need to specify is when q is a mix of
118 + The only case you need to specify is when q is a mix of
119 prefilling and decoding.
120 sm_scale: softmax scale, default to 1/sqrt(head_size).
121
@@ -171,7 +171,7 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
171
172 def spda(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
173 """For CPU, V100 or other older GPUs.
174 - NOTE: torch SPDA supports nested tensor,
174 + NOTE: torch SPDA supports nested tensor,
175 but seems extremely slow. Choose to pad instead.
176 """
177 assert (cu_seqlens_q is None or
@@ -201,8 +201,8 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
201 return self.transpose_and_unpad(spda_output, cu_seqlens)
202
203 def forward(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
204 - """Dispatch to `varlen_attn` (Ampere or newer) or
205 - `self.spda`(cpu, Volta, Turing or older)based on
204 + """Dispatch to `varlen_attn` (Ampere or newer) or
205 + `self.spda`(cpu, Volta, Turing or older)based on
206 the type of device used and cuda compute capability.
207
208 q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
@@ -213,8 +213,8 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
213 cu_seqlens_q: shape=(batch_size + 1, ).
214 Default None: same as cu_seqlens_k for prefilling or
215 [0, 1, .., batch_size] for decoding.
216 - The only case you need to specify
217 - is when q is a mix of prefilling
216 + The only case you need to specify
217 + is when q is a mix of prefilling
218 and decoding.
219 sm_scale: softmax scale, default to 1/sqrt(head_size).
220
vllm/attention/selector.py
+3 -3
@@ -10,7 +10,7 @@ import vllm.envs as envs
10 from vllm.attention.backends.abstract import AttentionBackend
11 from vllm.logger import init_logger
12 from vllm.platforms import current_platform
13 -from vllm.utils import STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino, is_xpu
13 +from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino, is_xpu
14
15 logger = init_logger(__name__)
16
@@ -121,7 +121,7 @@ def get_attn_backend(
121 ROCmFlashAttentionBackend)
122 return ROCmFlashAttentionBackend
123 elif backend == _Backend.TORCH_SDPA:
124 - assert is_cpu(), RuntimeError(
124 + assert current_platform.is_cpu(), RuntimeError(
125 "Torch SDPA backend is only used for the CPU device.")
126 logger.info("Using Torch SDPA backend.")
127 from vllm.attention.backends.torch_sdpa import TorchSDPABackend
@@ -183,7 +183,7 @@ def which_attn_to_use(
183 if backend_by_env_var is not None:
184 selected_backend = backend_name_to_enum(backend_by_env_var)
185
186 - if is_cpu():
186 + if current_platform.is_cpu():
187 if selected_backend != _Backend.TORCH_SDPA:
188 logger.info("Cannot use %s backend on CPU.", selected_backend)
189 return _Backend.TORCH_SDPA
vllm/distributed/parallel_state.py
+3 -3
@@ -7,7 +7,7 @@ It takes over the control of the distributed environment from PyTorch.
7 The typical workflow is:
8
9 - call `init_distributed_environment` to initialize the distributed environment.
10 -- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
10 +- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
11 initialize the model parallel groups.
12
13 - any code dealing with the distributed stuff
@@ -37,7 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37 import vllm.envs as envs
38 from vllm.logger import init_logger
39 from vllm.platforms import current_platform
40 -from vllm.utils import is_cpu, supports_custom_op
40 +from vllm.utils import supports_custom_op
41
42
43 @dataclass
@@ -1139,7 +1139,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1139 import ray # Lazy import Ray
1140 ray.shutdown()
1141 gc.collect()
1142 - if not is_cpu():
1142 + if not current_platform.is_cpu():
1143 torch.cuda.empty_cache()
1144
1145
vllm/model_executor/custom_op.py
+2 -2
@@ -7,7 +7,7 @@ import vllm.envs as envs
7 from vllm.compilation.levels import CompilationLevel
8 from vllm.logger import init_logger
9 from vllm.platforms import current_platform
10 -from vllm.utils import is_cpu, is_hip, is_xpu, print_warning_once
10 +from vllm.utils import is_hip, is_xpu, print_warning_once
11
12 logger = init_logger(__name__)
13
@@ -74,7 +74,7 @@ class CustomOp(nn.Module):
74
75 if is_hip():
76 return self.forward_hip
77 - elif is_cpu():
77 + elif current_platform.is_cpu():
78 return self.forward_cpu
79 elif current_platform.is_tpu():
80 return self.forward_tpu
vllm/model_executor/models/qwen2_vl.py
+4 -4
@@ -78,7 +78,7 @@ logger = init_logger(__name__)
78 class Qwen2VLImagePixelInputs(TypedDict):
79 type: Literal["pixel_values"]
80 data: torch.Tensor
81 - """Shape:
81 + """Shape:
82 `(num_patches, num_channels * patch_size * patch_size)`
83 """
84
@@ -102,14 +102,14 @@ Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
102
103 class Qwen2VLVideoInputs(TypedDict):
104 pixel_values_videos: torch.Tensor
105 - """Shape:
106 - `(num_patches,
105 + """Shape:
106 + `(num_patches,
107 num_channels * temporal_patch_size * patch_size * patch_size)`
108 """
109
110 video_grid_thw: torch.Tensor
111 """Shape: `(num_videos, 3)`
112 -
112 +
113 This should be in `(grid_t, grid_h, grid_w)` format.
114 """
115
vllm/model_executor/models/utils.py
+3 -3
@@ -21,7 +21,7 @@ from vllm.model_executor.models import ModelRegistry
21 from vllm.multimodal.base import NestedTensors
22 from vllm.platforms import current_platform
23 from vllm.sequence import IntermediateTensors
24 -from vllm.utils import is_cpu, is_pin_memory_available
24 +from vllm.utils import is_pin_memory_available
25
26 logger = init_logger(__name__)
27
@@ -474,7 +474,7 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
474
475 class LLMWrapper(nn.Module):
476 """
477 - To align with the key names of LoRA trained with PEFT, we need to add an
477 + To align with the key names of LoRA trained with PEFT, we need to add an
478 additional layer to the llm's implementation.
479 """
480
@@ -515,7 +515,7 @@ def get_vit_attn_backend() -> _Backend:
515 "so we use xformers backend instead. You can run "
516 "`pip install flash-attn` to use flash-attention backend.")
517 selected_backend = _Backend.XFORMERS
518 - elif is_cpu():
518 + elif current_platform.is_cpu():
519 selected_backend = _Backend.TORCH_SDPA
520 else:
521 selected_backend = _Backend.XFORMERS
vllm/utils.py
+1 -10
@@ -318,15 +318,6 @@ def is_hip() -> bool:
318 return torch.version.hip is not None
319
320
321 -@lru_cache(maxsize=None)
322 -def is_cpu() -> bool:
323 - from importlib.metadata import PackageNotFoundError, version
324 - try:
325 - return "cpu" in version("vllm")
326 - except PackageNotFoundError:
327 - return False
328 -
329 -
321 @lru_cache(maxsize=None)
322 def is_openvino() -> bool:
323 from importlib.metadata import PackageNotFoundError, version
@@ -798,7 +789,7 @@ def is_pin_memory_available() -> bool:
789 elif is_neuron():
790 print_warning_once("Pin memory is not supported on Neuron.")
791 return False
801 - elif is_cpu() or is_openvino():
792 + elif current_platform.is_cpu() or is_openvino():
793 return False
794 return True
795