A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Hardware][CPU] using current_platform.is_cpu (#9536)
wangshuai09 committed
Oct 22, 2024 at 15:50 UTC
3ddbe25502fb8c49e67096ba6e641ecdc3519757
17 files changed
+60
-64
tests/conftest.py
+4
-2
@@ -32,9 +32,10 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
32
to_enc_dec_tuple_list, zip_enc_dec_prompts)
33
from vllm.logger import init_logger
34
from vllm.outputs import RequestOutput
35
+from vllm.platforms import current_platform
36
from vllm.sampling_params import BeamSearchParams
37
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
37
- identity, is_cpu)
38
+ identity)
39
40
logger = init_logger(__name__)
41
@@ -236,7 +237,8 @@ class HfRunner:
237
238
def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
239
if device is None:
239
- return self.wrap_device(input, "cpu" if is_cpu() else "cuda")
240
+ return self.wrap_device(
241
+ input, "cpu" if current_platform.is_cpu() else "cuda")
242
243
if hasattr(input, "device") and input.device.type == device:
244
return input
tests/encoder_decoder/test_e2e_correctness.py
+3
-3
@@ -7,8 +7,8 @@ from typing import List, Optional, Tuple
7
import pytest
8
from transformers import AutoModelForSeq2SeqLM
9
10
+from vllm.platforms import current_platform
11
from vllm.sequence import SampleLogprobs
11
-from vllm.utils import is_cpu
12
13
from ..conftest import DecoderPromptType
14
from ..models.utils import check_logprobs_close
@@ -35,7 +35,7 @@ def vllm_to_hf_output(
35
@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
36
@pytest.mark.parametrize("enforce_eager", [True, False])
37
@pytest.mark.skipif(
38
- is_cpu(),
38
+ current_platform.is_cpu(),
39
reason="CPU backend is not currently supported with encoder/decoder models"
40
)
41
def test_encoder_decoder_e2e(
@@ -50,7 +50,7 @@ def test_encoder_decoder_e2e(
50
enforce_eager: bool,
51
) -> None:
52
'''
53
- End-to-End (E2E) test for the encoder-decoder framework.
53
+ End-to-End (E2E) test for the encoder-decoder framework.
54
This test evaluates the encoder-decoder functionality using the BART
55
model. We compare the outputs of the Hugging Face and vLLM
56
implementations to ensure that both implementations produce consistent
tests/kernels/test_attention_selector.py
+2
-1
@@ -19,7 +19,8 @@ def test_env(name: str, device: str, monkeypatch):
19
override_backend_env_variable(monkeypatch, name)
20
21
if device == "cpu":
22
- with patch("vllm.attention.selector.is_cpu", return_value=True):
22
+ with patch("vllm.attention.selector.current_platform.is_cpu",
23
+ return_value=True):
24
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
25
False)
26
assert backend.name == "TORCH_SDPA"
tests/models/decoder_only/language/test_phimoe.py
+2
-2
@@ -5,7 +5,7 @@ Run `pytest tests/models/test_phimoe.py`.
5
import pytest
6
import torch
7
8
-from vllm.utils import is_cpu
8
+from vllm.platforms import current_platform
9
10
from ....utils import large_gpu_test
11
from ...utils import check_logprobs_close
@@ -70,7 +70,7 @@ def test_phimoe_routing_function():
70
assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
71
72
73
-@pytest.mark.skipif(condition=is_cpu(),
73
+@pytest.mark.skipif(condition=current_platform.is_cpu(),
74
reason="This test takes a lot time to run on CPU, "
75
"and vllm CI's disk space is not enough for this model.")
76
@large_gpu_test(min_gb=80)
tests/models/decoder_only/vision_language/test_fuyu.py
+3
-3
@@ -3,8 +3,8 @@ from typing import List, Optional, Tuple, Type
3
import pytest
4
5
from vllm.multimodal.utils import rescale_image_size
6
+from vllm.platforms import current_platform
7
from vllm.sequence import SampleLogprobs
7
-from vllm.utils import is_cpu
8
9
from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
from ...utils import check_logprobs_close
@@ -46,7 +46,7 @@ def run_test(
46
47
All the image fixtures for the test are from IMAGE_ASSETS.
48
For huggingface runner, we provide the PIL images as input.
49
- For vllm runner, we provide MultiModalDataDict objects
49
+ For vllm runner, we provide MultiModalDataDict objects
50
and corresponding MultiModalConfig as input.
51
Note, the text input is also adjusted to abide by vllm contract.
52
The text output is sanitized to be able to compare with hf.
@@ -103,7 +103,7 @@ def run_test(
103
104
105
target_dtype = "half"
106
-if is_cpu():
106
+if current_platform.is_cpu():
107
target_dtype = "bfloat16"
108
109
tests/models/decoder_only/vision_language/test_internvl.py
+3
-3
@@ -7,7 +7,7 @@ from PIL.Image import Image
7
from transformers import AutoConfig
8
9
from vllm.multimodal.utils import rescale_image_size
10
-from vllm.utils import is_cpu
10
+from vllm.platforms import current_platform
11
12
from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13
_ImageAssets)
@@ -78,7 +78,7 @@ def run_test(
78
79
All the image fixtures for the test are from IMAGE_ASSETS.
80
For huggingface runner, we provide the PIL images as input.
81
- For vllm runner, we provide MultiModalDataDict objects
81
+ For vllm runner, we provide MultiModalDataDict objects
82
and corresponding MultiModalConfig as input.
83
Note, the text input is also adjusted to abide by vllm contract.
84
The text output is sanitized to be able to compare with hf.
@@ -244,7 +244,7 @@ def run_awq_test(
244
245
246
target_dtype = "half"
247
-if is_cpu():
247
+if current_platform.is_cpu():
248
target_dtype = "bfloat16"
249
250
tests/models/decoder_only/vision_language/test_phi3v.py
+3
-2
@@ -10,8 +10,9 @@ from vllm.inputs import InputContext, token_inputs
10
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
11
from vllm.multimodal import MultiModalRegistry
12
from vllm.multimodal.utils import rescale_image_size
13
+from vllm.platforms import current_platform
14
from vllm.sequence import SampleLogprobs
14
-from vllm.utils import is_cpu, is_hip
15
+from vllm.utils import is_hip
16
17
from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
18
_ImageAssets)
@@ -49,7 +50,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
50
51
52
target_dtype = "half"
52
-if is_cpu():
53
+if current_platform.is_cpu():
54
target_dtype = "bfloat16"
55
56
# ROCm Triton FA can run into shared memory issues with these models,
tests/models/utils.py
+4
-4
@@ -5,8 +5,8 @@ import torch
5
6
from vllm.config import ModelConfig, TaskOption
7
from vllm.inputs import InputContext
8
+from vllm.platforms import current_platform
9
from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9
-from vllm.utils import is_cpu
10
11
TokensText = Tuple[List[int], str]
12
@@ -19,7 +19,7 @@ def check_outputs_equal(
19
name_1: str,
20
):
21
"""
22
- Compare the two sequences generated by different models,
22
+ Compare the two sequences generated by different models,
23
which should be equal.
24
"""
25
assert len(outputs_0_lst) == len(outputs_1_lst)
@@ -255,7 +255,7 @@ def build_model_context(model_name: str,
255
mm_processor_kwargs: Optional[Dict] = None,
256
limit_mm_per_prompt: Optional[Dict] = None):
257
"""Creates an InputContext for a given model.
258
-
258
+
259
Args:
260
model_name: Name of the model being considered.
261
tokenizer_name: Name of the tokenizer being considered.
@@ -270,7 +270,7 @@ def build_model_context(model_name: str,
270
if tokenizer_name is None:
271
tokenizer_name = model_name
272
if dtype is None:
273
- dtype = "bfloat16" if is_cpu() else "half"
273
+ dtype = "bfloat16" if current_platform.is_cpu() else "half"
274
275
model_config = ModelConfig(
276
model_name,
tests/worker/test_encoder_decoder_model_runner.py
+6
-5
@@ -5,8 +5,9 @@ import pytest
5
import torch
6
7
from vllm.engine.arg_utils import EngineArgs
8
+from vllm.platforms import current_platform
9
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
9
-from vllm.utils import is_cpu, make_tensor_with_pad
10
+from vllm.utils import make_tensor_with_pad
11
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12
from vllm.worker.model_runner import _get_graph_batch_size
13
@@ -31,7 +32,7 @@ def _create_model_runner(model: str, *args,
32
return model_runner
33
34
34
-@pytest.mark.skipif(condition=is_cpu(),
35
+@pytest.mark.skipif(condition=current_platform.is_cpu(),
36
reason="CPU backend is currently "
37
"unsupported for encoder/ "
38
"decoder models")
@@ -74,7 +75,7 @@ def test_empty_seq_group():
75
assert return_seq_lens is None
76
77
77
-@pytest.mark.skipif(condition=is_cpu(),
78
+@pytest.mark.skipif(condition=current_platform.is_cpu(),
79
reason="CPU backend is currently "
80
"unsupported for encoder/ "
81
"decoder models")
@@ -264,7 +265,7 @@ def test_prepare_prompt(batch_size):
265
assert torch.equal(actual, expected)
266
267
267
-@pytest.mark.skipif(condition=is_cpu(),
268
+@pytest.mark.skipif(condition=current_platform.is_cpu(),
269
reason="CPU backend is currently "
270
"unsupported for encoder/ "
271
"decoder models")
@@ -490,7 +491,7 @@ def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
491
def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
492
"""
493
Tests that for encoder-decoder models with CUDA Graph capture and replay
493
- enabled, the tensors used during the decode phase are correctly padded
494
+ enabled, the tensors used during the decode phase are correctly padded
495
for varying input batch sizes.
496
"""
497
model_runner = _create_model_runner(
vllm/attention/backends/torch_sdpa.py
+4
-4
@@ -10,9 +10,9 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata, AttentionType)
11
from vllm.attention.backends.utils import CommonAttentionState
12
from vllm.attention.ops.paged_attn import PagedAttentionMetadata
13
-from vllm.utils import is_cpu
13
+from vllm.platforms import current_platform
14
15
-if is_cpu():
15
+if current_platform.is_cpu():
16
try:
17
from vllm.attention.ops.ipex_attn import PagedAttention
18
except ImportError:
@@ -234,10 +234,10 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
234
on the type of attention operation.
235
236
Decoder attn -> select entirely decoder self-attention-related fields
237
- Encoder/decoder cross-attn -> select encoder sequence lengths &
237
+ Encoder/decoder cross-attn -> select encoder sequence lengths &
238
cross-attn block-tables fields
239
Encoder attn -> select encoder sequence lengths fields & no block tables
240
-
240
+
241
Arguments:
242
243
* attn_metadata: Attention metadata structure associated with attention
vllm/attention/ops/blocksparse_attention/interface.py
+10
-10
@@ -3,7 +3,7 @@ import math
3
import torch
4
5
from vllm.platforms import current_platform
6
-from vllm.utils import is_cpu, is_hip
6
+from vllm.utils import is_hip
7
8
from .utils import (dense_to_crow_col, get_head_sliding_step,
9
get_sparse_attn_mask)
@@ -32,7 +32,7 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
32
):
33
super().__init__()
34
if use_spda is None:
35
- use_spda = is_hip() or is_cpu() or not \
35
+ use_spda = is_hip() or current_platform.is_cpu() or not \
36
IS_COMPUTE_8_OR_ABOVE
37
device = device or (torch.cuda.current_device()
38
if current_platform.is_cuda_alike() else "cpu")
@@ -109,13 +109,13 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
109
q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
110
Support grouped attention, with `q[:, i*r:(i*r + r)]`
111
is correspondent to `k[:, i]`, where `r` is the q/k ratio.
112
- cu_seqlens_k: shape=(batch_size + 1,),
113
- indicating segment of samples,
112
+ cu_seqlens_k: shape=(batch_size + 1,),
113
+ indicating segment of samples,
114
e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i
115
cu_seqlens_q: shape=(batch_size + 1, ).
116
Default None: same as cu_seqlens_k for prefilling or
117
[0, 1, .., batch_size] for decoding.
118
- The only case you need to specify is when q is a mix of
118
+ The only case you need to specify is when q is a mix of
119
prefilling and decoding.
120
sm_scale: softmax scale, default to 1/sqrt(head_size).
121
@@ -171,7 +171,7 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
171
172
def spda(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
173
"""For CPU, V100 or other older GPUs.
174
- NOTE: torch SPDA supports nested tensor,
174
+ NOTE: torch SPDA supports nested tensor,
175
but seems extremely slow. Choose to pad instead.
176
"""
177
assert (cu_seqlens_q is None or
@@ -201,8 +201,8 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
201
return self.transpose_and_unpad(spda_output, cu_seqlens)
202
203
def forward(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
204
- """Dispatch to `varlen_attn` (Ampere or newer) or
205
- `self.spda`(cpu, Volta, Turing or older)based on
204
+ """Dispatch to `varlen_attn` (Ampere or newer) or
205
+ `self.spda`(cpu, Volta, Turing or older)based on
206
the type of device used and cuda compute capability.
207
208
q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
@@ -213,8 +213,8 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
213
cu_seqlens_q: shape=(batch_size + 1, ).
214
Default None: same as cu_seqlens_k for prefilling or
215
[0, 1, .., batch_size] for decoding.
216
- The only case you need to specify
217
- is when q is a mix of prefilling
216
+ The only case you need to specify
217
+ is when q is a mix of prefilling
218
and decoding.
219
sm_scale: softmax scale, default to 1/sqrt(head_size).
220
vllm/attention/selector.py
+3
-3
@@ -10,7 +10,7 @@ import vllm.envs as envs
10
from vllm.attention.backends.abstract import AttentionBackend
11
from vllm.logger import init_logger
12
from vllm.platforms import current_platform
13
-from vllm.utils import STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino, is_xpu
13
+from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino, is_xpu
14
15
logger = init_logger(__name__)
16
@@ -121,7 +121,7 @@ def get_attn_backend(
121
ROCmFlashAttentionBackend)
122
return ROCmFlashAttentionBackend
123
elif backend == _Backend.TORCH_SDPA:
124
- assert is_cpu(), RuntimeError(
124
+ assert current_platform.is_cpu(), RuntimeError(
125
"Torch SDPA backend is only used for the CPU device.")
126
logger.info("Using Torch SDPA backend.")
127
from vllm.attention.backends.torch_sdpa import TorchSDPABackend
@@ -183,7 +183,7 @@ def which_attn_to_use(
183
if backend_by_env_var is not None:
184
selected_backend = backend_name_to_enum(backend_by_env_var)
185
186
- if is_cpu():
186
+ if current_platform.is_cpu():
187
if selected_backend != _Backend.TORCH_SDPA:
188
logger.info("Cannot use %s backend on CPU.", selected_backend)
189
return _Backend.TORCH_SDPA
vllm/distributed/parallel_state.py
+3
-3
@@ -7,7 +7,7 @@ It takes over the control of the distributed environment from PyTorch.
7
The typical workflow is:
8
9
- call `init_distributed_environment` to initialize the distributed environment.
10
-- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
10
+- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
11
initialize the model parallel groups.
12
13
- any code dealing with the distributed stuff
@@ -37,7 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37
import vllm.envs as envs
38
from vllm.logger import init_logger
39
from vllm.platforms import current_platform
40
-from vllm.utils import is_cpu, supports_custom_op
40
+from vllm.utils import supports_custom_op
41
42
43
@dataclass
@@ -1139,7 +1139,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1139
import ray # Lazy import Ray
1140
ray.shutdown()
1141
gc.collect()
1142
- if not is_cpu():
1142
+ if not current_platform.is_cpu():
1143
torch.cuda.empty_cache()
1144
1145
vllm/model_executor/custom_op.py
+2
-2
@@ -7,7 +7,7 @@ import vllm.envs as envs
7
from vllm.compilation.levels import CompilationLevel
8
from vllm.logger import init_logger
9
from vllm.platforms import current_platform
10
-from vllm.utils import is_cpu, is_hip, is_xpu, print_warning_once
10
+from vllm.utils import is_hip, is_xpu, print_warning_once
11
12
logger = init_logger(__name__)
13
@@ -74,7 +74,7 @@ class CustomOp(nn.Module):
74
75
if is_hip():
76
return self.forward_hip
77
- elif is_cpu():
77
+ elif current_platform.is_cpu():
78
return self.forward_cpu
79
elif current_platform.is_tpu():
80
return self.forward_tpu
vllm/model_executor/models/qwen2_vl.py
+4
-4
@@ -78,7 +78,7 @@ logger = init_logger(__name__)
78
class Qwen2VLImagePixelInputs(TypedDict):
79
type: Literal["pixel_values"]
80
data: torch.Tensor
81
- """Shape:
81
+ """Shape:
82
`(num_patches, num_channels * patch_size * patch_size)`
83
"""
84
@@ -102,14 +102,14 @@ Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
102
103
class Qwen2VLVideoInputs(TypedDict):
104
pixel_values_videos: torch.Tensor
105
- """Shape:
106
- `(num_patches,
105
+ """Shape:
106
+ `(num_patches,
107
num_channels * temporal_patch_size * patch_size * patch_size)`
108
"""
109
110
video_grid_thw: torch.Tensor
111
"""Shape: `(num_videos, 3)`
112
-
112
+
113
This should be in `(grid_t, grid_h, grid_w)` format.
114
"""
115
vllm/model_executor/models/utils.py
+3
-3
@@ -21,7 +21,7 @@ from vllm.model_executor.models import ModelRegistry
21
from vllm.multimodal.base import NestedTensors
22
from vllm.platforms import current_platform
23
from vllm.sequence import IntermediateTensors
24
-from vllm.utils import is_cpu, is_pin_memory_available
24
+from vllm.utils import is_pin_memory_available
25
26
logger = init_logger(__name__)
27
@@ -474,7 +474,7 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
474
475
class LLMWrapper(nn.Module):
476
"""
477
- To align with the key names of LoRA trained with PEFT, we need to add an
477
+ To align with the key names of LoRA trained with PEFT, we need to add an
478
additional layer to the llm's implementation.
479
"""
480
@@ -515,7 +515,7 @@ def get_vit_attn_backend() -> _Backend:
515
"so we use xformers backend instead. You can run "
516
"`pip install flash-attn` to use flash-attention backend.")
517
selected_backend = _Backend.XFORMERS
518
- elif is_cpu():
518
+ elif current_platform.is_cpu():
519
selected_backend = _Backend.TORCH_SDPA
520
else:
521
selected_backend = _Backend.XFORMERS
vllm/utils.py
+1
-10
@@ -318,15 +318,6 @@ def is_hip() -> bool:
318
return torch.version.hip is not None
319
320
321
-@lru_cache(maxsize=None)
322
-def is_cpu() -> bool:
323
- from importlib.metadata import PackageNotFoundError, version
324
- try:
325
- return "cpu" in version("vllm")
326
- except PackageNotFoundError:
327
- return False
328
-
329
-
321
@lru_cache(maxsize=None)
322
def is_openvino() -> bool:
323
from importlib.metadata import PackageNotFoundError, version
@@ -798,7 +789,7 @@ def is_pin_memory_available() -> bool:
789
elif is_neuron():
790
print_warning_once("Pin memory is not supported on Neuron.")
791
return False
801
- elif is_cpu() or is_openvino():
792
+ elif current_platform.is_cpu() or is_openvino():
793
return False
794
return True
795