@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4e2d95e372
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Hardware][ROCM] using current_platform.is_rocm (#9642)

Signed-off-by: wangshuai09 <391746016@qq.com>

wangshuai09 committed Oct 28, 2024 at 12:07 UTC 4e2d95e372ad5fbef7b27c66d527c37477c0c8bb
32 files changed +162 -148
tests/basic_correctness/test_basic_correctness.py
+2 -2
@@ -11,7 +11,7 @@ from unittest.mock import patch
11 import pytest
12
13 from vllm import LLM
14 -from vllm.utils import is_hip
14 +from vllm.platforms import current_platform
15 from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
16
17 from ..models.utils import check_outputs_equal
@@ -51,7 +51,7 @@ def test_models(
51 enforce_eager: bool,
52 ) -> None:
53
54 - if backend == "FLASHINFER" and is_hip():
54 + if backend == "FLASHINFER" and current_platform.is_rocm():
55 pytest.skip("Flashinfer does not support ROCm/HIP.")
56
57 os.environ["VLLM_ATTENTION_BACKEND"] = backend
tests/compile/utils.py
+2 -2
@@ -5,7 +5,7 @@ import torch
5 from tests.quantization.utils import is_quant_method_supported
6 from vllm import LLM, SamplingParams
7 from vllm.compilation.levels import CompilationLevel
8 -from vllm.utils import is_hip
8 +from vllm.platforms import current_platform
9
10 TEST_MODELS = [
11 ("facebook/opt-125m", {}),
@@ -55,7 +55,7 @@ if is_quant_method_supported("marlin"):
55 "quantization": "marlin"
56 }))
57
58 -if not is_hip() and is_quant_method_supported("awq"):
58 +if not current_platform.is_rocm() and is_quant_method_supported("awq"):
59 TEST_MODELS.append(("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {
60 "quantization": "AWQ"
61 }))
tests/kernels/quant_utils.py
+11 -6
@@ -2,12 +2,13 @@ from typing import Optional, Tuple, Union
2
3 import torch
4
5 -from vllm.utils import is_hip
5 +from vllm.platforms import current_platform
6
7 # Using the default value (240.0) from pytorch will cause accuracy
8 # issue on dynamic quantization models. Here use 224.0 for rocm.
9 ROCM_FP8_MAX = 224.0
10 -FP8_DTYPE = torch.float8_e4m3fnuz if is_hip() else torch.float8_e4m3fn
10 +FP8_DTYPE = torch.float8_e4m3fnuz if current_platform.is_rocm() \
11 + else torch.float8_e4m3fn
12
13
14 def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
@@ -24,8 +25,10 @@ def ref_dynamic_per_token_quant(x: torch.tensor,
25
26 qtype_traits = torch.iinfo(quant_dtype) if quant_dtype == torch.int8 \
27 else torch.finfo(quant_dtype)
27 - qtype_traits_max = ROCM_FP8_MAX if is_hip() else qtype_traits.max
28 - qtype_traits_min = -ROCM_FP8_MAX if is_hip() else qtype_traits.min
28 + qtype_traits_max = ROCM_FP8_MAX if current_platform.is_rocm() \
29 + else qtype_traits.max
30 + qtype_traits_min = -ROCM_FP8_MAX if current_platform.is_rocm() \
31 + else qtype_traits.min
32 qtype_max = as_float32_tensor(qtype_traits_max)
33 s_1 = as_float32_tensor(1.0)
34 s_512 = as_float32_tensor(512.0)
@@ -66,8 +69,10 @@ def ref_dynamic_per_tensor_fp8_quant(x: torch.tensor) \
69 -> Tuple[torch.tensor, torch.tensor]:
70
71 fp8_traits = torch.finfo(FP8_DTYPE)
69 - fp8_traits_max = ROCM_FP8_MAX if is_hip() else fp8_traits.max
70 - fp8_traits_min = -ROCM_FP8_MAX if is_hip() else fp8_traits.min
72 + fp8_traits_max = ROCM_FP8_MAX if current_platform.is_rocm() \
73 + else fp8_traits.max
74 + fp8_traits_min = -ROCM_FP8_MAX if current_platform.is_rocm() \
75 + else fp8_traits.min
76 fp8_max = as_float32_tensor(fp8_traits_max)
77 one = as_float32_tensor(1.0)
78
tests/kernels/test_attention.py
+13 -10
@@ -6,11 +6,12 @@ import torch
6
7 from tests.kernels.utils import opcheck
8 from vllm import _custom_ops as ops
9 -from vllm.utils import get_max_shared_memory_bytes, is_hip, seed_everything
9 +from vllm.platforms import current_platform
10 +from vllm.utils import get_max_shared_memory_bytes, seed_everything
11
12 from .allclose_default import get_default_atol, get_default_rtol
13
13 -if not is_hip():
14 +if not current_platform.is_rocm():
15 from xformers import ops as xops
16 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
17
@@ -23,8 +24,9 @@ MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
24 NUM_BLOCKS = 4321 # Arbitrary values for testing
25 PARTITION_SIZE = 512
26 # flshattF and tritonflashattF supported: {torch.float16, torch.bfloat16}
26 -DTYPES = [torch.half, torch.bfloat16, torch.float
27 - ] if not is_hip() else [torch.half, torch.bfloat16]
27 +DTYPES = [
28 + torch.half, torch.bfloat16, torch.float
29 +] if not current_platform.is_rocm() else [torch.half, torch.bfloat16]
30 NUM_GEN_SEQS = [7] # Arbitrary values for testing
31 NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
32 NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
@@ -114,7 +116,8 @@ def ref_single_query_cached_kv_attention(
116
117
118 @pytest.mark.parametrize(
117 - "version", ["v1", "v2"] if not is_hip() else ["v1", "v2", "rocm"])
119 + "version",
120 + ["v1", "v2"] if not current_platform.is_rocm() else ["v1", "v2", "rocm"])
121 @pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
122 @pytest.mark.parametrize("num_heads", NUM_HEADS)
123 @pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -317,8 +320,8 @@ def test_paged_attention(
320 # NOTE(woosuk): Due to the kernel-level differences in the two
321 # implementations, there is a small numerical difference in the two
322 # outputs. Thus, we use a relaxed tolerance for the test.
320 - atol = get_default_atol(output) if is_hip() else 1e-3
321 - rtol = get_default_rtol(output) if is_hip() else 1e-5
323 + atol = get_default_atol(output) if current_platform.is_rocm() else 1e-3
324 + rtol = get_default_rtol(output) if current_platform.is_rocm() else 1e-5
325
326 # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
327 # so we use a relaxed tolerance for the test.
@@ -368,7 +371,7 @@ def ref_multi_query_kv_attention(
371 @pytest.mark.parametrize("dtype", DTYPES)
372 @pytest.mark.parametrize("seed", SEEDS)
373 @pytest.mark.parametrize("device", CUDA_DEVICES)
371 -@pytest.mark.skipif(is_hip(),
374 +@pytest.mark.skipif(current_platform.is_rocm(),
375 reason="Xformers backend is not supported on ROCm.")
376 @torch.inference_mode()
377 def test_multi_query_kv_attention(
@@ -425,6 +428,6 @@ def test_multi_query_kv_attention(
428 scale,
429 dtype,
430 )
428 - atol = get_default_atol(output) if is_hip() else 1e-3
429 - rtol = get_default_rtol(output) if is_hip() else 1e-5
431 + atol = get_default_atol(output) if current_platform.is_rocm() else 1e-3
432 + rtol = get_default_rtol(output) if current_platform.is_rocm() else 1e-5
433 torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
tests/kernels/test_attention_selector.py
+2 -1
@@ -25,7 +25,8 @@ def test_env(name: str, device: str, monkeypatch):
25 False)
26 assert backend.name == "TORCH_SDPA"
27 elif device == "hip":
28 - with patch("vllm.attention.selector.is_hip", return_value=True):
28 + with patch("vllm.attention.selector.current_platform.is_rocm",
29 + return_value=True):
30 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
31 False)
32 assert backend.name == "ROCM_FLASH"
tests/kernels/test_blocksparse_attention.py
+4 -3
@@ -7,7 +7,8 @@ import torch
7 from vllm import _custom_ops as ops
8 from vllm.attention.ops.blocksparse_attention.interface import (
9 LocalStridedBlockSparseAttn)
10 -from vllm.utils import get_max_shared_memory_bytes, is_hip, seed_everything
10 +from vllm.platforms import current_platform
11 +from vllm.utils import get_max_shared_memory_bytes, seed_everything
12
13 from .allclose_default import get_default_atol, get_default_rtol
14
@@ -316,8 +317,8 @@ def test_paged_attention(
317 # NOTE(woosuk): Due to the kernel-level differences in the two
318 # implementations, there is a small numerical difference in the two
319 # outputs. Thus, we use a relaxed tolerance for the test.
319 - atol = get_default_atol(output) if is_hip() else 1e-3
320 - rtol = get_default_rtol(output) if is_hip() else 1e-5
320 + atol = get_default_atol(output) if current_platform.is_rocm() else 1e-3
321 + rtol = get_default_rtol(output) if current_platform.is_rocm() else 1e-5
322
323 # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
324 # so we use a relaxed tolerance for the test.
tests/kernels/test_encoder_decoder_attn.py
+40 -36
@@ -18,7 +18,7 @@ from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
18 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19 from vllm.attention.selector import (_Backend,
20 global_force_attn_backend_context_manager)
21 -from vllm.utils import is_hip
21 +from vllm.platforms import current_platform
22
23 # List of support backends for encoder/decoder models
24 LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS]
@@ -82,7 +82,7 @@ class TestResources(NamedTuple):
82 will leverage attn_backend for the purpose of
83 constructing backend-compatible attention
84 metadata instances
85 -
85 +
86 Attributes:
87
88 * scale: 1/sqrt(d) scale factor for attn
@@ -105,10 +105,10 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
105 Build key components for performing encoder/decoder attention test.
106
107 Note that
108 - (1) The Attention instance constructed here, automatically selects
108 + (1) The Attention instance constructed here, automatically selects
109 an attention backend class based on platform info & a set of canned
110 heuristics, so
111 - (2) The attention backend instance constructed here is thus *not
111 + (2) The attention backend instance constructed here is thus *not
112 the same backend instance* used by attn, but rather it is
113 intended to be a *different instance* of the *same backend class*;
114 therefore,
@@ -156,7 +156,7 @@ def _encoder_attn_setup(
156 '''
157 Set up test vectors & data structures for encoder attention test.
158
159 - A triplet of synthetic query/key/value tensors are constructed.
159 + A triplet of synthetic query/key/value tensors are constructed.
160 Given this is an encoder attention test, the key & value
161 sequences will have the same length as the corresponding queries.
162
@@ -169,14 +169,14 @@ def _encoder_attn_setup(
169 Arguments:
170
171 * test_pt: TestPoint data structure; this function relies on the
172 - following fields: batch_size, num_heads, head_size,
172 + following fields: batch_size, num_heads, head_size,
173 block_size, max_q_seq_len
174 * test_rsrcs: TestResources data structure; this function relies on the
175 scale field
176
177 -
177 +
178 Returns:
179 -
179 +
180 * PhaseTestParameters data structure comprising (1) packed query/key/value
181 tensors, (2) the ideal output of attention computed using a naive
182 implementation, and (3) KVCache field set to None
@@ -265,7 +265,7 @@ def _decoder_attn_setup(
265 Arguments:
266
267 * test_pt: TestPoint data structure; this function relies on the
268 - following fields: batch_size, num_heads, head_size,
268 + following fields: batch_size, num_heads, head_size,
269 block_size, max_q_seq_len
270 * test_rsrcs: TestResources data structure; this function relies on the
271 scale field
@@ -275,14 +275,14 @@ def _decoder_attn_setup(
275 * qkv: Unpacked (batch_size x padded_seq_len x num_heads x
276 head_size) query/key/value tensors
277 * Prefill-phase decoder self-attention PhaseTestParameters data structure,
278 - including (1) packed (number_of_tokens x num_heads x head_size)
278 + including (1) packed (number_of_tokens x num_heads x head_size)
279 query/key/value tensors along with (2) ideal attention output
280 - computed using a naive implementation, and (3) memory-mapping data
280 + computed using a naive implementation, and (3) memory-mapping data
281 structures appropriate for prefill phase.
282 - * Decode-phase decoder self-attention PhaseTestParameters data structure,
283 - including (1) packed (number_of_tokens x num_heads x head_size)
284 - query/key/value tensors along with (2) ideal attention output
285 - computed using a naive implementation, and (3) memory-mapping data
282 + * Decode-phase decoder self-attention PhaseTestParameters data structure,
283 + including (1) packed (number_of_tokens x num_heads x head_size)
284 + query/key/value tensors along with (2) ideal attention output
285 + computed using a naive implementation, and (3) memory-mapping data
286 structures appropriate for decode phase.
287 * max_block_idx: max physical address in decoder self-attention block-table
288 (intended to be used as the base address for the encoder/
@@ -436,12 +436,12 @@ def _enc_dec_cross_attn_setup_reuses_query(
436
437 This function also constructs the cross-attention KV cache memory mapping
438 (slot mapping and block table), ensuring that the block table starts at
439 - block_base_addr.
439 + block_base_addr.
440
441 Arguments:
442
443 * decoder_qkv: pre-existing unpacked (batch_size x padded_seq_len x
444 - num_heads x head_size) decoder self-attention inputs;
444 + num_heads x head_size) decoder self-attention inputs;
445 this function relies on the query and q_seq_lens
446 fields
447 * encoder_test_params: PhaseTestParameters data structure which was
@@ -452,7 +452,7 @@ def _enc_dec_cross_attn_setup_reuses_query(
452 self-attention; all fields
453 including KV cache required
454 * test_pt: TestPoint data structure; this function relies on the
455 - following fields: batch_size, num_heads, head_size,
455 + following fields: batch_size, num_heads, head_size,
456 block_size, max_q_seq_len
457 * test_rsrcs: TestResources data structure; this function relies on the
458 scale field
@@ -460,16 +460,16 @@ def _enc_dec_cross_attn_setup_reuses_query(
460
461 Returns:
462
463 - * Prefill-phase encoder/decoder cross-attention PhaseTestParameters data
464 - structure, including (1) packed
463 + * Prefill-phase encoder/decoder cross-attention PhaseTestParameters data
464 + structure, including (1) packed
465 (number_of_tokens x num_heads x head_size) query/key/value tensors
466 - along with (2) ideal attention output computed using a
466 + along with (2) ideal attention output computed using a
467 naive implementation, and (3) memory-mapping data structures appropriate
468 for prefill phase.
469 - * Decode-phase encoder/decoder cross-attention PhaseTestParameters data
469 + * Decode-phase encoder/decoder cross-attention PhaseTestParameters data
470 structure, including (1) packed
471 (number_of_tokens x num_heads x head_size) query/key/value tensors
472 - along with (2) ideal attention output computed using a
472 + along with (2) ideal attention output computed using a
473 naive implementation, and (3) memory-mapping data structures appropriate
474 for decode phase.
475 '''
@@ -596,7 +596,7 @@ def _run_encoder_attention_test(
596 '''
597 Run encoder attention.
598
599 - attn.forward() is passed attn_type=AttentionType.ENCODER in order
599 + attn.forward() is passed attn_type=AttentionType.ENCODER in order
600 to configure the kernel invocation for encoder attention
601
602 Requires attn_metadata.num_decode_tokens == 0
@@ -607,7 +607,7 @@ def _run_encoder_attention_test(
607 * attn: Attention wrapper instance
608 * encoder_test_params: encoder PhaseTestParameters data structure;
609 this function relies on the packed
610 - (number_of_tokens x num_heads x head_size)
610 + (number_of_tokens x num_heads x head_size)
611 query/key/value fields
612 * attn_metadata: attention metadata for encoder/decoder-self attention
613
@@ -646,7 +646,7 @@ def _run_decoder_self_attention_test(
646 and attn (Attention wrapper instance) fields
647 * decoder_test_params: decoder PhaseTestParameters data structure;
648 this function relies on the packed
649 - (number_of_tokens x num_heads x head_size)
649 + (number_of_tokens x num_heads x head_size)
650 query/key/value fields
651 * attn_metadata: attention metadata for decoder-self attention
652 (contains KV cache memory-mapping)
@@ -694,11 +694,11 @@ def _run_encoder_decoder_cross_attention_test(
694 and attn (Attention wrapper instance) fields
695 * decoder_test_params: decoder PhaseTestParameters data structure;
696 this function relies on the packed
697 - (number_of_tokens x num_heads x head_size)
697 + (number_of_tokens x num_heads x head_size)
698 query field
699 * cross_test_params: encoder/decoder PhaseTestParameters data structure;
700 this function relies on the packed
701 - (number_of_tokens x num_heads x head_size)
701 + (number_of_tokens x num_heads x head_size)
702 key/value fields
703 * attn_metadata: attention metadata for encoder/decoder-self attention
704
@@ -726,7 +726,8 @@ def _run_encoder_decoder_cross_attention_test(
726 attn_type=attn_type)
727
728
729 -@pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
729 +@pytest.mark.skipif(current_platform.is_rocm(),
730 + reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
731 @pytest.mark.parametrize("num_heads", NUM_HEADS)
732 @pytest.mark.parametrize("head_size", HEAD_SIZES)
733 @pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
@@ -755,7 +756,8 @@ def test_encoder_only(
756 No KV cache is required for encoder-only attention.
757
758 Note on ROCm/HIP: currently encoder/decoder models are not supported on
758 - AMD GPUs, therefore this test simply is skipped if is_hip().
759 + AMD GPUs, therefore this test simply is skipped if
760 + current_platform.is_rocm().
761
762 This test globally forces an override of the usual backend
763 auto-selection process, forcing the specific backend-under-test
@@ -811,7 +813,8 @@ def test_encoder_only(
813 assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
814
815
814 -@pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
816 +@pytest.mark.skipif(current_platform.is_rocm(),
817 + reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
818 @pytest.mark.parametrize("num_heads", NUM_HEADS)
819 @pytest.mark.parametrize("head_size", HEAD_SIZES)
820 @pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
@@ -837,14 +840,14 @@ def test_e2e_enc_dec_attn(
840 attributes for prefill-phase, and (2) an analogous attention metadata
841 structure but for decode-phase
842 * Test attention steps in the following order
840 -
843 +
844 * Encoder attention
845 * Prefill self-attention
846 * Prefill cross-attention
847 * Decode self-attention
848 * Decode cross-attention
846 - * Besides being reflective of realistic use-cases, this order would
847 - exacerbate any accidental overlap in the self-/cross-attention
849 + * Besides being reflective of realistic use-cases, this order would
850 + exacerbate any accidental overlap in the self-/cross-attention
851 block tables, which one hopes to avoid
852
853
@@ -864,10 +867,11 @@ def test_e2e_enc_dec_attn(
867 to be utilized.
868
869 Note on ROCm/HIP: currently encoder/decoder models are not supported on
867 - AMD GPUs, therefore this test simply is skipped if is_hip().
870 + AMD GPUs, therefore this test simply is skipped if
871 + current_platform.is_rocm().
872
873 Note on metadata: there is a single attention metadata structure shared by
870 - all prefill-phase attention operations (encoder, decoder, enc/dec cross),
874 + all prefill-phase attention operations (encoder, decoder, enc/dec cross),
875 and a single one shared by all decode-phase attention operations
876 (decoder & enc/dec cross.) This is intended to reflect the behavior
877 of EncoderDecoderModelRunner, which constructs a single attention metadata
tests/kernels/test_moe.py
+4 -3
@@ -18,8 +18,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
18 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
19 marlin_quantize)
20 from vllm.model_executor.models.mixtral import MixtralMoE
21 +from vllm.platforms import current_platform
22 from vllm.scalar_type import scalar_types
22 -from vllm.utils import is_hip, seed_everything
23 +from vllm.utils import seed_everything
24
25
26 @pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
@@ -103,7 +104,7 @@ def test_mixtral_moe(dtype: torch.dtype):
104 @pytest.mark.parametrize("act_order", [True, False])
105 @pytest.mark.parametrize("num_bits", [4, 8])
106 @pytest.mark.parametrize("is_k_full", [True, False])
106 -@pytest.mark.skipif(is_hip(), reason="Skip for rocm")
107 +@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
108 def test_fused_marlin_moe(
109 m: int,
110 n: int,
@@ -256,7 +257,7 @@ def test_fused_marlin_moe(
257 @pytest.mark.parametrize("act_order", [True, False])
258 @pytest.mark.parametrize("num_bits", [4, 8])
259 @pytest.mark.parametrize("is_k_full", [True, False])
259 -@pytest.mark.skipif(is_hip(), reason="Skip for rocm")
260 +@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
261 def test_single_marlin_moe_multiply(
262 m: int,
263 n: int,
tests/lora/test_gemma.py
+3 -2
@@ -4,7 +4,7 @@ import pytest
4
5 import vllm
6 from vllm.lora.request import LoRARequest
7 -from vllm.utils import is_hip
7 +from vllm.platforms import current_platform
8
9 MODEL_PATH = "google/gemma-7b"
10
@@ -31,7 +31,8 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
31 return generated_texts
32
33
34 -@pytest.mark.xfail(is_hip(), reason="There can be output mismatch on ROCm")
34 +@pytest.mark.xfail(current_platform.is_rocm(),
35 + reason="There can be output mismatch on ROCm")
36 def test_gemma_lora(gemma_lora_files):
37 llm = vllm.LLM(MODEL_PATH,
38 max_model_len=1024,
tests/lora/test_quant_model.py
+2 -2
@@ -8,7 +8,7 @@ import pytest
8 import vllm
9 from vllm.distributed import cleanup_dist_env_and_memory
10 from vllm.lora.request import LoRARequest
11 -from vllm.utils import is_hip
11 +from vllm.platforms import current_platform
12
13
14 @dataclass
@@ -19,7 +19,7 @@ class ModelWithQuantization:
19
20 MODELS: List[ModelWithQuantization]
21 #AWQ quantization is currently not supported in ROCm.
22 -if is_hip():
22 +if current_platform.is_rocm():
23 MODELS = [
24 ModelWithQuantization(
25 model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
tests/models/decoder_only/vision_language/test_paligemma.py
+5 -4
@@ -6,8 +6,9 @@ from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6 BatchEncoding)
7
8 from vllm.multimodal.utils import rescale_image_size
9 +from vllm.platforms import current_platform
10 from vllm.sequence import SampleLogprobs
10 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_hip
11 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
12
13 from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
14 from ...utils import check_logprobs_close
@@ -24,7 +25,7 @@ models = ["google/paligemma-3b-mix-224"]
25 # ROCm Triton FA can run into compilation issues with these models due to,
26 # excessive use of shared memory. Use other backends in the meantime.
27 # FIXME (mattwong, gshtrasb, hongxiayan)
27 -if is_hip():
28 +if current_platform.is_rocm():
29 os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
30
31
@@ -70,7 +71,7 @@ def run_test(
71
72 All the image fixtures for the test are from IMAGE_ASSETS.
73 For huggingface runner, we provide the PIL images as input.
73 - For vllm runner, we provide MultiModalDataDict objects
74 + For vllm runner, we provide MultiModalDataDict objects
75 and corresponding MultiModalConfig as input.
76 Note, the text input is also adjusted to abide by vllm contract.
77 The text output is sanitized to be able to compare with hf.
@@ -151,7 +152,7 @@ def run_test(
152 pytest.param(
153 "float",
154 marks=pytest.mark.skipif(
154 - is_hip(),
155 + current_platform.is_rocm(),
156 reason=
157 "ROCm FA does not yet fully support 32-bit precision on PaliGemma")
158 ), "half"
tests/models/decoder_only/vision_language/test_phi3v.py
+1 -2
@@ -12,7 +12,6 @@ from vllm.multimodal import MultiModalRegistry
12 from vllm.multimodal.utils import rescale_image_size
13 from vllm.platforms import current_platform
14 from vllm.sequence import SampleLogprobs
15 -from vllm.utils import is_hip
15
16 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
17 _ImageAssets)
@@ -56,7 +55,7 @@ if current_platform.is_cpu():
55 # ROCm Triton FA can run into shared memory issues with these models,
56 # use other backends in the meantime
57 # FIXME (mattwong, gshtrasb, hongxiayan)
59 -if is_hip():
58 +if current_platform.is_rocm():
59 os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
60
61
tests/spec_decode/e2e/test_integration_dist_tp2.py
+2 -2
@@ -5,7 +5,7 @@ tensor parallelism.
5 import pytest
6 import torch
7
8 -from vllm.utils import is_hip
8 +from vllm.platforms import current_platform
9
10 from .conftest import run_equality_correctness_test_tp
11
@@ -51,7 +51,7 @@ def test_target_model_tp_gt_1(common_llm_kwargs, per_test_common_llm_kwargs,
51 batch_size: int, output_len: int, seed: int):
52 """Verify greedy equality when tensor parallelism is used.
53 """
54 - if is_hip():
54 + if current_platform.is_rocm():
55 pytest.skip("hip is not well-supported yet")
56 run_equality_correctness_test_tp("JackFram/llama-68m",
57 common_llm_kwargs,
tests/utils.py
+2 -2
@@ -26,7 +26,7 @@ from vllm.model_executor.model_loader.loader import get_model_loader
26 from vllm.platforms import current_platform
27 from vllm.transformers_utils.tokenizer import get_tokenizer
28 from vllm.utils import (FlexibleArgumentParser, GB_bytes,
29 - cuda_device_count_stateless, get_open_port, is_hip)
29 + cuda_device_count_stateless, get_open_port)
30
31 if current_platform.is_rocm():
32 from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -487,7 +487,7 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
487 output: Dict[int, str] = {}
488 output_raw: Dict[int, float] = {}
489 for device in devices:
490 - if is_hip():
490 + if current_platform.is_rocm():
491 dev_handle = amdsmi_get_processor_handles()[device]
492 mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
493 gb_used = mem_info["vram_used"] / 2**10
vllm/_custom_ops.py
+4 -4
@@ -659,11 +659,11 @@ def scaled_fp8_quant(
659 Args:
660 input: The input tensor to be quantized to FP8
661 scale: Optional scaling factor for the FP8 quantization
662 - scale_ub: Optional upper bound for scaling factor in dynamic
662 + scale_ub: Optional upper bound for scaling factor in dynamic
663 per token case
664 num_token_padding: If specified, pad the first dimension
665 of the output to at least this value.
666 - use_per_token_if_dynamic: Whether to do per_tensor or per_token
666 + use_per_token_if_dynamic: Whether to do per_tensor or per_token
667 in the dynamic quantization case.
668
669 Returns:
@@ -674,8 +674,8 @@ def scaled_fp8_quant(
674 assert (input.ndim == 2)
675 shape: Union[Tuple[int, int], torch.Size] = input.shape
676 # For rocm, the output fp8 dtype is torch.float_e3m3fnuz
677 - out_dtype: torch.dtype = torch.float8_e4m3fnuz if vllm.utils.is_hip() \
678 - else torch.float8_e4m3fn
677 + out_dtype: torch.dtype = torch.float8_e4m3fnuz \
678 + if current_platform.is_rocm() else torch.float8_e4m3fn
679 if num_token_padding:
680 shape = (max(num_token_padding, input.shape[0]), shape[1])
681 output = torch.empty(shape, device=input.device, dtype=out_dtype)
vllm/attention/ops/blocksparse_attention/interface.py
+3 -3
@@ -3,7 +3,6 @@ import math
3 import torch
4
5 from vllm.platforms import current_platform
6 -from vllm.utils import is_hip
6
7 from .utils import (dense_to_crow_col, get_head_sliding_step,
8 get_sparse_attn_mask)
@@ -32,8 +31,9 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
31 ):
32 super().__init__()
33 if use_spda is None:
35 - use_spda = is_hip() or current_platform.is_cpu() or not \
36 - IS_COMPUTE_8_OR_ABOVE
34 + use_spda = current_platform.is_rocm() or \
35 + current_platform.is_cpu() or not \
36 + IS_COMPUTE_8_OR_ABOVE
37 device = device or (torch.cuda.current_device()
38 if current_platform.is_cuda_alike() else "cpu")
39 device = torch.device(device)
vllm/attention/selector.py
+2 -2
@@ -10,7 +10,7 @@ import vllm.envs as envs
10 from vllm.attention.backends.abstract import AttentionBackend
11 from vllm.logger import init_logger
12 from vllm.platforms import current_platform
13 -from vllm.utils import STR_BACKEND_ENV_VAR, is_hip
13 +from vllm.utils import STR_BACKEND_ENV_VAR
14
15 logger = init_logger(__name__)
16
@@ -208,7 +208,7 @@ def which_attn_to_use(
208 logger.info("Cannot use %s backend on TPU.", selected_backend)
209 return _Backend.PALLAS
210
211 - if is_hip():
211 + if current_platform.is_rocm():
212 # AMD GPUs.
213 selected_backend = (_Backend.ROCM_FLASH if selected_backend
214 == _Backend.FLASH_ATTN else selected_backend)
vllm/config.py
+25 -24
@@ -17,7 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 get_hf_text_config)
19 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20 - is_hip, print_warning_once)
20 + print_warning_once)
21
22 if TYPE_CHECKING:
23 from ray.util.placement_group import PlacementGroup
@@ -43,7 +43,7 @@ class ModelConfig:
43
44 Args:
45 model: Name or path of the huggingface model to use.
46 - It is also used as the content for `model_name` tag in metrics
46 + It is also used as the content for `model_name` tag in metrics
47 output when `served_model_name` is not specified.
48 task: The task to use the model for. Each vLLM instance only supports
49 one task, even if the same model can be used for multiple tasks.
@@ -99,15 +99,15 @@ class ModelConfig:
99 skip_tokenizer_init: If true, skip initialization of tokenizer and
100 detokenizer.
101 served_model_name: The model name used in metrics tag `model_name`,
102 - matches the model name exposed via the APIs. If multiple model
103 - names provided, the first name will be used. If not specified,
102 + matches the model name exposed via the APIs. If multiple model
103 + names provided, the first name will be used. If not specified,
104 the model name will be the same as `model`.
105 - limit_mm_per_prompt: Maximum number of data instances per modality
105 + limit_mm_per_prompt: Maximum number of data instances per modality
106 per prompt. Only applicable for multimodal models.
107 - override_neuron_config: Initialize non default neuron config or
108 - override default neuron config that are specific to Neuron devices,
109 - this argument will be used to configure the neuron config that
110 - can not be gathered from the vllm arguments.
107 + override_neuron_config: Initialize non default neuron config or
108 + override default neuron config that are specific to Neuron devices,
109 + this argument will be used to configure the neuron config that
110 + can not be gathered from the vllm arguments.
111 config_format: The config format which shall be loaded.
112 Defaults to 'auto' which defaults to 'hf'.
113 mm_processor_kwargs: Arguments to be forwarded to the model's processor
@@ -350,7 +350,7 @@ class ModelConfig:
350 raise ValueError(
351 f"Unknown quantization method: {self.quantization}. Must "
352 f"be one of {supported_quantization}.")
353 - if is_hip(
353 + if current_platform.is_rocm(
354 ) and self.quantization not in rocm_supported_quantization:
355 raise ValueError(
356 f"{self.quantization} quantization is currently not "
@@ -365,7 +365,7 @@ class ModelConfig:
365 "%s quantization is not fully "
366 "optimized yet. The speed can be slower than "
367 "non-quantized models.", self.quantization)
368 - if (self.quantization == "awq" and is_hip()
368 + if (self.quantization == "awq" and current_platform.is_rocm()
369 and not envs.VLLM_USE_TRITON_AWQ):
370 logger.warning(
371 "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
@@ -385,7 +385,7 @@ class ModelConfig:
385
386 def _verify_bnb_config(self) -> None:
387 """
388 - The current version of bitsandbytes (0.44.0) with 8-bit models does not
388 + The current version of bitsandbytes (0.44.0) with 8-bit models does not
389 yet support CUDA graph.
390 """
391 is_bitsandbytes = self.quantization == "bitsandbytes"
@@ -810,7 +810,7 @@ class LoadConfig:
810 fast weight loading.
811 "bitsandbytes" will load nf4 type weights.
812 ignore_patterns: The list of patterns to ignore when loading the model.
813 - Default to "original/**/*" to avoid repeated loading of llama's
813 + Default to "original/**/*" to avoid repeated loading of llama's
814 checkpoints.
815
816 """
@@ -843,7 +843,8 @@ class LoadConfig:
843 self.load_format = LoadFormat(load_format)
844
845 rocm_not_supported_load_format: List[str] = []
846 - if is_hip() and load_format in rocm_not_supported_load_format:
846 + if current_platform.is_rocm(
847 + ) and load_format in rocm_not_supported_load_format:
848 rocm_supported_load_format = [
849 f for f in LoadFormat.__members__
850 if (f not in rocm_not_supported_load_format)
@@ -967,7 +968,7 @@ class ParallelConfig:
968 if self.use_ray:
969 from vllm.executor import ray_utils
970 ray_utils.assert_ray_available()
970 - if is_hip():
971 + if current_platform.is_rocm():
972 self.disable_custom_all_reduce = True
973 logger.info(
974 "Disabled the custom all-reduce kernel because it is not "
@@ -996,7 +997,7 @@ class SchedulerConfig:
997 prompt latency) before scheduling next prompt.
998 enable_chunked_prefill: If True, prefill requests can be chunked based
999 on the remaining max_num_batched_tokens.
999 - preemption_mode: Whether to perform preemption by swapping or
1000 + preemption_mode: Whether to perform preemption by swapping or
1001 recomputation. If not specified, we determine the mode as follows:
1002 We use recomputation by default since it incurs lower overhead than
1003 swapping. However, when the sequence group has multiple sequences
@@ -1215,7 +1216,7 @@ class SpeculativeConfig:
1216 typical_acceptance_sampler_posterior_threshold (Optional[float]):
1217 A threshold value that sets a lower bound on the posterior
1218 probability of a token in the target model for it to be
1218 - accepted. This threshold is used only when we use the
1219 + accepted. This threshold is used only when we use the
1220 TypicalAcceptanceSampler for token acceptance.
1221 typical_acceptance_sampler_posterior_alpha (Optional[float]):
1222 A scaling factor for the entropy-based threshold in the
@@ -1225,7 +1226,7 @@ class SpeculativeConfig:
1226 If set to False, token log probabilities are returned
1227 according to the log probability settings in SamplingParams.
1228 If not specified, it defaults to True.
1228 -
1229 +
1230 Returns:
1231 Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
1232 the necessary conditions are met, else None.
@@ -1470,13 +1471,13 @@ class SpeculativeConfig:
1471 typical_acceptance_sampler_posterior_threshold (Optional[float]):
1472 A threshold value that sets a lower bound on the posterior
1473 probability of a token in the target model for it to be
1473 - accepted. This threshold is used only when we use the
1474 + accepted. This threshold is used only when we use the
1475 TypicalAcceptanceSampler for token acceptance.
1476 typical_acceptance_sampler_posterior_alpha (Optional[float]):
1477 A scaling factor for the entropy-based threshold in the
1478 TypicalAcceptanceSampler.
1479 disable_logprobs: If set to True, token log probabilities will not
1479 - be returned even if requested by sampling parameters. This
1480 + be returned even if requested by sampling parameters. This
1481 reduces latency by skipping logprob calculation in proposal
1482 sampling, target sampling, and after accepted tokens are
1483 determined. If set to False, log probabilities will be
@@ -1843,10 +1844,10 @@ def get_min_sliding_window(
1844 def get_served_model_name(model: str,
1845 served_model_name: Optional[Union[str, List[str]]]):
1846 """
1846 - If the input is a non-empty list, the first model_name in
1847 - `served_model_name` is taken.
1848 - If the input is a non-empty string, it is used directly.
1849 - For cases where the input is either an empty string or an
1847 + If the input is a non-empty list, the first model_name in
1848 + `served_model_name` is taken.
1849 + If the input is a non-empty string, it is used directly.
1850 + For cases where the input is either an empty string or an
1851 empty list, the fallback is to use `self.model`.
1852 """
1853 if not served_model_name:
vllm/executor/ray_utils.py
+2 -2
@@ -10,7 +10,7 @@ from vllm.executor.msgspec_utils import decode_hook, encode_hook
10 from vllm.logger import init_logger
11 from vllm.platforms import current_platform
12 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13 -from vllm.utils import get_ip, is_hip
13 +from vllm.utils import get_ip
14 from vllm.worker.worker_base import WorkerWrapperBase
15
16 logger = init_logger(__name__)
@@ -231,7 +231,7 @@ def initialize_ray_cluster(
231 assert_ray_available()
232
233 # Connect to a ray cluster.
234 - if is_hip() or current_platform.is_xpu():
234 + if current_platform.is_rocm() or current_platform.is_xpu():
235 ray.init(address=ray_address,
236 ignore_reinit_error=True,
237 num_gpus=parallel_config.world_size)
vllm/model_executor/custom_op.py
+2 -2
@@ -7,7 +7,7 @@ import vllm.envs as envs
7 from vllm.compilation.levels import CompilationLevel
8 from vllm.logger import init_logger
9 from vllm.platforms import current_platform
10 -from vllm.utils import is_hip, print_warning_once
10 +from vllm.utils import print_warning_once
11
12 logger = init_logger(__name__)
13
@@ -72,7 +72,7 @@ class CustomOp(nn.Module):
72 if not enabled:
73 return self.forward_native
74
75 - if is_hip():
75 + if current_platform.is_rocm():
76 return self.forward_hip
77 elif current_platform.is_cpu():
78 return self.forward_cpu
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+3 -2
@@ -14,7 +14,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
14 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
15 all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
16 from vllm.model_executor.utils import set_weight_attrs
17 -from vllm.utils import is_hip, print_warning_once
17 +from vllm.platforms import current_platform
18 +from vllm.utils import print_warning_once
19
20
21 class GPTQMarlinState(Enum):
@@ -150,7 +151,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
151 layer.w2_input_scale.max(), requires_grad=False)
152
153 # If rocm, normalize the weights and scales to e4m3fnuz
153 - if is_hip():
154 + if current_platform.is_rocm():
155 # Normalize the weights and scales
156 w13_weight, w13_weight_scale, w13_input_scale = \
157 normalize_e4m3fn_to_e4m3fnuz(
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+3 -3
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
13 ModelWeightParameter,
14 PerTensorScaleParameter)
15 -from vllm.utils import is_hip
15 +from vllm.platforms import current_platform
16
17 __all__ = ["CompressedTensorsW8A8Fp8"]
18
@@ -40,7 +40,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
40 logical_widths=layer.logical_widths,
41 )
42
43 - if is_hip():
43 + if current_platform.is_rocm():
44 weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
45 weight=weight,
46 weight_scale=max_w_scale,
@@ -56,7 +56,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
56 elif self.strategy == QuantizationStrategy.CHANNEL:
57 weight = layer.weight
58
59 - if is_hip():
59 + if current_platform.is_rocm():
60 weight, weight_scale, input_scale = \
61 normalize_e4m3fn_to_e4m3fnuz(
62 weight=weight,
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+1 -2
@@ -19,7 +19,6 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
19 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
20 ModelWeightParameter)
21 from vllm.platforms import current_platform
22 -from vllm.utils import is_hip
22
23 logger = init_logger(__name__)
24
@@ -127,7 +126,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
126
127 weight = layer.weight
128
130 - if is_hip():
129 + if current_platform.is_rocm():
130 weight, weight_scale, input_scale = \
131 normalize_e4m3fn_to_e4m3fnuz(
132 weight=weight,
vllm/model_executor/layers/quantization/fp8.py
+5 -5
@@ -26,7 +26,7 @@ from vllm.model_executor.parameter import (ModelWeightParameter,
26 PerTensorScaleParameter)
27 from vllm.model_executor.utils import set_weight_attrs
28 from vllm.platforms import current_platform
29 -from vllm.utils import is_hip, print_warning_once
29 +from vllm.utils import print_warning_once
30
31 ACTIVATION_SCHEMES = ["static", "dynamic"]
32
@@ -123,7 +123,7 @@ class Fp8LinearMethod(LinearMethodBase):
123 self.use_marlin = (not current_platform.has_device_capability(89)
124 or envs.VLLM_TEST_FORCE_FP8_MARLIN)
125 # Disable marlin for rocm
126 - if is_hip():
126 + if current_platform.is_rocm():
127 self.use_marlin = False
128
129 def create_weights(
@@ -226,7 +226,7 @@ class Fp8LinearMethod(LinearMethodBase):
226 weight_scale = layer.weight_scale
227
228 # If rocm, use float8_e4m3fnuz.
229 - if is_hip():
229 + if current_platform.is_rocm():
230 weight, weight_scale, input_scale = \
231 normalize_e4m3fn_to_e4m3fnuz(
232 weight=weight,
@@ -372,7 +372,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
372 if not self.quant_config.is_checkpoint_fp8_serialized:
373 # If rocm, use float8_e4m3fnuz as dtype
374 fp8_dtype = torch.float8_e4m3fnuz \
375 - if is_hip() else torch.float8_e4m3fn
375 + if current_platform.is_rocm() else torch.float8_e4m3fn
376 w13_weight = torch.empty_like(layer.w13_weight.data,
377 dtype=fp8_dtype)
378 w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype)
@@ -420,7 +420,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
420 layer.w2_input_scale = torch.nn.Parameter(
421 layer.w2_input_scale.max(), requires_grad=False)
422 # If rocm, normalize the weights and scales to e4m3fnuz
423 - if is_hip():
423 + if current_platform.is_rocm():
424 # Normalize the weights and scales
425 w13_weight, w13_weight_scale, w13_input_scale = \
426 normalize_e4m3fn_to_e4m3fnuz(
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3 -3
@@ -4,16 +4,16 @@ import torch
4
5 from vllm import _custom_ops as ops
6 from vllm.platforms import current_platform
7 -from vllm.utils import is_hip
7
8 # Input scaling factors are no longer optional in _scaled_mm starting
9 # from pytorch 2.5. Allocating a dummy tensor to pass as input_scale
11 -TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() if is_hip() else None
10 +TORCH_DEVICE_IDENTITY = torch.ones(1).cuda() \
11 + if current_platform.is_rocm() else None
12
13
14 def cutlass_fp8_supported() -> bool:
15 # cutlass is not supported on Rocm
16 - if is_hip():
16 + if current_platform.is_rocm():
17 return False
18
19 capability_tuple = current_platform.get_device_capability()
vllm/model_executor/models/exaone.py
+2 -2
@@ -49,9 +49,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
49 from vllm.model_executor.model_loader.weight_utils import (
50 default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 +from vllm.platforms import current_platform
53 from vllm.sequence import IntermediateTensors
54 from vllm.transformers_utils.configs.exaone import ExaoneConfig
54 -from vllm.utils import is_hip
55
56 from .interfaces import SupportsLoRA, SupportsPP
57 from .utils import (PPMissingLayer, is_pp_missing_parameter,
@@ -595,7 +595,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
595 if not isinstance(self.transformer.h[layer_idx], nn.Identity):
596 layer_self_attn = self.transformer.h[layer_idx].attn
597
598 - if is_hip():
598 + if current_platform.is_rocm():
599 # The scaling factor convention we are assuming is
600 # quantized_value * scaling_factor ~= true_value
601 # which is consistent with the practice of setting
vllm/model_executor/models/granite.py
+2 -2
@@ -49,8 +49,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
49 from vllm.model_executor.model_loader.weight_utils import (
50 default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 +from vllm.platforms import current_platform
53 from vllm.sequence import IntermediateTensors
53 -from vllm.utils import is_hip
54
55 from .interfaces import SupportsLoRA, SupportsPP
56 from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
@@ -534,7 +534,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
534 if not isinstance(self.model.layers[layer_idx], nn.Identity):
535 layer_self_attn = self.model.layers[layer_idx].self_attn
536
537 - if is_hip():
537 + if current_platform.is_rocm():
538 # The scaling factor convention we are assuming is
539 # quantized_value * scaling_factor ~= true_value
540 # which is consistent with the practice of setting
vllm/model_executor/models/llama.py
+2 -2
@@ -50,8 +50,8 @@ from vllm.model_executor.model_loader.weight_utils import (
50 default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
51 from vllm.model_executor.pooling_metadata import PoolingMetadata
52 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 +from vllm.platforms import current_platform
54 from vllm.sequence import IntermediateTensors, PoolerOutput
54 -from vllm.utils import is_hip
55
56 from .interfaces import SupportsLoRA, SupportsPP
57 from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
@@ -423,7 +423,7 @@ class LlamaModel(nn.Module):
423 if not isinstance(self.layers[layer_idx], nn.Identity):
424 layer_self_attn = self.layers[layer_idx].self_attn
425
426 - if is_hip():
426 + if current_platform.is_rocm():
427 # The scaling factor convention we are assuming is
428 # quantized_value * scaling_factor ~= true_value
429 # which is consistent with the practice of setting
vllm/model_executor/models/registry.py
+2 -2
@@ -12,7 +12,7 @@ import cloudpickle
12 import torch.nn as nn
13
14 from vllm.logger import init_logger
15 -from vllm.utils import is_hip
15 +from vllm.platforms import current_platform
16
17 from .interfaces import (has_inner_state, is_attention_free,
18 supports_multimodal, supports_pp)
@@ -247,7 +247,7 @@ def _try_load_model_cls(
247 model_arch: str,
248 model: _BaseRegisteredModel,
249 ) -> Optional[Type[nn.Module]]:
250 - if is_hip():
250 + if current_platform.is_rocm():
251 if model_arch in _ROCM_UNSUPPORTED_MODELS:
252 raise ValueError(f"Model architecture '{model_arch}' is not "
253 "supported by ROCm for now.")
vllm/model_executor/models/solar.py
+2 -2
@@ -49,8 +49,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
49 from vllm.model_executor.model_loader.weight_utils import (
50 default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 +from vllm.platforms import current_platform
53 from vllm.sequence import IntermediateTensors
53 -from vllm.utils import is_hip
54
55 from .interfaces import SupportsLoRA, SupportsPP
56 from .utils import (PPMissingLayer, is_pp_missing_parameter,
@@ -558,7 +558,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
558 if not isinstance(self.model.layers[layer_idx], nn.Identity):
559 layer_self_attn = self.model.layers[layer_idx].self_attn
560
561 - if is_hip():
561 + if current_platform.is_rocm():
562 # The scaling factor convention we are assuming is
563 # quantized_value * scaling_factor ~= true_value
564 # which is consistent with the practice of setting
vllm/utils.py
+1 -5
@@ -314,10 +314,6 @@ class PyObjectCache:
314 self._index = 0
315
316
317 -def is_hip() -> bool:
318 - return torch.version.hip is not None
319 -
320 -
317 @lru_cache(maxsize=None)
318 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
319 """Returns the maximum shared memory per thread block in bytes."""
@@ -1098,7 +1094,7 @@ def _cuda_device_count_stateless(
1094
1095 if not torch.cuda._is_compiled():
1096 return 0
1101 - if is_hip():
1097 + if current_platform.is_rocm():
1098 # ROCm uses amdsmi instead of nvml for stateless device count
1099 # This requires a sufficiently modern version of Torch 2.4.0
1100 raw_count = torch.cuda._device_count_amdsmi() if (hasattr(
vllm/worker/model_runner.py
+5 -4
@@ -41,6 +41,7 @@ from vllm.model_executor.models import supports_lora, supports_multimodal
41 from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
42 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
43 MultiModalInputs, MultiModalRegistry)
44 +from vllm.platforms import current_platform
45 from vllm.prompt_adapter.layers import PromptAdapterMapping
46 from vllm.prompt_adapter.request import PromptAdapterRequest
47 from vllm.prompt_adapter.worker_manager import (
@@ -49,7 +50,7 @@ from vllm.sampling_params import SamplingParams
50 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
51 from vllm.transformers_utils.config import uses_mrope
52 from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52 - flatten_2d_lists, is_hip, is_pin_memory_available,
53 + flatten_2d_lists, is_pin_memory_available,
54 supports_dynamo, weak_ref_tensor)
55 from vllm.worker.model_runner_base import (
56 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -737,13 +738,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
738 family of functions.
739
740 Args:
740 - num_seqs (int): Number of sequences scheduled to run.
741 + num_seqs (int): Number of sequences scheduled to run.
742 max_decode_seq_len (int): Greatest of all the decode sequence
743 lengths. Used only in checking the viablility of using
744 CUDA graphs.
745 max_encoder_seq_len (int, optional): Greatest of all the encode
746 sequence lengths. Defaults to 0. Used only in checking the
746 - viability of using CUDA graphs.
747 + viability of using CUDA graphs.
748 Returns:
749 int: Returns the determined number of padding sequences. If
750 CUDA graphs is not viable, returns -1.
@@ -1103,7 +1104,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1104 self.prompt_adapter_manager.create_prompt_adapter_manager(
1105 self.model))
1106
1106 - if self.kv_cache_dtype == "fp8" and is_hip():
1107 + if self.kv_cache_dtype == "fp8" and current_platform.is_rocm():
1108 # Currently only ROCm accepts kv-cache scaling factors
1109 # via quantization_param_path and this will be deprecated
1110 # in the future.