| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | import torch |
| 5 | |
| 6 | from vllm.platforms import current_platform |
| 7 | |
| 8 | from ....utils import large_gpu_test |
| 9 | from ...utils import check_logprobs_close |
| 10 | |
| 11 | MODELS = [ |
| 12 | "microsoft/Phi-3.5-MoE-instruct", |
| 13 | ] |
| 14 | |
| 15 | |
| 16 | def test_phimoe_routing_function(): |
| 17 | from vllm.model_executor.models.phimoe import phimoe_routing_function |
| 18 | |
| 19 | test_case = { |
| 20 | 0: { |
| 21 | "hidden_states": torch.tensor( |
| 22 | [1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False |
| 23 | ).view(4, 2), |
| 24 | "gating_output": torch.tensor( |
| 25 | [0.1, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False |
| 26 | ), |
| 27 | "topk": 2, |
| 28 | "renormalize": False, |
| 29 | }, |
| 30 | 1: { |
| 31 | "hidden_states": torch.tensor( |
| 32 | [1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False |
| 33 | ).view(4, 2), |
| 34 | "gating_output": torch.tensor( |
| 35 | [0.4, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False |
| 36 | ), |
| 37 | "topk": 2, |
| 38 | "renormalize": False, |
| 39 | }, |
| 40 | } |
| 41 | |
| 42 | ground_truth = { |
| 43 | 0: { |
| 44 | "topk_weights": torch.tensor( |
| 45 | [1.0, 1.0], dtype=torch.float32, requires_grad=False |
| 46 | ), |
| 47 | "topk_ids": torch.tensor([3, 2], dtype=torch.long, requires_grad=False), |
| 48 | }, |
| 49 | 1: { |
| 50 | "topk_weights": torch.tensor( |
| 51 | [0.5, 1.0], dtype=torch.float32, requires_grad=False |
| 52 | ), |
| 53 | "topk_ids": torch.tensor([0, 3], dtype=torch.long, requires_grad=False), |
| 54 | }, |
| 55 | } |
| 56 | |
| 57 | for test_id in test_case: |
| 58 | topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id]) |
| 59 | assert torch.allclose(topk_weights, ground_truth[test_id]["topk_weights"]) |
| 60 | assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"]) |
| 61 | |
| 62 | |
| 63 | # There is a known issue that triggers `AttributeError: 'DynamicCache' |
| 64 | # object has no attribute 'seen_tokens'` when running: |
| 65 | # `tests/models/language/generation/test_phimoe.py::test_models |
| 66 | # [5-64-bfloat16-microsoft/Phi-3.5-MoE-instruct]` |
| 67 | # This issue is being investigated and tracked in: |
| 68 | # https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 |
| 69 | # It is platform-agnostic. Therefore, we skip this test on all platforms for now. |
| 70 | @pytest.mark.skip( |
| 71 | reason="Skipping due to known issue: " |
| 72 | "'DynamicCache' object has no attribute 'seen_tokens'. See: " |
| 73 | "https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 " |
| 74 | "for details.", |
| 75 | ) |
| 76 | @pytest.mark.skipif( |
| 77 | condition=current_platform.is_cpu(), |
| 78 | reason="This test takes a lot time to run on CPU, " |
| 79 | "and vllm CI's disk space is not enough for this model.", |
| 80 | ) |
| 81 | @large_gpu_test(min_gb=80) |
| 82 | @pytest.mark.parametrize("model", MODELS) |
| 83 | @pytest.mark.parametrize("dtype", ["bfloat16"]) |
| 84 | @pytest.mark.parametrize("max_tokens", [64]) |
| 85 | @pytest.mark.parametrize("num_logprobs", [5]) |
| 86 | def test_models( |
| 87 | hf_runner, |
| 88 | vllm_runner, |
| 89 | example_prompts, |
| 90 | model: str, |
| 91 | dtype: str, |
| 92 | max_tokens: int, |
| 93 | num_logprobs: int, |
| 94 | ) -> None: |
| 95 | with hf_runner(model, dtype=dtype) as hf_model: |
| 96 | hf_outputs = hf_model.generate_greedy_logprobs_limit( |
| 97 | example_prompts, max_tokens, num_logprobs |
| 98 | ) |
| 99 | |
| 100 | with vllm_runner(model, dtype=dtype) as vllm_model: |
| 101 | vllm_outputs = vllm_model.generate_greedy_logprobs( |
| 102 | example_prompts, max_tokens, num_logprobs |
| 103 | ) |
| 104 | check_logprobs_close( |
| 105 | outputs_0_lst=hf_outputs, |
| 106 | outputs_1_lst=vllm_outputs, |
| 107 | name_0="hf", |
| 108 | name_1="vllm", |
| 109 | ) |