| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import weakref |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from vllm.exceptions import VLLMValidationError |
| 8 | from vllm.sampling_params import SamplingParams |
| 9 | |
| 10 | |
| 11 | @pytest.fixture(scope="function") |
| 12 | def text_llm(vllm_runner): |
| 13 | with vllm_runner( |
| 14 | "meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0 |
| 15 | ) as runner: |
| 16 | # pytest caches yielded fixtures until after teardown, so use a proxy to |
| 17 | # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory. |
| 18 | yield weakref.proxy(runner.llm) |
| 19 | |
| 20 | |
| 21 | @pytest.fixture(scope="function") |
| 22 | def llm_for_failure_test(vllm_runner): |
| 23 | """Fixture for testing issue #26081. |
| 24 | Uses a small max_model_len to easily trigger length errors. |
| 25 | """ |
| 26 | with vllm_runner( |
| 27 | "meta-llama/Llama-3.2-1B-Instruct", |
| 28 | enforce_eager=True, |
| 29 | seed=0, |
| 30 | max_model_len=128, |
| 31 | disable_log_stats=True, |
| 32 | ) as runner: |
| 33 | # pytest caches yielded fixtures until after teardown, so use a proxy to |
| 34 | # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory. |
| 35 | yield weakref.proxy(runner.llm) |
| 36 | |
| 37 | |
| 38 | def test_chat(text_llm): |
| 39 | prompt1 = "Explain the concept of entropy." |
| 40 | messages = [ |
| 41 | {"role": "system", "content": "You are a helpful assistant"}, |
| 42 | {"role": "user", "content": prompt1}, |
| 43 | ] |
| 44 | outputs = text_llm.chat(messages) |
| 45 | assert len(outputs) == 1 |
| 46 | |
| 47 | |
| 48 | def test_multi_chat(text_llm): |
| 49 | prompt1 = "Explain the concept of entropy." |
| 50 | prompt2 = "Explain what among us is." |
| 51 | |
| 52 | conversation1 = [ |
| 53 | {"role": "system", "content": "You are a helpful assistant"}, |
| 54 | {"role": "user", "content": prompt1}, |
| 55 | ] |
| 56 | |
| 57 | conversation2 = [ |
| 58 | {"role": "system", "content": "You are a helpful assistant"}, |
| 59 | {"role": "user", "content": prompt2}, |
| 60 | ] |
| 61 | |
| 62 | messages = [conversation1, conversation2] |
| 63 | |
| 64 | outputs = text_llm.chat(messages) |
| 65 | assert len(outputs) == 2 |
| 66 | |
| 67 | |
| 68 | def test_llm_chat_tokenization_no_double_bos(text_llm): |
| 69 | """LLM.chat() should not add special tokens when using chat templates. |
| 70 | Check we get a single BOS token for llama chat. |
| 71 | """ |
| 72 | messages = [ |
| 73 | {"role": "system", "content": "You are a helpful assistant"}, |
| 74 | {"role": "user", "content": "Hello!"}, |
| 75 | ] |
| 76 | outputs = text_llm.chat(messages) |
| 77 | assert len(outputs) == 1 |
| 78 | |
| 79 | prompt_token_ids = outputs[0].prompt_token_ids |
| 80 | assert prompt_token_ids is not None |
| 81 | |
| 82 | bos_token = text_llm.get_tokenizer().bos_token_id |
| 83 | |
| 84 | # Ensure we have a single BOS |
| 85 | assert prompt_token_ids[0] == bos_token |
| 86 | assert prompt_token_ids[1] != bos_token, "Double BOS" |
| 87 | |
| 88 | |
| 89 | @pytest.fixture(scope="function") |
| 90 | def thinking_llm(vllm_runner): |
| 91 | with vllm_runner( |
| 92 | "Qwen/Qwen3-0.6B", |
| 93 | max_model_len=4096, |
| 94 | enforce_eager=True, |
| 95 | seed=0, |
| 96 | ) as runner: |
| 97 | # pytest caches yielded fixtures until after teardown, so use a proxy to |
| 98 | # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory. |
| 99 | yield weakref.proxy(runner.llm) |
| 100 | |
| 101 | |
| 102 | @pytest.mark.parametrize("enable_thinking", [True, False]) |
| 103 | def test_chat_extra_kwargs(thinking_llm, enable_thinking): |
| 104 | messages = [ |
| 105 | {"role": "system", "content": "You are a helpful assistant"}, |
| 106 | {"role": "user", "content": "What is 1+1?"}, |
| 107 | ] |
| 108 | |
| 109 | outputs = thinking_llm.chat( |
| 110 | messages, |
| 111 | chat_template_kwargs={"enable_thinking": enable_thinking}, |
| 112 | ) |
| 113 | assert len(outputs) == 1 |
| 114 | |
| 115 | prompt_token_ids = outputs[0].prompt_token_ids |
| 116 | assert prompt_token_ids is not None |
| 117 | |
| 118 | think_id = thinking_llm.get_tokenizer().get_vocab()["<think>"] |
| 119 | |
| 120 | if enable_thinking: |
| 121 | assert think_id not in prompt_token_ids |
| 122 | else: |
| 123 | # The chat template includes dummy thinking process |
| 124 | assert think_id in prompt_token_ids |
| 125 | |
| 126 | |
| 127 | def test_chat_batch_failure_cleanup(llm_for_failure_test): |
| 128 | """Tests that if a batch call to llm.chat() fails mid-way |
| 129 | (e.g., due to one invalid prompt), the requests that |
| 130 | were already enqueued are properly aborted and do not |
| 131 | pollute the queue for subsequent calls. |
| 132 | (Fixes Issue #26081) |
| 133 | """ |
| 134 | llm = llm_for_failure_test |
| 135 | valid_msg = [{"role": "user", "content": "Hello"}] |
| 136 | long_text = "This is a very long text to test the error " * 50 |
| 137 | invalid_msg = [{"role": "user", "content": long_text}] |
| 138 | |
| 139 | batch_1 = [valid_msg, valid_msg, invalid_msg] |
| 140 | batch_2 = [valid_msg, valid_msg] |
| 141 | sampling_params = SamplingParams(temperature=0, max_tokens=10) |
| 142 | |
| 143 | with pytest.raises(VLLMValidationError, match="maximum context length is"): |
| 144 | llm.chat(batch_1, sampling_params=sampling_params) |
| 145 | assert llm.llm_engine.get_num_unfinished_requests() == 0 |
| 146 | |
| 147 | outputs_2 = llm.chat(batch_2, sampling_params=sampling_params) |
| 148 | assert len(outputs_2) == len(batch_2) |
| 149 | assert llm.llm_engine.get_num_unfinished_requests() == 0 |