| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from tests.utils import VLLM_PATH, RemoteOpenAIServer |
| 5 | |
| 6 | chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja" |
| 7 | assert chatml_jinja_path.exists() |
| 8 | |
| 9 | |
| 10 | def run_and_test_dummy_opt_api_server(model, tp=1): |
| 11 | # the model is registered through the plugin |
| 12 | server_args = [ |
| 13 | "--gpu-memory-utilization", |
| 14 | "0.10", |
| 15 | "--dtype", |
| 16 | "float32", |
| 17 | "--chat-template", |
| 18 | str(chatml_jinja_path), |
| 19 | "--load-format", |
| 20 | "dummy", |
| 21 | "-tp", |
| 22 | f"{tp}", |
| 23 | ] |
| 24 | with RemoteOpenAIServer(model, server_args) as server: |
| 25 | client = server.get_client() |
| 26 | completion = client.chat.completions.create( |
| 27 | model=model, |
| 28 | messages=[ |
| 29 | {"role": "system", "content": "You are a helpful assistant."}, |
| 30 | {"role": "user", "content": "Hello!"}, |
| 31 | ], |
| 32 | temperature=0, |
| 33 | ) |
| 34 | generated_text = completion.choices[0].message.content |
| 35 | assert generated_text is not None |
| 36 | # make sure only the first token is generated |
| 37 | rest = generated_text.replace("<s>", "") |
| 38 | assert rest == "" |
| 39 | |
| 40 | |
| 41 | def test_oot_registration_for_api_server(dummy_opt_path: str): |
| 42 | run_and_test_dummy_opt_api_server(dummy_opt_path) |