@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 069d3bd8d0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend] Add Early Validation For Chat Template / Tool Call Parser (#9151)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Oct 8, 2024 at 08:31 UTC 069d3bd8d01a72e93c0a5b51f8b567e8aaddc6e9
5 files changed +155 -72
tests/entrypoints/openai/test_cli_args.py
+109 -69
@@ -1,91 +1,131 @@
1 import json
2 -import unittest
2
4 -from vllm.entrypoints.openai.cli_args import make_arg_parser
3 +import pytest
4 +
5 +from vllm.entrypoints.openai.cli_args import (make_arg_parser,
6 + validate_parsed_serve_args)
7 from vllm.entrypoints.openai.serving_engine import LoRAModulePath
8 from vllm.utils import FlexibleArgumentParser
9
10 +from ...utils import VLLM_PATH
11 +
12 LORA_MODULE = {
13 "name": "module2",
14 "path": "/path/to/module2",
15 "base_model_name": "llama"
16 }
17 +CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
18 +assert CHATML_JINJA_PATH.exists()
19
20
15 -class TestLoraParserAction(unittest.TestCase):
21 +@pytest.fixture
22 +def serve_parser():
23 + parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
24 + return make_arg_parser(parser)
25
17 - def setUp(self):
18 - # Setting up argparse parser for tests
19 - parser = FlexibleArgumentParser(
20 - description="vLLM's remote OpenAI server.")
21 - self.parser = make_arg_parser(parser)
26
23 - def test_valid_key_value_format(self):
24 - # Test old format: name=path
25 - args = self.parser.parse_args([
26 - '--lora-modules',
27 - 'module1=/path/to/module1',
27 +### Tests for Lora module parsing
28 +def test_valid_key_value_format(serve_parser):
29 + # Test old format: name=path
30 + args = serve_parser.parse_args([
31 + '--lora-modules',
32 + 'module1=/path/to/module1',
33 + ])
34 + expected = [LoRAModulePath(name='module1', path='/path/to/module1')]
35 + assert args.lora_modules == expected
36 +
37 +
38 +def test_valid_json_format(serve_parser):
39 + # Test valid JSON format input
40 + args = serve_parser.parse_args([
41 + '--lora-modules',
42 + json.dumps(LORA_MODULE),
43 + ])
44 + expected = [
45 + LoRAModulePath(name='module2',
46 + path='/path/to/module2',
47 + base_model_name='llama')
48 + ]
49 + assert args.lora_modules == expected
50 +
51 +
52 +def test_invalid_json_format(serve_parser):
53 + # Test invalid JSON format input, missing closing brace
54 + with pytest.raises(SystemExit):
55 + serve_parser.parse_args([
56 + '--lora-modules', '{"name": "module3", "path": "/path/to/module3"'
57 ])
29 - expected = [LoRAModulePath(name='module1', path='/path/to/module1')]
30 - self.assertEqual(args.lora_modules, expected)
58
32 - def test_valid_json_format(self):
33 - # Test valid JSON format input
34 - args = self.parser.parse_args([
59 +
60 +def test_invalid_type_error(serve_parser):
61 + # Test type error when values are not JSON or key=value
62 + with pytest.raises(SystemExit):
63 + serve_parser.parse_args([
64 '--lora-modules',
36 - json.dumps(LORA_MODULE),
65 + 'invalid_format' # This is not JSON or key=value format
66 ])
38 - expected = [
39 - LoRAModulePath(name='module2',
40 - path='/path/to/module2',
41 - base_model_name='llama')
42 - ]
43 - self.assertEqual(args.lora_modules, expected)
44 -
45 - def test_invalid_json_format(self):
46 - # Test invalid JSON format input, missing closing brace
47 - with self.assertRaises(SystemExit):
48 - self.parser.parse_args([
49 - '--lora-modules',
50 - '{"name": "module3", "path": "/path/to/module3"'
51 - ])
52 -
53 - def test_invalid_type_error(self):
54 - # Test type error when values are not JSON or key=value
55 - with self.assertRaises(SystemExit):
56 - self.parser.parse_args([
57 - '--lora-modules',
58 - 'invalid_format' # This is not JSON or key=value format
59 - ])
60 -
61 - def test_invalid_json_field(self):
62 - # Test valid JSON format but missing required fields
63 - with self.assertRaises(SystemExit):
64 - self.parser.parse_args([
65 - '--lora-modules',
66 - '{"name": "module4"}' # Missing required 'path' field
67 - ])
68 -
69 - def test_empty_values(self):
70 - # Test when no LoRA modules are provided
71 - args = self.parser.parse_args(['--lora-modules', ''])
72 - self.assertEqual(args.lora_modules, [])
73 -
74 - def test_multiple_valid_inputs(self):
75 - # Test multiple valid inputs (both old and JSON format)
76 - args = self.parser.parse_args([
67 +
68 +
69 +def test_invalid_json_field(serve_parser):
70 + # Test valid JSON format but missing required fields
71 + with pytest.raises(SystemExit):
72 + serve_parser.parse_args([
73 '--lora-modules',
78 - 'module1=/path/to/module1',
79 - json.dumps(LORA_MODULE),
74 + '{"name": "module4"}' # Missing required 'path' field
75 ])
81 - expected = [
82 - LoRAModulePath(name='module1', path='/path/to/module1'),
83 - LoRAModulePath(name='module2',
84 - path='/path/to/module2',
85 - base_model_name='llama')
86 - ]
87 - self.assertEqual(args.lora_modules, expected)
76
77
90 -if __name__ == '__main__':
91 - unittest.main()
78 +def test_empty_values(serve_parser):
79 + # Test when no LoRA modules are provided
80 + args = serve_parser.parse_args(['--lora-modules', ''])
81 + assert args.lora_modules == []
82 +
83 +
84 +def test_multiple_valid_inputs(serve_parser):
85 + # Test multiple valid inputs (both old and JSON format)
86 + args = serve_parser.parse_args([
87 + '--lora-modules',
88 + 'module1=/path/to/module1',
89 + json.dumps(LORA_MODULE),
90 + ])
91 + expected = [
92 + LoRAModulePath(name='module1', path='/path/to/module1'),
93 + LoRAModulePath(name='module2',
94 + path='/path/to/module2',
95 + base_model_name='llama')
96 + ]
97 + assert args.lora_modules == expected
98 +
99 +
100 +### Tests for serve argument validation that run prior to loading
101 +def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
102 + """Ensure validation fails if tool choice is enabled with no call parser"""
103 + # If we enable-auto-tool-choice, explode with no tool-call-parser
104 + args = serve_parser.parse_args(args=["--enable-auto-tool-choice"])
105 + with pytest.raises(TypeError):
106 + validate_parsed_serve_args(args)
107 +
108 +
109 +def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
110 + """Ensure validation passes with tool choice enabled with a call parser"""
111 + args = serve_parser.parse_args(args=[
112 + "--enable-auto-tool-choice",
113 + "--tool-call-parser",
114 + "mistral",
115 + ])
116 + validate_parsed_serve_args(args)
117 +
118 +
119 +def test_chat_template_validation_for_happy_paths(serve_parser):
120 + """Ensure validation passes if the chat template exists"""
121 + args = serve_parser.parse_args(
122 + args=["--chat-template",
123 + CHATML_JINJA_PATH.absolute().as_posix()])
124 + validate_parsed_serve_args(args)
125 +
126 +
127 +def test_chat_template_validation_for_sad_paths(serve_parser):
128 + """Ensure validation fails if the chat template doesn't exist"""
129 + args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"])
130 + with pytest.raises(ValueError):
131 + validate_parsed_serve_args(args)
vllm/entrypoints/chat_utils.py
+22
@@ -303,6 +303,28 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
303 self._add_placeholder(placeholder)
304
305
306 +def validate_chat_template(chat_template: Optional[Union[Path, str]]):
307 + """Raises if the provided chat template appears invalid."""
308 + if chat_template is None:
309 + return
310 +
311 + elif isinstance(chat_template, Path) and not chat_template.exists():
312 + raise FileNotFoundError(
313 + "the supplied chat template path doesn't exist")
314 +
315 + elif isinstance(chat_template, str):
316 + JINJA_CHARS = "{}\n"
317 + if not any(c in chat_template
318 + for c in JINJA_CHARS) and not Path(chat_template).exists():
319 + raise ValueError(
320 + f"The supplied chat template string ({chat_template}) "
321 + f"appears path-like, but doesn't exist!")
322 +
323 + else:
324 + raise TypeError(
325 + f"{type(chat_template)} is not a valid chat template type")
326 +
327 +
328 def load_chat_template(
329 chat_template: Optional[Union[Path, str]]) -> Optional[str]:
330 if chat_template is None:
vllm/entrypoints/openai/api_server.py
+3 -1
@@ -31,7 +31,8 @@ from vllm.engine.multiprocessing.engine import run_mp_engine
31 from vllm.engine.protocol import EngineClient
32 from vllm.entrypoints.launcher import serve_http
33 from vllm.entrypoints.logger import RequestLogger
34 -from vllm.entrypoints.openai.cli_args import make_arg_parser
34 +from vllm.entrypoints.openai.cli_args import (make_arg_parser,
35 + validate_parsed_serve_args)
36 # yapf conflicts with isort for this block
37 # yapf: disable
38 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -577,5 +578,6 @@ if __name__ == "__main__":
578 description="vLLM OpenAI-Compatible RESTful API server.")
579 parser = make_arg_parser(parser)
580 args = parser.parse_args()
581 + validate_parsed_serve_args(args)
582
583 uvloop.run(run_server(args))
vllm/entrypoints/openai/cli_args.py
+15
@@ -10,6 +10,7 @@ import ssl
10 from typing import List, Optional, Sequence, Union
11
12 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13 +from vllm.entrypoints.chat_utils import validate_chat_template
14 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
15 PromptAdapterPath)
16 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -231,6 +232,20 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
232 return parser
233
234
235 +def validate_parsed_serve_args(args: argparse.Namespace):
236 + """Quick checks for model serve args that raise prior to loading."""
237 + if hasattr(args, "subparser") and args.subparser != "serve":
238 + return
239 +
240 + # Ensure that the chat template is valid; raises if it likely isn't
241 + validate_chat_template(args.chat_template)
242 +
243 + # Enable auto tool needs a tool call parser to be valid
244 + if args.enable_auto_tool_choice and not args.tool_call_parser:
245 + raise TypeError("Error: --enable-auto-tool-choice requires "
246 + "--tool-call-parser")
247 +
248 +
249 def create_parser_for_docs() -> FlexibleArgumentParser:
250 parser_for_docs = FlexibleArgumentParser(
251 prog="-m vllm.entrypoints.openai.api_server")
vllm/scripts.py
+6 -2
@@ -11,7 +11,8 @@ from openai.types.chat import ChatCompletionMessageParam
11
12 from vllm.engine.arg_utils import EngineArgs
13 from vllm.entrypoints.openai.api_server import run_server
14 -from vllm.entrypoints.openai.cli_args import make_arg_parser
14 +from vllm.entrypoints.openai.cli_args import (make_arg_parser,
15 + validate_parsed_serve_args)
16 from vllm.logger import init_logger
17 from vllm.utils import FlexibleArgumentParser
18
@@ -142,7 +143,7 @@ def main():
143 env_setup()
144
145 parser = FlexibleArgumentParser(description="vLLM CLI")
145 - subparsers = parser.add_subparsers(required=True)
146 + subparsers = parser.add_subparsers(required=True, dest="subparser")
147
148 serve_parser = subparsers.add_parser(
149 "serve",
@@ -186,6 +187,9 @@ def main():
187 chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat")
188
189 args = parser.parse_args()
190 + if args.subparser == "serve":
191 + validate_parsed_serve_args(args)
192 +
193 # One of the sub commands should be executed.
194 if hasattr(args, "dispatch_function"):
195 args.dispatch_function(args)