A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] rework test plans (#9866)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 31, 2024 at 22:20 UTC
566cd277979bc1a46b7e99657112416af9874a58
4 files changed
+226
-31
tests/compile/test_basic_correctness.py
+95
-18
@@ -1,3 +1,4 @@
1
+import dataclasses
2
from typing import Dict, List, Optional
3
4
import pytest
@@ -8,33 +9,109 @@ from vllm.utils import cuda_device_count_stateless
9
from ..utils import compare_all_settings
10
11
12
+@dataclasses.dataclass
13
+class TestSetting:
14
+ model: str
15
+ model_args: List[str]
16
+ pp_size: int
17
+ tp_size: int
18
+ attn_backend: str
19
+ method: str
20
+ fullgraph: bool
21
+
22
+
23
+# representative settings for testing
24
+test_settings = [
25
+ # basic llama model
26
+ TestSetting(
27
+ model="meta-llama/Llama-3.2-1B",
28
+ model_args=[],
29
+ pp_size=2,
30
+ tp_size=2,
31
+ attn_backend="FLASHINFER",
32
+ method="generate",
33
+ fullgraph=True,
34
+ ),
35
+ # llama model with quantization
36
+ TestSetting(
37
+ model="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
38
+ model_args=["--quantization", "gptq"],
39
+ pp_size=1,
40
+ tp_size=1,
41
+ attn_backend="FLASH_ATTN",
42
+ method="generate",
43
+ fullgraph=True,
44
+ ),
45
+ # MoE model
46
+ TestSetting(
47
+ model="ibm/PowerMoE-3b",
48
+ model_args=[],
49
+ pp_size=1,
50
+ tp_size=2,
51
+ attn_backend="FLASH_ATTN",
52
+ method="generate",
53
+ fullgraph=True,
54
+ ),
55
+ # embedding model
56
+ TestSetting(
57
+ model="BAAI/bge-multilingual-gemma2",
58
+ model_args=["--task", "embedding"],
59
+ pp_size=1,
60
+ tp_size=1,
61
+ attn_backend="FLASHINFER",
62
+ method="encode",
63
+ fullgraph=True,
64
+ ),
65
+ # vision language model
66
+ TestSetting(
67
+ model="microsoft/Phi-3.5-vision-instruct",
68
+ model_args=["--trust-remote-code", "--max-model-len", "2048"],
69
+ pp_size=2,
70
+ tp_size=1,
71
+ attn_backend="FLASH_ATTN",
72
+ method="generate_with_image",
73
+ fullgraph=False,
74
+ ),
75
+]
76
+
77
+
78
# we cannot afford testing the full Catesian product
79
# of all models and all levels
13
-@pytest.mark.parametrize(
14
- "model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15
- [
16
- ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASHINFER", "generate", True),
17
- ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
18
- ["--quantization", "compressed-tensors"
19
- ], 1, 1, "FLASH_ATTN", "generate", True),
20
- ("ibm/PowerMoE-3b", [], 1, 2, "FLASH_ATTN", "generate", True),
21
- # TODO: add multi-modality test for llava
22
- ("llava-hf/llava-1.5-7b-hf", [], 2, 1, "FLASHINFER", "generate", False)
23
- ])
24
-def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
25
- method, fullgraph):
80
+@pytest.mark.parametrize("test_setting", test_settings)
81
+def test_compile_correctness(test_setting: TestSetting):
82
# this test is run under multiple suits, with different GPUs.
83
# make sure we only run the test with correct CUDA devices.
84
# don't use "<", as it will duplicate the tests.
85
+ model = test_setting.model
86
+ model_args = test_setting.model_args
87
+ pp_size = test_setting.pp_size
88
+ tp_size = test_setting.tp_size
89
+ attn_backend = test_setting.attn_backend
90
+ method = test_setting.method
91
+ fullgraph = test_setting.fullgraph
92
if cuda_device_count_stateless() != pp_size * tp_size:
93
pytest.skip("Not correct CUDA devices for the test.")
94
import os
95
os.environ["VLLM_ATTENTION_BACKEND"] = attn_backend
33
- all_args = [["--enforce-eager"] + model_args + ["-pp", str(pp_size)] +
34
- ["-tp", str(tp_size)]] * 3
35
- # don't test VLLM_TORCH_COMPILE_LEVEL == 3 case
36
- # inductor will change the output, so we cannot compare them.
96
+ final_args = ["--enforce-eager"] + model_args + ["-pp", str(pp_size)] + \
97
+ ["-tp", str(tp_size)]
98
+
99
all_envs: List[Optional[Dict[str, str]]] = []
100
+
101
+ for level in [
102
+ CompilationLevel.NO_COMPILATION,
103
+ CompilationLevel.PIECEWISE,
104
+ ]:
105
+ all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
106
+
107
+ # inductor will change the output, so we only compare if the output
108
+ # is close, not exactly the same.
109
+ compare_all_settings(
110
+ model, [final_args] * 2,
111
+ all_envs,
112
+ method=method if method != "generate" else "generate_close")
113
+ all_envs.clear()
114
+
115
for level in [
116
CompilationLevel.NO_COMPILATION,
117
CompilationLevel.DYNAMO_AS_IS,
@@ -46,4 +123,4 @@ def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
123
all_envs[-1][
124
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0" # type: ignore
125
49
- compare_all_settings(model, all_args, all_envs, method=method)
126
+ compare_all_settings(model, [final_args] * 3, all_envs, method=method)
tests/utils.py
+119
-5
@@ -1,4 +1,5 @@
1
import asyncio
2
+import copy
3
import functools
4
import os
5
import signal
@@ -8,13 +9,14 @@ import time
9
import warnings
10
from contextlib import contextmanager
11
from pathlib import Path
11
-from typing import Any, Callable, Dict, List, Literal, Optional, Type, Union
12
+from typing import Any, Callable, Dict, List, Optional, Type, Union
13
14
import openai
15
import pytest
16
import requests
17
+import torch
18
from openai.types.completion import Completion
17
-from typing_extensions import ParamSpec, assert_never
19
+from typing_extensions import ParamSpec
20
21
import vllm.envs as envs
22
from tests.models.utils import TextTextLogprobs
@@ -272,6 +274,31 @@ def _test_completion(
274
return results
275
276
277
+def _test_completion_close(
278
+ client: openai.OpenAI,
279
+ model: str,
280
+ prompt: str,
281
+):
282
+ results = []
283
+
284
+ # test with text prompt
285
+ completion = client.completions.create(model=model,
286
+ prompt=prompt,
287
+ max_tokens=1,
288
+ logprobs=5,
289
+ temperature=0.0)
290
+
291
+ logporbs = completion.choices[0].logprobs.top_logprobs[0]
292
+ logporbs = {k: round(v, 2) for k, v in logporbs.items()}
293
+
294
+ results.append({
295
+ "test": "completion_close",
296
+ "logprobs": logporbs,
297
+ })
298
+
299
+ return results
300
+
301
+
302
def _test_embeddings(
303
client: openai.OpenAI,
304
model: str,
@@ -295,13 +322,81 @@ def _test_embeddings(
322
return results
323
324
325
+def _test_image_text(
326
+ client: openai.OpenAI,
327
+ model_name: str,
328
+ image_url: str,
329
+):
330
+ results = []
331
+
332
+ # test pure text input
333
+ messages = [{
334
+ "role":
335
+ "user",
336
+ "content": [
337
+ {
338
+ "type": "text",
339
+ "text": "How do you feel today?"
340
+ },
341
+ ],
342
+ }]
343
+
344
+ chat_completion = client.chat.completions.create(model=model_name,
345
+ messages=messages,
346
+ temperature=0.0,
347
+ max_tokens=1,
348
+ logprobs=True,
349
+ top_logprobs=5)
350
+ top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
351
+
352
+ for x in top_logprobs:
353
+ x.logprob = round(x.logprob, 2)
354
+
355
+ results.append({
356
+ "test": "pure_text",
357
+ "logprobs": top_logprobs,
358
+ })
359
+
360
+ messages = [{
361
+ "role":
362
+ "user",
363
+ "content": [
364
+ {
365
+ "type": "image_url",
366
+ "image_url": {
367
+ "url": image_url
368
+ }
369
+ },
370
+ {
371
+ "type": "text",
372
+ "text": "What's in this image?"
373
+ },
374
+ ],
375
+ }]
376
+
377
+ chat_completion = client.chat.completions.create(model=model_name,
378
+ messages=messages,
379
+ temperature=0.0,
380
+ max_tokens=1,
381
+ logprobs=True,
382
+ top_logprobs=5)
383
+ top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
384
+
385
+ results.append({
386
+ "test": "text_image",
387
+ "logprobs": top_logprobs,
388
+ })
389
+
390
+ return results
391
+
392
+
393
def compare_two_settings(model: str,
394
arg1: List[str],
395
arg2: List[str],
396
env1: Optional[Dict[str, str]] = None,
397
env2: Optional[Dict[str, str]] = None,
398
*,
304
- method: Literal["generate", "encode"] = "generate",
399
+ method: str = "generate",
400
max_wait_seconds: Optional[float] = None) -> None:
401
"""
402
Launch API server with two different sets of arguments/environments
@@ -328,7 +423,7 @@ def compare_all_settings(model: str,
423
all_args: List[List[str]],
424
all_envs: List[Optional[Dict[str, str]]],
425
*,
331
- method: Literal["generate", "encode"] = "generate",
426
+ method: str = "generate",
427
max_wait_seconds: Optional[float] = None) -> None:
428
"""
429
Launch API server with several different sets of arguments/environments
@@ -397,10 +492,17 @@ def compare_all_settings(model: str,
492
493
if method == "generate":
494
results += _test_completion(client, model, prompt, token_ids)
495
+ elif method == "generate_close":
496
+ results += _test_completion_close(client, model, prompt)
497
+ elif method == "generate_with_image":
498
+ results += _test_image_text(
499
+ client, model,
500
+ "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png"
501
+ )
502
elif method == "encode":
503
results += _test_embeddings(client, model, prompt)
504
else:
403
- assert_never(method)
505
+ raise ValueError(f"Unknown method: {method}")
506
507
if i > 0:
508
# if any setting fails, raise an error early
@@ -410,6 +512,18 @@ def compare_all_settings(model: str,
512
compare_envs = all_envs[i]
513
for ref_result, compare_result in zip(ref_results,
514
compare_results):
515
+ ref_result = copy.deepcopy(ref_result)
516
+ compare_result = copy.deepcopy(compare_result)
517
+ if "embedding" in ref_result and method == "encode":
518
+ ref_embedding = torch.tensor(ref_result["embedding"])
519
+ compare_embedding = torch.tensor(
520
+ compare_result["embedding"])
521
+ mse = ((ref_embedding - compare_embedding)**2).mean()
522
+ assert mse < 1e-6, (
523
+ f"Embedding for {model=} are not the same.\n"
524
+ f"mse={mse}\n")
525
+ del ref_result["embedding"]
526
+ del compare_result["embedding"]
527
assert ref_result == compare_result, (
528
f"Results for {model=} are not the same.\n"
529
f"{ref_args=} {ref_envs=}\n"
vllm/model_executor/models/llava.py
+5
-5
@@ -493,13 +493,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
493
:class:`LlavaImageInputs`
494
"""
495
if intermediate_tensors is not None:
496
- input_ids = None
496
inputs_embeds = None
497
else:
499
- # always pass the input via `inputs_embeds`
500
- # to make sure the computation graph is consistent
498
image_input = self._parse_and_validate_image_input(**kwargs)
502
-
499
if image_input is not None:
500
vision_embeddings = self._process_image_input(image_input)
501
inputs_embeds = self.language_model.model.get_input_embeddings(
@@ -511,7 +507,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
507
else:
508
inputs_embeds = self.language_model.model.get_input_embeddings(
509
input_ids)
514
- input_ids = None
510
+
511
+ # always pass the input via `inputs_embeds`
512
+ # to make sure the computation graph is consistent
513
+ # for `torch.compile` integration
514
+ input_ids = None
515
516
hidden_states = self.language_model.model(input_ids,
517
positions,
vllm/model_executor/models/phi3v.py
+7
-3
@@ -679,7 +679,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
679
intermediate_tensors: Optional[IntermediateTensors] = None,
680
**kwargs: object):
681
if intermediate_tensors is not None:
682
- input_ids = None
682
inputs_embeds = None
683
else:
684
image_input = self._parse_and_validate_image_input(**kwargs)
@@ -690,9 +689,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
689
inputs_embeds = merge_multimodal_embeddings(
690
input_ids, inputs_embeds, vision_embeddings,
691
self.image_token_id)
693
- input_ids = None
692
else:
695
- inputs_embeds = None
693
+ inputs_embeds = self.language_model.model.embed_tokens(
694
+ input_ids)
695
+
696
+ # always pass the input via `inputs_embeds`
697
+ # to make sure the computation graph is consistent
698
+ # for `torch.compile` integration
699
+ input_ids = None
700
701
hidden_states = self.language_model.model(input_ids,
702
positions,