@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 566cd27797
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] rework test plans (#9866)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 31, 2024 at 22:20 UTC 566cd277979bc1a46b7e99657112416af9874a58
4 files changed +226 -31
tests/compile/test_basic_correctness.py
+95 -18
@@ -1,3 +1,4 @@
1 +import dataclasses
2 from typing import Dict, List, Optional
3
4 import pytest
@@ -8,33 +9,109 @@ from vllm.utils import cuda_device_count_stateless
9 from ..utils import compare_all_settings
10
11
12 +@dataclasses.dataclass
13 +class TestSetting:
14 + model: str
15 + model_args: List[str]
16 + pp_size: int
17 + tp_size: int
18 + attn_backend: str
19 + method: str
20 + fullgraph: bool
21 +
22 +
23 +# representative settings for testing
24 +test_settings = [
25 + # basic llama model
26 + TestSetting(
27 + model="meta-llama/Llama-3.2-1B",
28 + model_args=[],
29 + pp_size=2,
30 + tp_size=2,
31 + attn_backend="FLASHINFER",
32 + method="generate",
33 + fullgraph=True,
34 + ),
35 + # llama model with quantization
36 + TestSetting(
37 + model="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
38 + model_args=["--quantization", "gptq"],
39 + pp_size=1,
40 + tp_size=1,
41 + attn_backend="FLASH_ATTN",
42 + method="generate",
43 + fullgraph=True,
44 + ),
45 + # MoE model
46 + TestSetting(
47 + model="ibm/PowerMoE-3b",
48 + model_args=[],
49 + pp_size=1,
50 + tp_size=2,
51 + attn_backend="FLASH_ATTN",
52 + method="generate",
53 + fullgraph=True,
54 + ),
55 + # embedding model
56 + TestSetting(
57 + model="BAAI/bge-multilingual-gemma2",
58 + model_args=["--task", "embedding"],
59 + pp_size=1,
60 + tp_size=1,
61 + attn_backend="FLASHINFER",
62 + method="encode",
63 + fullgraph=True,
64 + ),
65 + # vision language model
66 + TestSetting(
67 + model="microsoft/Phi-3.5-vision-instruct",
68 + model_args=["--trust-remote-code", "--max-model-len", "2048"],
69 + pp_size=2,
70 + tp_size=1,
71 + attn_backend="FLASH_ATTN",
72 + method="generate_with_image",
73 + fullgraph=False,
74 + ),
75 +]
76 +
77 +
78 # we cannot afford testing the full Catesian product
79 # of all models and all levels
13 -@pytest.mark.parametrize(
14 - "model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15 - [
16 - ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASHINFER", "generate", True),
17 - ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
18 - ["--quantization", "compressed-tensors"
19 - ], 1, 1, "FLASH_ATTN", "generate", True),
20 - ("ibm/PowerMoE-3b", [], 1, 2, "FLASH_ATTN", "generate", True),
21 - # TODO: add multi-modality test for llava
22 - ("llava-hf/llava-1.5-7b-hf", [], 2, 1, "FLASHINFER", "generate", False)
23 - ])
24 -def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
25 - method, fullgraph):
80 +@pytest.mark.parametrize("test_setting", test_settings)
81 +def test_compile_correctness(test_setting: TestSetting):
82 # this test is run under multiple suits, with different GPUs.
83 # make sure we only run the test with correct CUDA devices.
84 # don't use "<", as it will duplicate the tests.
85 + model = test_setting.model
86 + model_args = test_setting.model_args
87 + pp_size = test_setting.pp_size
88 + tp_size = test_setting.tp_size
89 + attn_backend = test_setting.attn_backend
90 + method = test_setting.method
91 + fullgraph = test_setting.fullgraph
92 if cuda_device_count_stateless() != pp_size * tp_size:
93 pytest.skip("Not correct CUDA devices for the test.")
94 import os
95 os.environ["VLLM_ATTENTION_BACKEND"] = attn_backend
33 - all_args = [["--enforce-eager"] + model_args + ["-pp", str(pp_size)] +
34 - ["-tp", str(tp_size)]] * 3
35 - # don't test VLLM_TORCH_COMPILE_LEVEL == 3 case
36 - # inductor will change the output, so we cannot compare them.
96 + final_args = ["--enforce-eager"] + model_args + ["-pp", str(pp_size)] + \
97 + ["-tp", str(tp_size)]
98 +
99 all_envs: List[Optional[Dict[str, str]]] = []
100 +
101 + for level in [
102 + CompilationLevel.NO_COMPILATION,
103 + CompilationLevel.PIECEWISE,
104 + ]:
105 + all_envs.append({"VLLM_TORCH_COMPILE_LEVEL": str(level)})
106 +
107 + # inductor will change the output, so we only compare if the output
108 + # is close, not exactly the same.
109 + compare_all_settings(
110 + model, [final_args] * 2,
111 + all_envs,
112 + method=method if method != "generate" else "generate_close")
113 + all_envs.clear()
114 +
115 for level in [
116 CompilationLevel.NO_COMPILATION,
117 CompilationLevel.DYNAMO_AS_IS,
@@ -46,4 +123,4 @@ def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
123 all_envs[-1][
124 "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0" # type: ignore
125
49 - compare_all_settings(model, all_args, all_envs, method=method)
126 + compare_all_settings(model, [final_args] * 3, all_envs, method=method)
tests/utils.py
+119 -5
@@ -1,4 +1,5 @@
1 import asyncio
2 +import copy
3 import functools
4 import os
5 import signal
@@ -8,13 +9,14 @@ import time
9 import warnings
10 from contextlib import contextmanager
11 from pathlib import Path
11 -from typing import Any, Callable, Dict, List, Literal, Optional, Type, Union
12 +from typing import Any, Callable, Dict, List, Optional, Type, Union
13
14 import openai
15 import pytest
16 import requests
17 +import torch
18 from openai.types.completion import Completion
17 -from typing_extensions import ParamSpec, assert_never
19 +from typing_extensions import ParamSpec
20
21 import vllm.envs as envs
22 from tests.models.utils import TextTextLogprobs
@@ -272,6 +274,31 @@ def _test_completion(
274 return results
275
276
277 +def _test_completion_close(
278 + client: openai.OpenAI,
279 + model: str,
280 + prompt: str,
281 +):
282 + results = []
283 +
284 + # test with text prompt
285 + completion = client.completions.create(model=model,
286 + prompt=prompt,
287 + max_tokens=1,
288 + logprobs=5,
289 + temperature=0.0)
290 +
291 + logporbs = completion.choices[0].logprobs.top_logprobs[0]
292 + logporbs = {k: round(v, 2) for k, v in logporbs.items()}
293 +
294 + results.append({
295 + "test": "completion_close",
296 + "logprobs": logporbs,
297 + })
298 +
299 + return results
300 +
301 +
302 def _test_embeddings(
303 client: openai.OpenAI,
304 model: str,
@@ -295,13 +322,81 @@ def _test_embeddings(
322 return results
323
324
325 +def _test_image_text(
326 + client: openai.OpenAI,
327 + model_name: str,
328 + image_url: str,
329 +):
330 + results = []
331 +
332 + # test pure text input
333 + messages = [{
334 + "role":
335 + "user",
336 + "content": [
337 + {
338 + "type": "text",
339 + "text": "How do you feel today?"
340 + },
341 + ],
342 + }]
343 +
344 + chat_completion = client.chat.completions.create(model=model_name,
345 + messages=messages,
346 + temperature=0.0,
347 + max_tokens=1,
348 + logprobs=True,
349 + top_logprobs=5)
350 + top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
351 +
352 + for x in top_logprobs:
353 + x.logprob = round(x.logprob, 2)
354 +
355 + results.append({
356 + "test": "pure_text",
357 + "logprobs": top_logprobs,
358 + })
359 +
360 + messages = [{
361 + "role":
362 + "user",
363 + "content": [
364 + {
365 + "type": "image_url",
366 + "image_url": {
367 + "url": image_url
368 + }
369 + },
370 + {
371 + "type": "text",
372 + "text": "What's in this image?"
373 + },
374 + ],
375 + }]
376 +
377 + chat_completion = client.chat.completions.create(model=model_name,
378 + messages=messages,
379 + temperature=0.0,
380 + max_tokens=1,
381 + logprobs=True,
382 + top_logprobs=5)
383 + top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
384 +
385 + results.append({
386 + "test": "text_image",
387 + "logprobs": top_logprobs,
388 + })
389 +
390 + return results
391 +
392 +
393 def compare_two_settings(model: str,
394 arg1: List[str],
395 arg2: List[str],
396 env1: Optional[Dict[str, str]] = None,
397 env2: Optional[Dict[str, str]] = None,
398 *,
304 - method: Literal["generate", "encode"] = "generate",
399 + method: str = "generate",
400 max_wait_seconds: Optional[float] = None) -> None:
401 """
402 Launch API server with two different sets of arguments/environments
@@ -328,7 +423,7 @@ def compare_all_settings(model: str,
423 all_args: List[List[str]],
424 all_envs: List[Optional[Dict[str, str]]],
425 *,
331 - method: Literal["generate", "encode"] = "generate",
426 + method: str = "generate",
427 max_wait_seconds: Optional[float] = None) -> None:
428 """
429 Launch API server with several different sets of arguments/environments
@@ -397,10 +492,17 @@ def compare_all_settings(model: str,
492
493 if method == "generate":
494 results += _test_completion(client, model, prompt, token_ids)
495 + elif method == "generate_close":
496 + results += _test_completion_close(client, model, prompt)
497 + elif method == "generate_with_image":
498 + results += _test_image_text(
499 + client, model,
500 + "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png"
501 + )
502 elif method == "encode":
503 results += _test_embeddings(client, model, prompt)
504 else:
403 - assert_never(method)
505 + raise ValueError(f"Unknown method: {method}")
506
507 if i > 0:
508 # if any setting fails, raise an error early
@@ -410,6 +512,18 @@ def compare_all_settings(model: str,
512 compare_envs = all_envs[i]
513 for ref_result, compare_result in zip(ref_results,
514 compare_results):
515 + ref_result = copy.deepcopy(ref_result)
516 + compare_result = copy.deepcopy(compare_result)
517 + if "embedding" in ref_result and method == "encode":
518 + ref_embedding = torch.tensor(ref_result["embedding"])
519 + compare_embedding = torch.tensor(
520 + compare_result["embedding"])
521 + mse = ((ref_embedding - compare_embedding)**2).mean()
522 + assert mse < 1e-6, (
523 + f"Embedding for {model=} are not the same.\n"
524 + f"mse={mse}\n")
525 + del ref_result["embedding"]
526 + del compare_result["embedding"]
527 assert ref_result == compare_result, (
528 f"Results for {model=} are not the same.\n"
529 f"{ref_args=} {ref_envs=}\n"
vllm/model_executor/models/llava.py
+5 -5
@@ -493,13 +493,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
493 :class:`LlavaImageInputs`
494 """
495 if intermediate_tensors is not None:
496 - input_ids = None
496 inputs_embeds = None
497 else:
499 - # always pass the input via `inputs_embeds`
500 - # to make sure the computation graph is consistent
498 image_input = self._parse_and_validate_image_input(**kwargs)
502 -
499 if image_input is not None:
500 vision_embeddings = self._process_image_input(image_input)
501 inputs_embeds = self.language_model.model.get_input_embeddings(
@@ -511,7 +507,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
507 else:
508 inputs_embeds = self.language_model.model.get_input_embeddings(
509 input_ids)
514 - input_ids = None
510 +
511 + # always pass the input via `inputs_embeds`
512 + # to make sure the computation graph is consistent
513 + # for `torch.compile` integration
514 + input_ids = None
515
516 hidden_states = self.language_model.model(input_ids,
517 positions,
vllm/model_executor/models/phi3v.py
+7 -3
@@ -679,7 +679,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
679 intermediate_tensors: Optional[IntermediateTensors] = None,
680 **kwargs: object):
681 if intermediate_tensors is not None:
682 - input_ids = None
682 inputs_embeds = None
683 else:
684 image_input = self._parse_and_validate_image_input(**kwargs)
@@ -690,9 +689,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
689 inputs_embeds = merge_multimodal_embeddings(
690 input_ids, inputs_embeds, vision_embeddings,
691 self.image_token_id)
693 - input_ids = None
692 else:
695 - inputs_embeds = None
693 + inputs_embeds = self.language_model.model.embed_tokens(
694 + input_ids)
695 +
696 + # always pass the input via `inputs_embeds`
697 + # to make sure the computation graph is consistent
698 + # for `torch.compile` integration
699 + input_ids = None
700
701 hidden_states = self.language_model.model(input_ids,
702 positions,