A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[torch.compile] limit inductor threads and lazy import quant (#10482)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 20, 2024 at 18:36 UTC
388ee3de665c3055fbe610b66ebeef096a23cfe1
11 files changed
+178
-64
.buildkite/test-pipeline.yaml
+2
@@ -50,7 +50,9 @@ steps:
50
- tests/multimodal
51
- tests/test_utils
52
- tests/worker
53
+ - tests/test_lazy_torch_compile.py
54
commands:
55
+ - python3 test_lazy_torch_compile.py
56
- pytest -v -s mq_llm_engine # MQLLMEngine
57
- pytest -v -s async_engine # AsyncLLMEngine
58
- NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
tests/quantization/utils.py
+2
-2
@@ -1,4 +1,4 @@
1
-from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
1
+from vllm.model_executor.layers.quantization import get_quantization_config
2
from vllm.platforms import current_platform
3
4
@@ -10,6 +10,6 @@ def is_quant_method_supported(quant_method: str) -> bool:
10
capability = current_platform.get_device_capability()
11
assert capability is not None
12
13
- min_capability = QUANTIZATION_METHODS[quant_method].get_min_capability()
13
+ min_capability = get_quantization_config(quant_method).get_min_capability()
14
15
return capability.to_int() >= min_capability
tests/test_lazy_torch_compile.py
new
+68
@@ -0,0 +1,68 @@
1
+# Description: Test the lazy import module
2
+# The utility function cannot be placed in `vllm.utils`
3
+# this needs to be a standalone script
4
+
5
+import contextlib
6
+import dataclasses
7
+import sys
8
+import traceback
9
+from typing import Callable, Generator
10
+
11
+
12
+@dataclasses.dataclass
13
+class BlameResult:
14
+ found: bool = False
15
+ trace_stack: str = ""
16
+
17
+
18
+@contextlib.contextmanager
19
+def blame(func: Callable) -> Generator[BlameResult, None, None]:
20
+ """
21
+ Trace the function calls to find the first function that satisfies the
22
+ condition. The trace stack will be stored in the result.
23
+
24
+ Usage:
25
+
26
+ ```python
27
+ with blame(lambda: some_condition()) as result:
28
+ # do something
29
+
30
+ if result.found:
31
+ print(result.trace_stack)
32
+ """
33
+ result = BlameResult()
34
+
35
+ def _trace_calls(frame, event, arg=None):
36
+ nonlocal result
37
+ if event in ['call', 'return']:
38
+ # for every function call or return
39
+ try:
40
+ # Temporarily disable the trace function
41
+ sys.settrace(None)
42
+ # check condition here
43
+ if not result.found and func():
44
+ result.found = True
45
+ result.trace_stack = "".join(traceback.format_stack())
46
+ # Re-enable the trace function
47
+ sys.settrace(_trace_calls)
48
+ except NameError:
49
+ # modules are deleted during shutdown
50
+ pass
51
+ return _trace_calls
52
+
53
+ sys.settrace(_trace_calls)
54
+
55
+ yield result
56
+
57
+ sys.settrace(None)
58
+
59
+
60
+module_name = "torch._inductor.async_compile"
61
+
62
+with blame(lambda: module_name in sys.modules) as result:
63
+ import vllm # noqa
64
+
65
+assert not result.found, (f"Module {module_name} is already imported, the"
66
+ f" first import location is:\n{result.trace_stack}")
67
+
68
+print(f"Module {module_name} is not imported yet")
vllm/_custom_ops.py
-3
@@ -19,9 +19,6 @@ if not current_platform.is_tpu() and not current_platform.is_hpu():
19
except ImportError as e:
20
logger.warning("Failed to import from vllm._C with %r", e)
21
22
-if current_platform.is_rocm():
23
- import vllm._rocm_C # noqa: F401
24
-
22
supports_moe_ops = False
23
with contextlib.suppress(ImportError):
24
import vllm._moe_C # noqa: F401
vllm/config.py
+5
-3
@@ -14,7 +14,8 @@ from transformers import PretrainedConfig
14
15
import vllm.envs as envs
16
from vllm.logger import init_logger
17
-from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
17
+from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
18
+ get_quantization_config)
19
from vllm.model_executor.models import ModelRegistry
20
from vllm.platforms import current_platform
21
from vllm.tracing import is_otel_available, otel_import_error_traceback
@@ -370,7 +371,7 @@ class ModelConfig:
371
return quant_cfg
372
373
def _verify_quantization(self) -> None:
373
- supported_quantization = [*QUANTIZATION_METHODS]
374
+ supported_quantization = QUANTIZATION_METHODS
375
rocm_supported_quantization = [
376
"awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
377
"fbgemm_fp8"
@@ -392,7 +393,8 @@ class ModelConfig:
393
quant_method = quant_cfg.get("quant_method", "").lower()
394
395
# Detect which checkpoint is it
395
- for _, method in QUANTIZATION_METHODS.items():
396
+ for name in QUANTIZATION_METHODS:
397
+ method = get_quantization_config(name)
398
quantization_override = method.override_quantization_method(
399
quant_cfg, self.quantization)
400
if quantization_override:
vllm/model_executor/layers/quantization/__init__.py
+73
-51
@@ -1,65 +1,87 @@
1
-from typing import Dict, Type
1
+from typing import Dict, List, Type
2
3
-from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
-from vllm.model_executor.layers.quantization.awq import AWQConfig
5
-from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
3
from vllm.model_executor.layers.quantization.base_config import (
4
QuantizationConfig)
8
-from vllm.model_executor.layers.quantization.bitsandbytes import (
9
- BitsAndBytesConfig)
10
-from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11
- CompressedTensorsConfig)
12
-from vllm.model_executor.layers.quantization.deepspeedfp import (
13
- DeepSpeedFPConfig)
14
-from vllm.model_executor.layers.quantization.experts_int8 import (
15
- ExpertsInt8Config)
16
-from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
17
-from vllm.model_executor.layers.quantization.fp8 import Fp8Config
18
-from vllm.model_executor.layers.quantization.gguf import GGUFConfig
19
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
20
-from vllm.model_executor.layers.quantization.gptq_marlin import (
21
- GPTQMarlinConfig)
22
-from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
23
- GPTQMarlin24Config)
24
-from vllm.model_executor.layers.quantization.hqq_marlin import HQQMarlinConfig
25
-from vllm.model_executor.layers.quantization.ipex_quant import IPEXConfig
26
-from vllm.model_executor.layers.quantization.marlin import MarlinConfig
27
-from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config
28
-from vllm.model_executor.layers.quantization.neuron_quant import (
29
- NeuronQuantConfig)
30
-from vllm.model_executor.layers.quantization.qqq import QQQConfig
31
-from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
5
33
-QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
34
- "aqlm": AQLMConfig,
35
- "awq": AWQConfig,
36
- "deepspeedfp": DeepSpeedFPConfig,
37
- "tpu_int8": Int8TpuConfig,
38
- "fp8": Fp8Config,
39
- "fbgemm_fp8": FBGEMMFp8Config,
40
- "modelopt": ModelOptFp8Config,
6
+QUANTIZATION_METHODS: List[str] = [
7
+ "aqlm",
8
+ "awq",
9
+ "deepspeedfp",
10
+ "tpu_int8",
11
+ "fp8",
12
+ "fbgemm_fp8",
13
+ "modelopt",
14
# The order of gptq methods is important for config.py iteration over
15
# override_quantization_method(..)
43
- "marlin": MarlinConfig,
44
- "gguf": GGUFConfig,
45
- "gptq_marlin_24": GPTQMarlin24Config,
46
- "gptq_marlin": GPTQMarlinConfig,
47
- "awq_marlin": AWQMarlinConfig,
48
- "gptq": GPTQConfig,
49
- "compressed-tensors": CompressedTensorsConfig,
50
- "bitsandbytes": BitsAndBytesConfig,
51
- "qqq": QQQConfig,
52
- "hqq": HQQMarlinConfig,
53
- "experts_int8": ExpertsInt8Config,
54
- "neuron_quant": NeuronQuantConfig,
55
- "ipex": IPEXConfig,
56
-}
16
+ "marlin",
17
+ "gguf",
18
+ "gptq_marlin_24",
19
+ "gptq_marlin",
20
+ "awq_marlin",
21
+ "gptq",
22
+ "compressed-tensors",
23
+ "bitsandbytes",
24
+ "qqq",
25
+ "hqq",
26
+ "experts_int8",
27
+ "neuron_quant",
28
+ "ipex",
29
+]
30
31
32
def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
33
if quantization not in QUANTIZATION_METHODS:
34
raise ValueError(f"Invalid quantization method: {quantization}")
62
- return QUANTIZATION_METHODS[quantization]
35
+
36
+ # lazy import to avoid triggering `torch.compile` too early
37
+ from .aqlm import AQLMConfig
38
+ from .awq import AWQConfig
39
+ from .awq_marlin import AWQMarlinConfig
40
+ from .bitsandbytes import BitsAndBytesConfig
41
+ from .compressed_tensors.compressed_tensors import ( # noqa: E501
42
+ CompressedTensorsConfig)
43
+ from .deepspeedfp import DeepSpeedFPConfig
44
+ from .experts_int8 import ExpertsInt8Config
45
+ from .fbgemm_fp8 import FBGEMMFp8Config
46
+ from .fp8 import Fp8Config
47
+ from .gguf import GGUFConfig
48
+ from .gptq import GPTQConfig
49
+ from .gptq_marlin import GPTQMarlinConfig
50
+ from .gptq_marlin_24 import GPTQMarlin24Config
51
+ from .hqq_marlin import HQQMarlinConfig
52
+ from .ipex_quant import IPEXConfig
53
+ from .marlin import MarlinConfig
54
+ from .modelopt import ModelOptFp8Config
55
+ from .neuron_quant import NeuronQuantConfig
56
+ from .qqq import QQQConfig
57
+ from .tpu_int8 import Int8TpuConfig
58
+
59
+ method_to_config: Dict[str, Type[QuantizationConfig]] = {
60
+ "aqlm": AQLMConfig,
61
+ "awq": AWQConfig,
62
+ "deepspeedfp": DeepSpeedFPConfig,
63
+ "tpu_int8": Int8TpuConfig,
64
+ "fp8": Fp8Config,
65
+ "fbgemm_fp8": FBGEMMFp8Config,
66
+ "modelopt": ModelOptFp8Config,
67
+ # The order of gptq methods is important for config.py iteration over
68
+ # override_quantization_method(..)
69
+ "marlin": MarlinConfig,
70
+ "gguf": GGUFConfig,
71
+ "gptq_marlin_24": GPTQMarlin24Config,
72
+ "gptq_marlin": GPTQMarlinConfig,
73
+ "awq_marlin": AWQMarlinConfig,
74
+ "gptq": GPTQConfig,
75
+ "compressed-tensors": CompressedTensorsConfig,
76
+ "bitsandbytes": BitsAndBytesConfig,
77
+ "qqq": QQQConfig,
78
+ "hqq": HQQMarlinConfig,
79
+ "experts_int8": ExpertsInt8Config,
80
+ "neuron_quant": NeuronQuantConfig,
81
+ "ipex": IPEXConfig,
82
+ }
83
+
84
+ return method_to_config[quantization]
85
86
87
__all__ = [
vllm/model_executor/models/internvl.py
+2
-2
@@ -19,8 +19,8 @@ from vllm.attention import AttentionMetadata
19
from vllm.config import VllmConfig
20
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
InputContext, token_inputs)
22
-from vllm.model_executor.layers.quantization import (AWQConfig,
23
- QuantizationConfig)
22
+from vllm.model_executor.layers.quantization import QuantizationConfig
23
+from vllm.model_executor.layers.quantization.awq import AWQConfig
24
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
vllm/model_executor/models/qwen2_vl.py
+4
-3
@@ -51,9 +51,10 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51
RowParallelLinear)
52
from vllm.model_executor.layers.logits_processor import LogitsProcessor
53
from vllm.model_executor.layers.pooler import Pooler, PoolingType
54
-from vllm.model_executor.layers.quantization import (GPTQConfig,
55
- GPTQMarlinConfig,
56
- QuantizationConfig)
54
+from vllm.model_executor.layers.quantization import QuantizationConfig
55
+from vllm.model_executor.layers.quantization.gptq import GPTQConfig
56
+from vllm.model_executor.layers.quantization.gptq_marlin import (
57
+ GPTQMarlinConfig)
58
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
59
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
60
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
vllm/platforms/cuda.py
+2
@@ -10,6 +10,8 @@ import pynvml
10
import torch
11
from typing_extensions import ParamSpec
12
13
+# import custom ops, trigger op registration
14
+import vllm._C # noqa
15
from vllm.logger import init_logger
16
17
from .interface import DeviceCapability, Platform, PlatformEnum
vllm/platforms/rocm.py
+11
@@ -9,6 +9,17 @@ from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
9
10
logger = init_logger(__name__)
11
12
+try:
13
+ import vllm._C # noqa: F401
14
+except ImportError as e:
15
+ logger.warning("Failed to import from vllm._C with %r", e)
16
+
17
+# import custom ops, trigger op registration
18
+try:
19
+ import vllm._rocm_C # noqa: F401
20
+except ImportError as e:
21
+ logger.warning("Failed to import from vllm._rocm_C with %r", e)
22
+
23
if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
24
logger.warning("`fork` method is not supported by ROCm. "
25
"VLLM_WORKER_MULTIPROC_METHOD is overridden to"
vllm/plugins/__init__.py
+9
@@ -1,4 +1,5 @@
1
import logging
2
+import os
3
from contextlib import contextmanager
4
from typing import TYPE_CHECKING, Optional
5
@@ -18,6 +19,14 @@ def load_general_plugins():
19
processes. They should be designed in a way that they can be loaded
20
multiple times without causing issues.
21
"""
22
+
23
+ # all processes created by vllm will load plugins,
24
+ # and here we can inject some common environment variables
25
+ # for all processes.
26
+
27
+ # see https://github.com/vllm-project/vllm/issues/10480
28
+ os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
29
+
30
global plugins_loaded
31
if plugins_loaded:
32
return