@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 388ee3de66
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[torch.compile] limit inductor threads and lazy import quant (#10482)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 20, 2024 at 18:36 UTC 388ee3de665c3055fbe610b66ebeef096a23cfe1
11 files changed +178 -64
.buildkite/test-pipeline.yaml
+2
@@ -50,7 +50,9 @@ steps:
50 - tests/multimodal
51 - tests/test_utils
52 - tests/worker
53 + - tests/test_lazy_torch_compile.py
54 commands:
55 + - python3 test_lazy_torch_compile.py
56 - pytest -v -s mq_llm_engine # MQLLMEngine
57 - pytest -v -s async_engine # AsyncLLMEngine
58 - NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
tests/quantization/utils.py
+2 -2
@@ -1,4 +1,4 @@
1 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
1 +from vllm.model_executor.layers.quantization import get_quantization_config
2 from vllm.platforms import current_platform
3
4
@@ -10,6 +10,6 @@ def is_quant_method_supported(quant_method: str) -> bool:
10 capability = current_platform.get_device_capability()
11 assert capability is not None
12
13 - min_capability = QUANTIZATION_METHODS[quant_method].get_min_capability()
13 + min_capability = get_quantization_config(quant_method).get_min_capability()
14
15 return capability.to_int() >= min_capability
tests/test_lazy_torch_compile.py new
+68
@@ -0,0 +1,68 @@
1 +# Description: Test the lazy import module
2 +# The utility function cannot be placed in `vllm.utils`
3 +# this needs to be a standalone script
4 +
5 +import contextlib
6 +import dataclasses
7 +import sys
8 +import traceback
9 +from typing import Callable, Generator
10 +
11 +
12 +@dataclasses.dataclass
13 +class BlameResult:
14 + found: bool = False
15 + trace_stack: str = ""
16 +
17 +
18 +@contextlib.contextmanager
19 +def blame(func: Callable) -> Generator[BlameResult, None, None]:
20 + """
21 + Trace the function calls to find the first function that satisfies the
22 + condition. The trace stack will be stored in the result.
23 +
24 + Usage:
25 +
26 + ```python
27 + with blame(lambda: some_condition()) as result:
28 + # do something
29 +
30 + if result.found:
31 + print(result.trace_stack)
32 + """
33 + result = BlameResult()
34 +
35 + def _trace_calls(frame, event, arg=None):
36 + nonlocal result
37 + if event in ['call', 'return']:
38 + # for every function call or return
39 + try:
40 + # Temporarily disable the trace function
41 + sys.settrace(None)
42 + # check condition here
43 + if not result.found and func():
44 + result.found = True
45 + result.trace_stack = "".join(traceback.format_stack())
46 + # Re-enable the trace function
47 + sys.settrace(_trace_calls)
48 + except NameError:
49 + # modules are deleted during shutdown
50 + pass
51 + return _trace_calls
52 +
53 + sys.settrace(_trace_calls)
54 +
55 + yield result
56 +
57 + sys.settrace(None)
58 +
59 +
60 +module_name = "torch._inductor.async_compile"
61 +
62 +with blame(lambda: module_name in sys.modules) as result:
63 + import vllm # noqa
64 +
65 +assert not result.found, (f"Module {module_name} is already imported, the"
66 + f" first import location is:\n{result.trace_stack}")
67 +
68 +print(f"Module {module_name} is not imported yet")
vllm/_custom_ops.py
-3
@@ -19,9 +19,6 @@ if not current_platform.is_tpu() and not current_platform.is_hpu():
19 except ImportError as e:
20 logger.warning("Failed to import from vllm._C with %r", e)
21
22 -if current_platform.is_rocm():
23 - import vllm._rocm_C # noqa: F401
24 -
22 supports_moe_ops = False
23 with contextlib.suppress(ImportError):
24 import vllm._moe_C # noqa: F401
vllm/config.py
+5 -3
@@ -14,7 +14,8 @@ from transformers import PretrainedConfig
14
15 import vllm.envs as envs
16 from vllm.logger import init_logger
17 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
17 +from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
18 + get_quantization_config)
19 from vllm.model_executor.models import ModelRegistry
20 from vllm.platforms import current_platform
21 from vllm.tracing import is_otel_available, otel_import_error_traceback
@@ -370,7 +371,7 @@ class ModelConfig:
371 return quant_cfg
372
373 def _verify_quantization(self) -> None:
373 - supported_quantization = [*QUANTIZATION_METHODS]
374 + supported_quantization = QUANTIZATION_METHODS
375 rocm_supported_quantization = [
376 "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
377 "fbgemm_fp8"
@@ -392,7 +393,8 @@ class ModelConfig:
393 quant_method = quant_cfg.get("quant_method", "").lower()
394
395 # Detect which checkpoint is it
395 - for _, method in QUANTIZATION_METHODS.items():
396 + for name in QUANTIZATION_METHODS:
397 + method = get_quantization_config(name)
398 quantization_override = method.override_quantization_method(
399 quant_cfg, self.quantization)
400 if quantization_override:
vllm/model_executor/layers/quantization/__init__.py
+73 -51
@@ -1,65 +1,87 @@
1 -from typing import Dict, Type
1 +from typing import Dict, List, Type
2
3 -from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 -from vllm.model_executor.layers.quantization.awq import AWQConfig
5 -from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
3 from vllm.model_executor.layers.quantization.base_config import (
4 QuantizationConfig)
8 -from vllm.model_executor.layers.quantization.bitsandbytes import (
9 - BitsAndBytesConfig)
10 -from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
11 - CompressedTensorsConfig)
12 -from vllm.model_executor.layers.quantization.deepspeedfp import (
13 - DeepSpeedFPConfig)
14 -from vllm.model_executor.layers.quantization.experts_int8 import (
15 - ExpertsInt8Config)
16 -from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
17 -from vllm.model_executor.layers.quantization.fp8 import Fp8Config
18 -from vllm.model_executor.layers.quantization.gguf import GGUFConfig
19 -from vllm.model_executor.layers.quantization.gptq import GPTQConfig
20 -from vllm.model_executor.layers.quantization.gptq_marlin import (
21 - GPTQMarlinConfig)
22 -from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
23 - GPTQMarlin24Config)
24 -from vllm.model_executor.layers.quantization.hqq_marlin import HQQMarlinConfig
25 -from vllm.model_executor.layers.quantization.ipex_quant import IPEXConfig
26 -from vllm.model_executor.layers.quantization.marlin import MarlinConfig
27 -from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config
28 -from vllm.model_executor.layers.quantization.neuron_quant import (
29 - NeuronQuantConfig)
30 -from vllm.model_executor.layers.quantization.qqq import QQQConfig
31 -from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
5
33 -QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
34 - "aqlm": AQLMConfig,
35 - "awq": AWQConfig,
36 - "deepspeedfp": DeepSpeedFPConfig,
37 - "tpu_int8": Int8TpuConfig,
38 - "fp8": Fp8Config,
39 - "fbgemm_fp8": FBGEMMFp8Config,
40 - "modelopt": ModelOptFp8Config,
6 +QUANTIZATION_METHODS: List[str] = [
7 + "aqlm",
8 + "awq",
9 + "deepspeedfp",
10 + "tpu_int8",
11 + "fp8",
12 + "fbgemm_fp8",
13 + "modelopt",
14 # The order of gptq methods is important for config.py iteration over
15 # override_quantization_method(..)
43 - "marlin": MarlinConfig,
44 - "gguf": GGUFConfig,
45 - "gptq_marlin_24": GPTQMarlin24Config,
46 - "gptq_marlin": GPTQMarlinConfig,
47 - "awq_marlin": AWQMarlinConfig,
48 - "gptq": GPTQConfig,
49 - "compressed-tensors": CompressedTensorsConfig,
50 - "bitsandbytes": BitsAndBytesConfig,
51 - "qqq": QQQConfig,
52 - "hqq": HQQMarlinConfig,
53 - "experts_int8": ExpertsInt8Config,
54 - "neuron_quant": NeuronQuantConfig,
55 - "ipex": IPEXConfig,
56 -}
16 + "marlin",
17 + "gguf",
18 + "gptq_marlin_24",
19 + "gptq_marlin",
20 + "awq_marlin",
21 + "gptq",
22 + "compressed-tensors",
23 + "bitsandbytes",
24 + "qqq",
25 + "hqq",
26 + "experts_int8",
27 + "neuron_quant",
28 + "ipex",
29 +]
30
31
32 def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
33 if quantization not in QUANTIZATION_METHODS:
34 raise ValueError(f"Invalid quantization method: {quantization}")
62 - return QUANTIZATION_METHODS[quantization]
35 +
36 + # lazy import to avoid triggering `torch.compile` too early
37 + from .aqlm import AQLMConfig
38 + from .awq import AWQConfig
39 + from .awq_marlin import AWQMarlinConfig
40 + from .bitsandbytes import BitsAndBytesConfig
41 + from .compressed_tensors.compressed_tensors import ( # noqa: E501
42 + CompressedTensorsConfig)
43 + from .deepspeedfp import DeepSpeedFPConfig
44 + from .experts_int8 import ExpertsInt8Config
45 + from .fbgemm_fp8 import FBGEMMFp8Config
46 + from .fp8 import Fp8Config
47 + from .gguf import GGUFConfig
48 + from .gptq import GPTQConfig
49 + from .gptq_marlin import GPTQMarlinConfig
50 + from .gptq_marlin_24 import GPTQMarlin24Config
51 + from .hqq_marlin import HQQMarlinConfig
52 + from .ipex_quant import IPEXConfig
53 + from .marlin import MarlinConfig
54 + from .modelopt import ModelOptFp8Config
55 + from .neuron_quant import NeuronQuantConfig
56 + from .qqq import QQQConfig
57 + from .tpu_int8 import Int8TpuConfig
58 +
59 + method_to_config: Dict[str, Type[QuantizationConfig]] = {
60 + "aqlm": AQLMConfig,
61 + "awq": AWQConfig,
62 + "deepspeedfp": DeepSpeedFPConfig,
63 + "tpu_int8": Int8TpuConfig,
64 + "fp8": Fp8Config,
65 + "fbgemm_fp8": FBGEMMFp8Config,
66 + "modelopt": ModelOptFp8Config,
67 + # The order of gptq methods is important for config.py iteration over
68 + # override_quantization_method(..)
69 + "marlin": MarlinConfig,
70 + "gguf": GGUFConfig,
71 + "gptq_marlin_24": GPTQMarlin24Config,
72 + "gptq_marlin": GPTQMarlinConfig,
73 + "awq_marlin": AWQMarlinConfig,
74 + "gptq": GPTQConfig,
75 + "compressed-tensors": CompressedTensorsConfig,
76 + "bitsandbytes": BitsAndBytesConfig,
77 + "qqq": QQQConfig,
78 + "hqq": HQQMarlinConfig,
79 + "experts_int8": ExpertsInt8Config,
80 + "neuron_quant": NeuronQuantConfig,
81 + "ipex": IPEXConfig,
82 + }
83 +
84 + return method_to_config[quantization]
85
86
87 __all__ = [
vllm/model_executor/models/internvl.py
+2 -2
@@ -19,8 +19,8 @@ from vllm.attention import AttentionMetadata
19 from vllm.config import VllmConfig
20 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 InputContext, token_inputs)
22 -from vllm.model_executor.layers.quantization import (AWQConfig,
23 - QuantizationConfig)
22 +from vllm.model_executor.layers.quantization import QuantizationConfig
23 +from vllm.model_executor.layers.quantization.awq import AWQConfig
24 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
vllm/model_executor/models/qwen2_vl.py
+4 -3
@@ -51,9 +51,10 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51 RowParallelLinear)
52 from vllm.model_executor.layers.logits_processor import LogitsProcessor
53 from vllm.model_executor.layers.pooler import Pooler, PoolingType
54 -from vllm.model_executor.layers.quantization import (GPTQConfig,
55 - GPTQMarlinConfig,
56 - QuantizationConfig)
54 +from vllm.model_executor.layers.quantization import QuantizationConfig
55 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig
56 +from vllm.model_executor.layers.quantization.gptq_marlin import (
57 + GPTQMarlinConfig)
58 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
59 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
60 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
vllm/platforms/cuda.py
+2
@@ -10,6 +10,8 @@ import pynvml
10 import torch
11 from typing_extensions import ParamSpec
12
13 +# import custom ops, trigger op registration
14 +import vllm._C # noqa
15 from vllm.logger import init_logger
16
17 from .interface import DeviceCapability, Platform, PlatformEnum
vllm/platforms/rocm.py
+11
@@ -9,6 +9,17 @@ from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
9
10 logger = init_logger(__name__)
11
12 +try:
13 + import vllm._C # noqa: F401
14 +except ImportError as e:
15 + logger.warning("Failed to import from vllm._C with %r", e)
16 +
17 +# import custom ops, trigger op registration
18 +try:
19 + import vllm._rocm_C # noqa: F401
20 +except ImportError as e:
21 + logger.warning("Failed to import from vllm._rocm_C with %r", e)
22 +
23 if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
24 logger.warning("`fork` method is not supported by ROCm. "
25 "VLLM_WORKER_MULTIPROC_METHOD is overridden to"
vllm/plugins/__init__.py
+9
@@ -1,4 +1,5 @@
1 import logging
2 +import os
3 from contextlib import contextmanager
4 from typing import TYPE_CHECKING, Optional
5
@@ -18,6 +19,14 @@ def load_general_plugins():
19 processes. They should be designed in a way that they can be loaded
20 multiple times without causing issues.
21 """
22 +
23 + # all processes created by vllm will load plugins,
24 + # and here we can inject some common environment variables
25 + # for all processes.
26 +
27 + # see https://github.com/vllm-project/vllm/issues/10480
28 + os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
29 +
30 global plugins_loaded
31 if plugins_loaded:
32 return