A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[misc] move functions to config.py (#10624)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 25, 2024 at 01:27 UTC
05d1f8c9c64b4458ae7cee2650eb97498146ee50
11 files changed
+62
-73
tests/compile/piecewise/test_simple.py
+2
-2
@@ -10,8 +10,8 @@ from torch.library import Library
10
from vllm.compilation.compile_context import set_compile_context
11
from vllm.compilation.counter import compilation_counter
12
from vllm.compilation.decorators import support_torch_compile
13
-from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
14
-from vllm.plugins import set_current_vllm_config
13
+from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
14
+ set_current_vllm_config)
15
from vllm.utils import direct_register_custom_op
16
17
global_counter = 0
tests/compile/piecewise/test_toy_llama.py
+2
-2
@@ -16,8 +16,8 @@ from torch.library import Library
16
from vllm.compilation.compile_context import set_compile_context
17
from vllm.compilation.counter import compilation_counter
18
from vllm.compilation.decorators import support_torch_compile
19
-from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
20
-from vllm.plugins import set_current_vllm_config
19
+from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
20
+ set_current_vllm_config)
21
from vllm.utils import direct_register_custom_op
22
23
# create a library to hold the custom op
tests/kernels/test_encoder_decoder_attn.py
+1
-2
@@ -18,10 +18,9 @@ from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
18
from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19
from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
20
global_force_attn_backend_context_manager)
21
-from vllm.config import VllmConfig
21
+from vllm.config import VllmConfig, set_current_vllm_config
22
from vllm.forward_context import set_forward_context
23
from vllm.platforms import current_platform
24
-from vllm.plugins import set_current_vllm_config
24
25
# List of support backends for encoder/decoder models
26
LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
tests/model_executor/test_enabled_custom_ops.py
+1
-2
@@ -2,13 +2,12 @@ from typing import List
2
3
import pytest
4
5
-from vllm.config import CompilationConfig, VllmConfig
5
+from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
6
from vllm.model_executor.custom_op import CustomOp
7
from vllm.model_executor.layers.activation import (GeluAndMul,
8
ReLUSquaredActivation,
9
SiluAndMul)
10
from vllm.model_executor.layers.layernorm import RMSNorm
11
-from vllm.plugins import set_current_vllm_config
11
12
13
# Registered subclass for test
vllm/attention/layer.py
+1
-2
@@ -7,13 +7,12 @@ import torch.nn as nn
7
import vllm.envs as envs
8
from vllm.attention import AttentionMetadata, AttentionType
9
from vllm.attention.selector import backend_name_to_enum, get_attn_backend
10
-from vllm.config import CacheConfig
10
+from vllm.config import CacheConfig, get_current_vllm_config
11
from vllm.forward_context import ForwardContext, get_forward_context
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15
from vllm.platforms import current_platform
16
-from vllm.plugins import get_current_vllm_config
16
from vllm.utils import direct_register_custom_op
17
18
vllm/compilation/wrapper.py
+1
-2
@@ -8,7 +8,7 @@ from typing import Callable, List, Optional
8
import torch
9
10
import vllm.envs as envs
11
-from vllm.config import CompilationLevel
11
+from vllm.config import CompilationLevel, get_current_vllm_config
12
13
14
class TorchCompileWrapperWithCustomDispatcher:
@@ -32,7 +32,6 @@ class TorchCompileWrapperWithCustomDispatcher:
32
# default compilation settings
33
# compiling the forward method
34
35
- from vllm.plugins import get_current_vllm_config
35
backend = get_current_vllm_config(
36
).compilation_config.init_backend()
37
vllm/config.py
+51
@@ -3,6 +3,7 @@ import enum
3
import hashlib
4
import json
5
import warnings
6
+from contextlib import contextmanager
7
from dataclasses import dataclass, field, replace
8
from pathlib import Path
9
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,
@@ -2450,3 +2451,53 @@ class VllmConfig:
2451
self.cache_config.enable_prefix_caching,
2452
self.model_config.use_async_output_proc,
2453
self.model_config.mm_processor_kwargs)
2454
+
2455
+
2456
+_current_vllm_config: Optional[VllmConfig] = None
2457
+
2458
+
2459
+@contextmanager
2460
+def set_current_vllm_config(vllm_config: VllmConfig):
2461
+ """
2462
+ Temporarily set the current VLLM config.
2463
+ Used during model initialization.
2464
+ We save the current VLLM config in a global variable,
2465
+ so that all modules can access it, e.g. custom ops
2466
+ can access the VLLM config to determine how to dispatch.
2467
+ """
2468
+ global _current_vllm_config
2469
+ old_vllm_config = _current_vllm_config
2470
+ from vllm.compilation.counter import compilation_counter
2471
+ num_models_seen = compilation_counter.num_models_seen
2472
+ try:
2473
+ _current_vllm_config = vllm_config
2474
+ yield
2475
+ finally:
2476
+ logger.debug("enabled custom ops: %s",
2477
+ vllm_config.compilation_config.enabled_custom_ops)
2478
+ logger.debug("disabled custom ops: %s",
2479
+ vllm_config.compilation_config.disabled_custom_ops)
2480
+ if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
2481
+ and compilation_counter.num_models_seen == num_models_seen:
2482
+ # If the model supports compilation,
2483
+ # compilation_counter.num_models_seen should be increased
2484
+ # by at least 1.
2485
+ # If it is not increased, it means the model does not support
2486
+ # compilation (does not have @support_torch_compile decorator).
2487
+ logger.warning(
2488
+ "`torch.compile` is turned on, but the model %s"
2489
+ " does not support it. Please open an issue on GitHub"
2490
+ "if you want it to be supported.",
2491
+ vllm_config.model_config.model)
2492
+ _current_vllm_config = old_vllm_config
2493
+
2494
+
2495
+def get_current_vllm_config() -> VllmConfig:
2496
+ if _current_vllm_config is None:
2497
+ # in ci, usually when we test custom ops/modules directly,
2498
+ # we don't set the vllm config. In that case, we set a default
2499
+ # config.
2500
+ logger.warning("Current VLLM config is not set.")
2501
+ from vllm.config import VllmConfig
2502
+ return VllmConfig()
2503
+ return _current_vllm_config
vllm/model_executor/custom_op.py
+1
-1
@@ -2,9 +2,9 @@ from typing import Dict, Type
2
3
import torch.nn as nn
4
5
+from vllm.config import get_current_vllm_config
6
from vllm.logger import init_logger
7
from vllm.platforms import current_platform
7
-from vllm.plugins import get_current_vllm_config
8
from vllm.utils import print_warning_once
9
10
logger = init_logger(__name__)
vllm/model_executor/model_loader/loader.py
+1
-2
@@ -23,7 +23,7 @@ from transformers import AutoModelForCausalLM
23
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
24
25
from vllm.config import (LoadConfig, LoadFormat, ModelConfig, ParallelConfig,
26
- VllmConfig)
26
+ VllmConfig, set_current_vllm_config)
27
from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.envs import VLLM_USE_MODELSCOPE
@@ -47,7 +47,6 @@ from vllm.model_executor.model_loader.weight_utils import (
47
safetensors_weights_iterator)
48
from vllm.model_executor.utils import set_weight_attrs
49
from vllm.platforms import current_platform
50
-from vllm.plugins import set_current_vllm_config
50
from vllm.utils import is_pin_memory_available
51
52
vllm/model_executor/model_loader/tensorizer.py
+1
-2
@@ -13,13 +13,12 @@ from torch import nn
13
from transformers import PretrainedConfig
14
15
import vllm.envs as envs
16
-from vllm.config import ModelConfig, ParallelConfig
16
+from vllm.config import ModelConfig, ParallelConfig, set_current_vllm_config
17
from vllm.engine.arg_utils import EngineArgs
18
from vllm.engine.llm_engine import LLMEngine
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
VocabParallelEmbedding)
22
-from vllm.plugins import set_current_vllm_config
22
from vllm.utils import FlexibleArgumentParser
23
24
tensorizer_error_msg = None
vllm/plugins/__init__.py
-56
@@ -1,15 +1,10 @@
1
import logging
2
import os
3
-from contextlib import contextmanager
4
-from typing import TYPE_CHECKING, Optional
3
4
import torch
5
6
import vllm.envs as envs
7
10
-if TYPE_CHECKING:
11
- from vllm.config import VllmConfig
12
-
8
logger = logging.getLogger(__name__)
9
10
# make sure one process only loads plugins once
@@ -64,54 +59,3 @@ def load_general_plugins():
59
logger.info("plugin %s loaded.", plugin.name)
60
except Exception:
61
logger.exception("Failed to load plugin %s", plugin.name)
67
-
68
-
69
-_current_vllm_config: Optional["VllmConfig"] = None
70
-
71
-
72
-@contextmanager
73
-def set_current_vllm_config(vllm_config: "VllmConfig"):
74
- """
75
- Temporarily set the current VLLM config.
76
- Used during model initialization.
77
- We save the current VLLM config in a global variable,
78
- so that all modules can access it, e.g. custom ops
79
- can access the VLLM config to determine how to dispatch.
80
- """
81
- global _current_vllm_config
82
- old_vllm_config = _current_vllm_config
83
- from vllm.compilation.counter import compilation_counter
84
- from vllm.config import CompilationLevel
85
- num_models_seen = compilation_counter.num_models_seen
86
- try:
87
- _current_vllm_config = vllm_config
88
- yield
89
- finally:
90
- logger.debug("enabled custom ops: %s",
91
- vllm_config.compilation_config.enabled_custom_ops)
92
- logger.debug("disabled custom ops: %s",
93
- vllm_config.compilation_config.disabled_custom_ops)
94
- if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
95
- and compilation_counter.num_models_seen == num_models_seen:
96
- # If the model supports compilation,
97
- # compilation_counter.num_models_seen should be increased
98
- # by at least 1.
99
- # If it is not increased, it means the model does not support
100
- # compilation (does not have @support_torch_compile decorator).
101
- logger.warning(
102
- "`torch.compile` is turned on, but the model %s"
103
- " does not support it. Please open an issue on GitHub"
104
- "if you want it to be supported.",
105
- vllm_config.model_config.model)
106
- _current_vllm_config = old_vllm_config
107
-
108
-
109
-def get_current_vllm_config() -> "VllmConfig":
110
- if _current_vllm_config is None:
111
- # in ci, usually when we test custom ops/modules directly,
112
- # we don't set the vllm config. In that case, we set a default
113
- # config.
114
- logger.warning("Current VLLM config is not set.")
115
- from vllm.config import VllmConfig
116
- return VllmConfig()
117
- return _current_vllm_config