@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05d1f8c9c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[misc] move functions to config.py (#10624)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 25, 2024 at 01:27 UTC 05d1f8c9c64b4458ae7cee2650eb97498146ee50
11 files changed +62 -73
tests/compile/piecewise/test_simple.py
+2 -2
@@ -10,8 +10,8 @@ from torch.library import Library
10 from vllm.compilation.compile_context import set_compile_context
11 from vllm.compilation.counter import compilation_counter
12 from vllm.compilation.decorators import support_torch_compile
13 -from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
14 -from vllm.plugins import set_current_vllm_config
13 +from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
14 + set_current_vllm_config)
15 from vllm.utils import direct_register_custom_op
16
17 global_counter = 0
tests/compile/piecewise/test_toy_llama.py
+2 -2
@@ -16,8 +16,8 @@ from torch.library import Library
16 from vllm.compilation.compile_context import set_compile_context
17 from vllm.compilation.counter import compilation_counter
18 from vllm.compilation.decorators import support_torch_compile
19 -from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
20 -from vllm.plugins import set_current_vllm_config
19 +from vllm.config import (CompilationConfig, CompilationLevel, VllmConfig,
20 + set_current_vllm_config)
21 from vllm.utils import direct_register_custom_op
22
23 # create a library to hold the custom op
tests/kernels/test_encoder_decoder_attn.py
+1 -2
@@ -18,10 +18,9 @@ from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
18 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19 from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
20 global_force_attn_backend_context_manager)
21 -from vllm.config import VllmConfig
21 +from vllm.config import VllmConfig, set_current_vllm_config
22 from vllm.forward_context import set_forward_context
23 from vllm.platforms import current_platform
24 -from vllm.plugins import set_current_vllm_config
24
25 # List of support backends for encoder/decoder models
26 LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
tests/model_executor/test_enabled_custom_ops.py
+1 -2
@@ -2,13 +2,12 @@ from typing import List
2
3 import pytest
4
5 -from vllm.config import CompilationConfig, VllmConfig
5 +from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
6 from vllm.model_executor.custom_op import CustomOp
7 from vllm.model_executor.layers.activation import (GeluAndMul,
8 ReLUSquaredActivation,
9 SiluAndMul)
10 from vllm.model_executor.layers.layernorm import RMSNorm
11 -from vllm.plugins import set_current_vllm_config
11
12
13 # Registered subclass for test
vllm/attention/layer.py
+1 -2
@@ -7,13 +7,12 @@ import torch.nn as nn
7 import vllm.envs as envs
8 from vllm.attention import AttentionMetadata, AttentionType
9 from vllm.attention.selector import backend_name_to_enum, get_attn_backend
10 -from vllm.config import CacheConfig
10 +from vllm.config import CacheConfig, get_current_vllm_config
11 from vllm.forward_context import ForwardContext, get_forward_context
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15 from vllm.platforms import current_platform
16 -from vllm.plugins import get_current_vllm_config
16 from vllm.utils import direct_register_custom_op
17
18
vllm/compilation/wrapper.py
+1 -2
@@ -8,7 +8,7 @@ from typing import Callable, List, Optional
8 import torch
9
10 import vllm.envs as envs
11 -from vllm.config import CompilationLevel
11 +from vllm.config import CompilationLevel, get_current_vllm_config
12
13
14 class TorchCompileWrapperWithCustomDispatcher:
@@ -32,7 +32,6 @@ class TorchCompileWrapperWithCustomDispatcher:
32 # default compilation settings
33 # compiling the forward method
34
35 - from vllm.plugins import get_current_vllm_config
35 backend = get_current_vllm_config(
36 ).compilation_config.init_backend()
37
vllm/config.py
+51
@@ -3,6 +3,7 @@ import enum
3 import hashlib
4 import json
5 import warnings
6 +from contextlib import contextmanager
7 from dataclasses import dataclass, field, replace
8 from pathlib import Path
9 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,
@@ -2450,3 +2451,53 @@ class VllmConfig:
2451 self.cache_config.enable_prefix_caching,
2452 self.model_config.use_async_output_proc,
2453 self.model_config.mm_processor_kwargs)
2454 +
2455 +
2456 +_current_vllm_config: Optional[VllmConfig] = None
2457 +
2458 +
2459 +@contextmanager
2460 +def set_current_vllm_config(vllm_config: VllmConfig):
2461 + """
2462 + Temporarily set the current VLLM config.
2463 + Used during model initialization.
2464 + We save the current VLLM config in a global variable,
2465 + so that all modules can access it, e.g. custom ops
2466 + can access the VLLM config to determine how to dispatch.
2467 + """
2468 + global _current_vllm_config
2469 + old_vllm_config = _current_vllm_config
2470 + from vllm.compilation.counter import compilation_counter
2471 + num_models_seen = compilation_counter.num_models_seen
2472 + try:
2473 + _current_vllm_config = vllm_config
2474 + yield
2475 + finally:
2476 + logger.debug("enabled custom ops: %s",
2477 + vllm_config.compilation_config.enabled_custom_ops)
2478 + logger.debug("disabled custom ops: %s",
2479 + vllm_config.compilation_config.disabled_custom_ops)
2480 + if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
2481 + and compilation_counter.num_models_seen == num_models_seen:
2482 + # If the model supports compilation,
2483 + # compilation_counter.num_models_seen should be increased
2484 + # by at least 1.
2485 + # If it is not increased, it means the model does not support
2486 + # compilation (does not have @support_torch_compile decorator).
2487 + logger.warning(
2488 + "`torch.compile` is turned on, but the model %s"
2489 + " does not support it. Please open an issue on GitHub"
2490 + "if you want it to be supported.",
2491 + vllm_config.model_config.model)
2492 + _current_vllm_config = old_vllm_config
2493 +
2494 +
2495 +def get_current_vllm_config() -> VllmConfig:
2496 + if _current_vllm_config is None:
2497 + # in ci, usually when we test custom ops/modules directly,
2498 + # we don't set the vllm config. In that case, we set a default
2499 + # config.
2500 + logger.warning("Current VLLM config is not set.")
2501 + from vllm.config import VllmConfig
2502 + return VllmConfig()
2503 + return _current_vllm_config
vllm/model_executor/custom_op.py
+1 -1
@@ -2,9 +2,9 @@ from typing import Dict, Type
2
3 import torch.nn as nn
4
5 +from vllm.config import get_current_vllm_config
6 from vllm.logger import init_logger
7 from vllm.platforms import current_platform
7 -from vllm.plugins import get_current_vllm_config
8 from vllm.utils import print_warning_once
9
10 logger = init_logger(__name__)
vllm/model_executor/model_loader/loader.py
+1 -2
@@ -23,7 +23,7 @@ from transformers import AutoModelForCausalLM
23 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
24
25 from vllm.config import (LoadConfig, LoadFormat, ModelConfig, ParallelConfig,
26 - VllmConfig)
26 + VllmConfig, set_current_vllm_config)
27 from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
@@ -47,7 +47,6 @@ from vllm.model_executor.model_loader.weight_utils import (
47 safetensors_weights_iterator)
48 from vllm.model_executor.utils import set_weight_attrs
49 from vllm.platforms import current_platform
50 -from vllm.plugins import set_current_vllm_config
50 from vllm.utils import is_pin_memory_available
51
52
vllm/model_executor/model_loader/tensorizer.py
+1 -2
@@ -13,13 +13,12 @@ from torch import nn
13 from transformers import PretrainedConfig
14
15 import vllm.envs as envs
16 -from vllm.config import ModelConfig, ParallelConfig
16 +from vllm.config import ModelConfig, ParallelConfig, set_current_vllm_config
17 from vllm.engine.arg_utils import EngineArgs
18 from vllm.engine.llm_engine import LLMEngine
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
22 -from vllm.plugins import set_current_vllm_config
22 from vllm.utils import FlexibleArgumentParser
23
24 tensorizer_error_msg = None
vllm/plugins/__init__.py
-56
@@ -1,15 +1,10 @@
1 import logging
2 import os
3 -from contextlib import contextmanager
4 -from typing import TYPE_CHECKING, Optional
3
4 import torch
5
6 import vllm.envs as envs
7
10 -if TYPE_CHECKING:
11 - from vllm.config import VllmConfig
12 -
8 logger = logging.getLogger(__name__)
9
10 # make sure one process only loads plugins once
@@ -64,54 +59,3 @@ def load_general_plugins():
59 logger.info("plugin %s loaded.", plugin.name)
60 except Exception:
61 logger.exception("Failed to load plugin %s", plugin.name)
67 -
68 -
69 -_current_vllm_config: Optional["VllmConfig"] = None
70 -
71 -
72 -@contextmanager
73 -def set_current_vllm_config(vllm_config: "VllmConfig"):
74 - """
75 - Temporarily set the current VLLM config.
76 - Used during model initialization.
77 - We save the current VLLM config in a global variable,
78 - so that all modules can access it, e.g. custom ops
79 - can access the VLLM config to determine how to dispatch.
80 - """
81 - global _current_vllm_config
82 - old_vllm_config = _current_vllm_config
83 - from vllm.compilation.counter import compilation_counter
84 - from vllm.config import CompilationLevel
85 - num_models_seen = compilation_counter.num_models_seen
86 - try:
87 - _current_vllm_config = vllm_config
88 - yield
89 - finally:
90 - logger.debug("enabled custom ops: %s",
91 - vllm_config.compilation_config.enabled_custom_ops)
92 - logger.debug("disabled custom ops: %s",
93 - vllm_config.compilation_config.disabled_custom_ops)
94 - if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
95 - and compilation_counter.num_models_seen == num_models_seen:
96 - # If the model supports compilation,
97 - # compilation_counter.num_models_seen should be increased
98 - # by at least 1.
99 - # If it is not increased, it means the model does not support
100 - # compilation (does not have @support_torch_compile decorator).
101 - logger.warning(
102 - "`torch.compile` is turned on, but the model %s"
103 - " does not support it. Please open an issue on GitHub"
104 - "if you want it to be supported.",
105 - vllm_config.model_config.model)
106 - _current_vllm_config = old_vllm_config
107 -
108 -
109 -def get_current_vllm_config() -> "VllmConfig":
110 - if _current_vllm_config is None:
111 - # in ci, usually when we test custom ops/modules directly,
112 - # we don't set the vllm config. In that case, we set a default
113 - # config.
114 - logger.warning("Current VLLM config is not set.")
115 - from vllm.config import VllmConfig
116 - return VllmConfig()
117 - return _current_vllm_config