A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[v1] EngineArgs for better config handling for v1 (#10382)
Signed-off-by: rickyx <rickyx@anyscale.com>
Ricky Xu committed
Nov 25, 2024 at 21:09 UTC
519e8e4182af8e25d78b062ba5e613df661e6e5d
13 files changed
+109
-27
.buildkite/test-pipeline.yaml
+1
-1
@@ -172,7 +172,7 @@ steps:
172
- vllm/
173
- tests/v1
174
commands:
175
- - pytest -v -s v1
175
+ - VLLM_USE_V1=1 pytest -v -s v1
176
177
- label: Examples Test # 15min
178
working_dir: "/vllm-workspace/examples"
tests/v1/engine/test_async_llm.py
+3
@@ -32,6 +32,9 @@ async def generate(engine: AsyncLLM, request_id: str,
32
33
@pytest.mark.asyncio
34
async def test_load(monkeypatch):
35
+ # TODO(rickyx): Remove monkeypatch once we have a better way to test V1
36
+ # so that in the future when we switch, we don't have to change all the
37
+ # tests.
38
with monkeypatch.context() as m:
39
m.setenv("VLLM_USE_V1", "1")
40
tests/v1/engine/test_engine_args.py
new
+42
@@ -0,0 +1,42 @@
1
+import pytest
2
+
3
+from vllm import envs
4
+from vllm.config import VllmConfig
5
+from vllm.engine.arg_utils import EngineArgs
6
+from vllm.usage.usage_lib import UsageContext
7
+
8
+if not envs.VLLM_USE_V1:
9
+ pytest.skip(
10
+ "Skipping V1 tests. Rerun with `VLLM_USE_V1=1` to test.",
11
+ allow_module_level=True,
12
+ )
13
+
14
+
15
+def test_defaults():
16
+ engine_args = EngineArgs(model="facebook/opt-125m")
17
+
18
+ # Assert V1 defaults
19
+ assert (engine_args.enable_prefix_caching
20
+ ), "V1 turns on prefix caching by default"
21
+
22
+
23
+def test_defaults_with_usage_context():
24
+ engine_args = EngineArgs(model="facebook/opt-125m")
25
+ vllm_config: VllmConfig = engine_args.create_engine_config(
26
+ UsageContext.LLM_CLASS)
27
+
28
+ assert vllm_config.scheduler_config.max_num_seqs == 1024
29
+ assert vllm_config.scheduler_config.max_num_batched_tokens == 8192
30
+
31
+ engine_args = EngineArgs(model="facebook/opt-125m")
32
+ vllm_config = engine_args.create_engine_config(
33
+ UsageContext.OPENAI_API_SERVER)
34
+ assert vllm_config.scheduler_config.max_num_seqs == 1024
35
+ assert vllm_config.scheduler_config.max_num_batched_tokens == 2048
36
+
37
+
38
+def test_prefix_cache_disabled_with_multimodel():
39
+ engine_args = EngineArgs(model="llava-hf/llava-1.5-7b-hf")
40
+
41
+ vllm_config = engine_args.create_engine_config(UsageContext.LLM_CLASS)
42
+ assert not vllm_config.cache_config.enable_prefix_caching
tests/v1/engine/test_engine_core.py
+2
-1
@@ -43,7 +43,8 @@ def test_engine_core(monkeypatch):
43
m.setenv("VLLM_USE_V1", "1")
44
"""Setup the EngineCore."""
45
engine_args = EngineArgs(model=MODEL_NAME)
46
- vllm_config = engine_args.create_engine_config()
46
+ vllm_config = engine_args.create_engine_config(
47
+ usage_context=UsageContext.UNKNOWN_CONTEXT)
48
executor_class = AsyncLLM._get_executor_cls(vllm_config)
49
50
engine_core = EngineCore(vllm_config=vllm_config,
tests/v1/engine/test_engine_core_client.py
+4
-2
@@ -82,7 +82,8 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
82
m.setenv("VLLM_USE_V1", "1")
83
84
engine_args = EngineArgs(model=MODEL_NAME, compilation_config=3)
85
- vllm_config = engine_args.create_engine_config()
85
+ vllm_config = engine_args.create_engine_config(
86
+ UsageContext.UNKNOWN_CONTEXT)
87
executor_class = AsyncLLM._get_executor_cls(vllm_config)
88
client = EngineCoreClient.make_client(
89
vllm_config,
@@ -153,7 +154,8 @@ async def test_engine_core_client_asyncio(monkeypatch):
154
m.setenv("VLLM_USE_V1", "1")
155
156
engine_args = EngineArgs(model=MODEL_NAME)
156
- vllm_config = engine_args.create_engine_config()
157
+ vllm_config = engine_args.create_engine_config(
158
+ usage_context=UsageContext.UNKNOWN_CONTEXT)
159
executor_class = AsyncLLM._get_executor_cls(vllm_config)
160
client = EngineCoreClient.make_client(
161
vllm_config,
vllm/engine/arg_utils.py
+50
-3
@@ -20,6 +20,7 @@ from vllm.logger import init_logger
20
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21
from vllm.platforms import current_platform
22
from vllm.transformers_utils.utils import check_gguf_file
23
+from vllm.usage.usage_lib import UsageContext
24
from vllm.utils import FlexibleArgumentParser, StoreBoolean
25
26
if TYPE_CHECKING:
@@ -113,7 +114,7 @@ class EngineArgs:
114
# NOTE(kzawora): default block size for Gaudi should be 128
115
# smaller sizes still work, but very inefficiently
116
block_size: int = 16 if not current_platform.is_hpu() else 128
116
- enable_prefix_caching: bool = False
117
+ enable_prefix_caching: Optional[bool] = None
118
disable_sliding_window: bool = False
119
use_v2_block_manager: bool = True
120
swap_space: float = 4 # GiB
@@ -197,6 +198,11 @@ class EngineArgs:
198
if not self.tokenizer:
199
self.tokenizer = self.model
200
201
+ # Override the default value of enable_prefix_caching if it's not set
202
+ # by user.
203
+ if self.enable_prefix_caching is None:
204
+ self.enable_prefix_caching = bool(envs.VLLM_USE_V1)
205
+
206
# support `EngineArgs(compilation_config={...})`
207
# without having to manually construct a
208
# CompilationConfig object
@@ -953,7 +959,12 @@ class EngineArgs:
959
ignore_patterns=self.ignore_patterns,
960
)
961
956
- def create_engine_config(self) -> VllmConfig:
962
+ def create_engine_config(self,
963
+ usage_context: Optional[UsageContext] = None
964
+ ) -> VllmConfig:
965
+ if envs.VLLM_USE_V1:
966
+ self._override_v1_engine_args(usage_context)
967
+
968
# gguf file needs a specific model loader and doesn't use hf_repo
969
if check_gguf_file(self.model):
970
self.quantization = self.load_format = "gguf"
@@ -1170,7 +1181,7 @@ class EngineArgs:
1181
or "all" in detailed_trace_modules,
1182
)
1183
1173
- return VllmConfig(
1184
+ config = VllmConfig(
1185
model_config=model_config,
1186
cache_config=cache_config,
1187
parallel_config=parallel_config,
@@ -1185,6 +1196,42 @@ class EngineArgs:
1196
compilation_config=self.compilation_config,
1197
)
1198
1199
+ if envs.VLLM_USE_V1:
1200
+ self._override_v1_engine_config(config)
1201
+ return config
1202
+
1203
+ def _override_v1_engine_args(self, usage_context: UsageContext) -> None:
1204
+ """
1205
+ Override the EngineArgs's args based on the usage context for V1.
1206
+ """
1207
+ assert envs.VLLM_USE_V1, "V1 is not enabled"
1208
+
1209
+ if self.max_num_batched_tokens is None:
1210
+ # When no user override, set the default values based on the
1211
+ # usage context.
1212
+ if usage_context == UsageContext.LLM_CLASS:
1213
+ logger.warning("Setting max_num_batched_tokens to 8192 "
1214
+ "for LLM_CLASS usage context.")
1215
+ self.max_num_seqs = 1024
1216
+ self.max_num_batched_tokens = 8192
1217
+ elif usage_context == UsageContext.OPENAI_API_SERVER:
1218
+ logger.warning("Setting max_num_batched_tokens to 2048 "
1219
+ "for OPENAI_API_SERVER usage context.")
1220
+ self.max_num_seqs = 1024
1221
+ self.max_num_batched_tokens = 2048
1222
+
1223
+ def _override_v1_engine_config(self, engine_config: VllmConfig) -> None:
1224
+ """
1225
+ Override the EngineConfig's configs based on the usage context for V1.
1226
+ """
1227
+ assert envs.VLLM_USE_V1, "V1 is not enabled"
1228
+ # TODO (ywang96): Enable APC by default when VLM supports it.
1229
+ if engine_config.model_config.is_multimodal_model:
1230
+ logger.warning(
1231
+ "Prefix caching is currently not supported for multimodal "
1232
+ "models and has been disabled.")
1233
+ engine_config.cache_config.enable_prefix_caching = False
1234
+
1235
1236
@dataclass
1237
class AsyncEngineArgs(EngineArgs):
vllm/engine/async_llm_engine.py
+1
-1
@@ -680,7 +680,7 @@ class AsyncLLMEngine(EngineClient):
680
"""Creates an async LLM engine from the engine arguments."""
681
# Create the engine configs.
682
if engine_config is None:
683
- engine_config = engine_args.create_engine_config()
683
+ engine_config = engine_args.create_engine_config(usage_context)
684
685
executor_class = cls._get_executor_cls(engine_config)
686
vllm/engine/llm_engine.py
+1
-1
@@ -568,7 +568,7 @@ class LLMEngine:
568
) -> "LLMEngine":
569
"""Creates an LLM engine from the engine arguments."""
570
# Create the engine configs.
571
- engine_config = engine_args.create_engine_config()
571
+ engine_config = engine_args.create_engine_config(usage_context)
572
executor_class = cls._get_executor_cls(engine_config)
573
# Create the LLM engine.
574
engine = cls(
vllm/engine/multiprocessing/engine.py
+1
-1
@@ -111,7 +111,7 @@ class MQLLMEngine:
111
from vllm.plugins import load_general_plugins
112
load_general_plugins()
113
114
- engine_config = engine_args.create_engine_config()
114
+ engine_config = engine_args.create_engine_config(usage_context)
115
executor_class = LLMEngine._get_executor_cls(engine_config)
116
117
use_async_sockets = engine_config.model_config.use_async_output_proc
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -135,8 +135,8 @@ async def build_async_engine_client_from_engine_args(
135
# TODO: fill out feature matrix.
136
if (MQLLMEngineClient.is_unsupported_config(engine_args)
137
or envs.VLLM_USE_V1 or disable_frontend_multiprocessing):
138
-
139
- engine_config = engine_args.create_engine_config()
138
+ engine_config = engine_args.create_engine_config(
139
+ UsageContext.OPENAI_API_SERVER)
140
uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
141
"uses_ray", False)
142
vllm/v1/engine/async_llm.py
+1
-1
@@ -94,7 +94,7 @@ class AsyncLLM(EngineClient):
94
95
# Create the engine configs.
96
if engine_config is None:
97
- vllm_config = engine_args.create_engine_config()
97
+ vllm_config = engine_args.create_engine_config(usage_context)
98
else:
99
vllm_config = engine_config
100
vllm/v1/engine/core.py
-13
@@ -41,19 +41,6 @@ class EngineCore:
41
executor_class: Type[GPUExecutor],
42
usage_context: UsageContext,
43
):
44
- # Override the configs for V1.
45
- # FIXME
46
- if usage_context == UsageContext.LLM_CLASS:
47
- vllm_config.scheduler_config.max_num_seqs = 1024
48
- vllm_config.scheduler_config.max_num_batched_tokens = 8192
49
- elif usage_context == UsageContext.OPENAI_API_SERVER:
50
- vllm_config.scheduler_config.max_num_seqs = 1024
51
- vllm_config.scheduler_config.max_num_batched_tokens = 2048
52
-
53
- # TODO (ywang96): Enable APC by default when VLM supports it.
54
- if not vllm_config.model_config.is_multimodal_model:
55
- vllm_config.cache_config.enable_prefix_caching = True
56
-
44
assert vllm_config.model_config.task != "embedding"
45
46
logger.info("Initializing an LLM engine (v%s) with config: %s",
vllm/v1/engine/llm_engine.py
+1
-1
@@ -82,7 +82,7 @@ class LLMEngine:
82
"""Creates an LLM engine from the engine arguments."""
83
84
# Create the engine configs.
85
- vllm_config = engine_args.create_engine_config()
85
+ vllm_config = engine_args.create_engine_config(usage_context)
86
executor_class = cls._get_executor_cls(vllm_config)
87
88
if VLLM_ENABLE_V1_MULTIPROCESSING: