@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2bc3fbba0c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[FlashInfer] Upgrade to 0.2.0 (#11194)

Signed-off-by: Bowen Wang <abmfy@icloud.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Bowen Wang committed Jan 28, 2025 at 02:19 UTC 2bc3fbba0cf5b07fabb798d41b153b895d30c7b4
10 files changed +257 -75
.buildkite/test-pipeline.yaml
+10 -1
@@ -183,7 +183,16 @@ steps:
183 - vllm/
184 - tests/v1
185 commands:
186 - - VLLM_USE_V1=1 pytest -v -s v1
186 + # split the test to avoid interference
187 + - VLLM_USE_V1=1 pytest -v -s v1/core
188 + - VLLM_USE_V1=1 pytest -v -s v1/engine
189 + - VLLM_USE_V1=1 pytest -v -s v1/sample
190 + - VLLM_USE_V1=1 pytest -v -s v1/worker
191 + - VLLM_USE_V1=1 pytest -v -s v1/test_stats.py
192 + - VLLM_USE_V1=1 pytest -v -s v1/test_utils.py
193 + # TODO: accuracy does not match, whether setting
194 + # VLLM_USE_FLASHINFER_SAMPLER or not on H100.
195 + - VLLM_USE_V1=1 pytest -v -s v1/e2e
196
197 - label: Examples Test # 25min
198 working_dir: "/vllm-workspace/examples"
Dockerfile
+21 -2
@@ -149,7 +149,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
149
150 #################### vLLM installation IMAGE ####################
151 # image with vLLM installed
152 -FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu22.04 AS vllm-base
152 +# TODO: Restore to base image after FlashInfer AOT wheel fixed
153 +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS vllm-base
154 ARG CUDA_VERSION=12.4.1
155 ARG PYTHON_VERSION=3.12
156 WORKDIR /vllm-workspace
@@ -194,12 +195,30 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
195 --mount=type=cache,target=/root/.cache/pip \
196 python3 -m pip install dist/*.whl --verbose
197
198 +# How to build this FlashInfer wheel:
199 +# $ export FLASHINFER_ENABLE_AOT=1
200 +# $ # Note we remove 7.0 from the arch list compared to the list below, since FlashInfer only supports sm75+
201 +# $ export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.6 8.9 9.0+PTX'
202 +# $ git clone https://github.com/flashinfer-ai/flashinfer.git --recursive
203 +# $ cd flashinfer
204 +# $ git checkout 524304395bd1d8cd7d07db083859523fcaa246a4
205 +# $ python3 setup.py bdist_wheel --dist-dir=dist --verbose
206 +
207 RUN --mount=type=cache,target=/root/.cache/pip \
208 . /etc/environment && \
209 if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \
200 - python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.6/flashinfer-0.1.6+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl; \
210 + python3 -m pip install https://wheels.vllm.ai/flashinfer/524304395bd1d8cd7d07db083859523fcaa246a4/flashinfer_python-0.2.0.post1-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl; \
211 fi
212 COPY examples examples
213 +
214 +# Although we build Flashinfer with AOT mode, there's still
215 +# some issues w.r.t. JIT compilation. Therefore we need to
216 +# install build dependencies for JIT compilation.
217 +# TODO: Remove this once FlashInfer AOT wheel is fixed
218 +COPY requirements-build.txt requirements-build.txt
219 +RUN --mount=type=cache,target=/root/.cache/pip \
220 + python3 -m pip install -r requirements-build.txt
221 +
222 #################### vLLM installation IMAGE ####################
223
224 #################### TEST IMAGE ####################
tests/basic_correctness/test_basic_correctness.py
+3 -2
@@ -61,9 +61,10 @@ def test_models(
61 if backend == "FLASHINFER" and current_platform.is_rocm():
62 pytest.skip("Flashinfer does not support ROCm/HIP.")
63
64 - if backend == "XFORMERS" and model == "google/gemma-2-2b-it":
64 + if backend in ("XFORMERS",
65 + "FLASHINFER") and model == "google/gemma-2-2b-it":
66 pytest.skip(
66 - "XFORMERS does not support gemma2 with full context length.")
67 + f"{backend} does not support gemma2 with full context length.")
68
69 os.environ["VLLM_ATTENTION_BACKEND"] = backend
70
tests/compile/test_basic_correctness.py
+1 -1
@@ -58,7 +58,7 @@ test_settings = [
58 model_args=["--task", "embed"],
59 pp_size=1,
60 tp_size=1,
61 - attn_backend="FLASHINFER",
61 + attn_backend="FLASH_ATTN",
62 method="encode",
63 fullgraph=True,
64 ),
tests/kernels/test_flashinfer.py
+37 -37
@@ -133,17 +133,19 @@ def test_flashinfer_decode_with_paged_kv(
133 use_tensor_cores=(
134 (num_query_heads//num_kv_heads) > 4)
135 )
136 - wrapper.begin_forward(kv_indptr,
137 - kv_indices,
138 - kv_last_page_lens,
139 - num_query_heads,
140 - num_kv_heads,
141 - head_size,
142 - block_size,
143 - "NONE",
144 - data_type=dtype)
145 -
146 - output = wrapper.forward(query, key_value_cache, logits_soft_cap=soft_cap)
136 + wrapper.plan(kv_indptr,
137 + kv_indices,
138 + kv_last_page_lens,
139 + num_query_heads,
140 + num_kv_heads,
141 + head_size,
142 + block_size,
143 + "NONE",
144 + q_data_type=dtype,
145 + kv_data_type=dtype,
146 + logits_soft_cap=soft_cap)
147 +
148 + output = wrapper.run(query, key_value_cache)
149
150 ref_output = ref_paged_attn(query=query,
151 key_cache=key_cache,
@@ -228,7 +230,7 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
230 workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
231 wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(
232 workspace_buffer, "NHD")
231 - wrapper.begin_forward(
233 + wrapper.plan(
234 qo_indptr,
235 kv_indptr,
236 kv_indices,
@@ -237,12 +239,14 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
239 num_kv_heads,
240 head_size,
241 block_size,
242 + q_data_type=dtype,
243 + kv_data_type=dtype,
244 + logits_soft_cap=soft_cap,
245 )
246
242 - output = wrapper.forward(
247 + output = wrapper.run(
248 query,
249 key_value_cache,
245 - logits_soft_cap=soft_cap,
250 )
251
252 ref_output = ref_paged_attn(query=query,
@@ -253,7 +257,7 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
257 block_tables=block_tables,
258 scale=scale,
259 soft_cap=soft_cap)
256 - torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2), \
260 + torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2), \
261 f"{torch.max(torch.abs(output - ref_output))}"
262
263
@@ -332,7 +336,7 @@ def test_flashinfer_prefill_with_paged_fp8_kv(
336 workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
337 wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(
338 workspace_buffer, "NHD")
335 - wrapper.begin_forward(
339 + wrapper.plan(
340 qo_indptr,
341 kv_indptr,
342 kv_indices,
@@ -341,13 +345,12 @@ def test_flashinfer_prefill_with_paged_fp8_kv(
345 num_kv_heads,
346 head_size,
347 block_size,
348 + q_data_type=dtype,
349 + kv_data_type=kv_cache_dtype,
350 + logits_soft_cap=soft_cap,
351 )
352
346 - output = wrapper.forward(query,
347 - kv_cache_fp8,
348 - logits_soft_cap=soft_cap,
349 - k_scale=k_scale,
350 - v_scale=v_scale)
353 + output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
354
355 ref_output = ref_paged_attn(query=query,
356 key_cache=key_cache.squeeze(1),
@@ -360,7 +363,7 @@ def test_flashinfer_prefill_with_paged_fp8_kv(
363 del query
364 del block_tables
365 # verify prefill fp8
363 - torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2), \
366 + torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2), \
367 f"{torch.max(torch.abs(output - ref_output))}"
368
369
@@ -439,21 +442,18 @@ def test_flashinfer_decode_with_paged_fp8_kv(
442 wrapper = flashinfer.\
443 BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD",
444 use_tensor_cores=use_tensor_cores)
442 - wrapper.begin_forward(kv_indptr,
443 - kv_indices,
444 - kv_last_page_lens,
445 - num_query_heads,
446 - num_kv_heads,
447 - head_size,
448 - block_size,
449 - "NONE",
450 - data_type=dtype,
451 - q_data_type=dtype)
452 - output = wrapper.forward(query,
453 - kv_cache_fp8,
454 - logits_soft_cap=soft_cap,
455 - k_scale=k_scale,
456 - v_scale=v_scale)
445 + wrapper.plan(kv_indptr,
446 + kv_indices,
447 + kv_last_page_lens,
448 + num_query_heads,
449 + num_kv_heads,
450 + head_size,
451 + block_size,
452 + "NONE",
453 + q_data_type=dtype,
454 + kv_data_type=kv_cache_dtype,
455 + logits_soft_cap=soft_cap)
456 + output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
457 key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
458 value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
459
vllm/attention/backends/flashinfer.py
+162 -21
@@ -1,3 +1,4 @@
1 +import dataclasses
2 from collections import defaultdict
3 from contextlib import contextmanager
4 from dataclasses import dataclass
@@ -13,9 +14,11 @@ try:
14 from vllm.vllm_flash_attn import flash_attn_varlen_func
15 FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
16 except ImportError:
16 - BatchDecodeWithPagedKVCacheWrapper = None
17 - CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
18 - BatchPrefillWithPagedKVCacheWrapper = None
17 + # Avoid turning these types into variables during type checking
18 + if not TYPE_CHECKING:
19 + BatchDecodeWithPagedKVCacheWrapper = None
20 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
21 + BatchPrefillWithPagedKVCacheWrapper = None
22 FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
23
24 import torch
@@ -30,7 +33,9 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
33 from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
34 compute_slot_mapping_start_idx,
35 is_block_tables_empty)
36 +from vllm.attention.layer import Attention
37 from vllm.attention.ops.paged_attn import PagedAttention
38 +from vllm.config import VllmConfig, get_current_vllm_config
39 from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
40 make_tensor_with_pad)
41
@@ -99,6 +104,72 @@ class FlashInferBackend(AttentionBackend):
104 raise ValueError(f"Unrecognized FP8 dtype: {kv_cache_dtype}")
105
106
107 +@dataclass
108 +class PerLayerParameters:
109 + """
110 + Currently, FlashInfer backend only support models in which all layers share
111 + the same values for the following hyperparameters.
112 + """
113 +
114 + window_left: int
115 + logits_soft_cap: Optional[float]
116 + sm_scale: float
117 +
118 +
119 +def get_per_layer_parameters(
120 + vllm_config: VllmConfig) -> Dict[str, PerLayerParameters]:
121 + """
122 + Scan all attention layers and determine some hyperparameters
123 + to use during `plan`.
124 + """
125 +
126 + layers = vllm_config.compilation_config.static_forward_context
127 + per_layer_params: Dict[str, PerLayerParameters] = {}
128 +
129 + for key, layer in layers.items():
130 + assert isinstance(layer, Attention)
131 +
132 + impl = layer.impl
133 + assert isinstance(impl, FlashInferImpl)
134 +
135 + # Infer hyperparameters from the attention layer
136 + window_size = impl.sliding_window
137 + window_left = window_size[0] if window_size is not None else -1
138 + logits_soft_cap = impl.logits_soft_cap
139 + sm_scale = impl.scale
140 +
141 + per_layer_params[key] = PerLayerParameters(window_left,
142 + logits_soft_cap, sm_scale)
143 +
144 + return per_layer_params
145 +
146 +
147 +def infer_global_hyperparameters(
148 + per_layer_params: Dict[str, PerLayerParameters]) -> PerLayerParameters:
149 + """
150 + Currently, FlashInfer backend only support models in which all layers share
151 + the same values for the following hyperparameters:
152 + - `window_left`
153 + - `logits_soft_cap`
154 + - `sm_scale`
155 +
156 + So this function asserts that all layers share the same values for these
157 + hyperparameters and returns the global values.
158 + """
159 +
160 + assert len(per_layer_params) > 0, "No attention layers found in the model."
161 +
162 + param_sets = list(per_layer_params.values())
163 + global_params = param_sets[0]
164 + for params in param_sets:
165 + assert params == global_params, (
166 + "FlashInfer backend currently only supports models in which all "
167 + "layers share the same values for the following hyperparameters: "
168 + "`window_left`, `logits_soft_cap`, `sm_scale`.")
169 +
170 + return global_params
171 +
172 +
173 class FlashInferState(AttentionState):
174
175 def __init__(self, runner):
@@ -108,6 +179,11 @@ class FlashInferState(AttentionState):
179 self._decode_wrapper = None
180 self._prefill_wrapper = None
181
182 + # Global hyperparameters shared by all attention layers
183 + self.global_hyperparameters: Optional[PerLayerParameters] = None
184 +
185 + self.vllm_config = get_current_vllm_config()
186 +
187 def _get_workspace_buffer(self):
188 if self._workspace_buffer is None:
189 self._workspace_buffer = torch.empty(
@@ -215,6 +291,9 @@ class FlashInferState(AttentionState):
291 batch_size + 1,
292 dtype=torch.int32)
293
294 + global_params = infer_global_hyperparameters(
295 + get_per_layer_parameters(self.vllm_config))
296 +
297 attn_metadata = self.runner.attn_backend.make_metadata(
298 num_prefills=0,
299 slot_mapping=self._graph_slot_mapping[:batch_size],
@@ -238,7 +317,9 @@ class FlashInferState(AttentionState):
317 q_data_type=self.runner.model_config.dtype,
318 use_cuda_graph=True,
319 decode_wrapper=self._graph_decode_wrapper,
241 - prefill_wrapper=None)
320 + prefill_wrapper=None,
321 + **dataclasses.asdict(global_params),
322 + )
323 attn_metadata.begin_forward()
324 return attn_metadata
325
@@ -325,9 +406,28 @@ class FlashInferMetadata(AttentionMetadata):
406 data_type: torch.dtype = None
407 # The data type of the query
408 q_data_type: torch.dtype = None
328 - device: torch.device = torch.device("cuda")
409 + # FlashInfer 0.2 encourages passing host tensors
410 + device: torch.device = torch.device("cpu")
411 is_profile_run: bool = False
412
413 + # The FlashInfer backend currently supports only models in which all layers
414 + # share the same following hyperparameters:
415 +
416 + # The left (inclusive) window size for the attention window, when
417 + # set to `-1`, the window size will be set to the full length of
418 + # the sequence. Defaults to `-1`.
419 + window_left: int = -1
420 + # The attention logits soft capping value (used in Gemini, Grok and
421 + # Gemma-2, etc.), if not provided, will be set to `0`. If greater
422 + # than 0, the logits will be capped according to formula:
423 + # $$\texttt{logits\_soft\_cap} \times
424 + # \mathrm{tanh}(x / \texttt{logits\_soft\_cap})$$,
425 + # where $x$ is the input logits.
426 + logits_soft_cap: Optional[float] = None
427 + # The scale used in softmax, if not provided, will be set to
428 + # `1.0 / sqrt(head_dim)`.
429 + sm_scale: Optional[float] = None
430 +
431 def __post_init__(self):
432 # Refer to
433 # https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
@@ -363,14 +463,21 @@ class FlashInferMetadata(AttentionMetadata):
463 self.block_table_bound = self.block_table_bound.to(self.device)
464 self.seq_lens_tensor = self.seq_lens_tensor.to(self.device)
465 self.paged_kv_indices = self.paged_kv_indices.to(self.device)
366 - self.prefill_wrapper.end_forward()
367 - self.prefill_wrapper.begin_forward(
466 + self.prefill_wrapper.plan(
467 self.query_start_loc,
468 self.paged_kv_indptr[:self.num_prefills + 1],
469 self.paged_kv_indices,
470 self.paged_kv_last_page_len[:self.num_prefills],
372 - self.num_qo_heads, self.num_kv_heads, self.head_dim,
373 - self.page_size)
471 + self.num_qo_heads,
472 + self.num_kv_heads,
473 + self.head_dim,
474 + self.page_size,
475 + causal=True,
476 + sm_scale=self.sm_scale,
477 + window_left=self.window_left,
478 + logits_soft_cap=self.logits_soft_cap,
479 + q_data_type=self.q_data_type,
480 + kv_data_type=self.data_type)
481 if self.num_decode_tokens > 0:
482 assert self.paged_kv_indices is not None
483 assert self.paged_kv_indptr is not None
@@ -386,8 +493,7 @@ class FlashInferMetadata(AttentionMetadata):
493 self.seq_lens_tensor = self.seq_lens_tensor.to(self.device)
494
495 assert self.decode_wrapper is not None
389 - self.decode_wrapper.end_forward()
390 - self.decode_wrapper.begin_forward(
496 + self.decode_wrapper.plan(
497 self.paged_kv_indptr[self.num_prefills:],
498 self.paged_kv_indices,
499 self.paged_kv_last_page_len[self.num_prefills:],
@@ -397,8 +503,11 @@ class FlashInferMetadata(AttentionMetadata):
503 self.page_size,
504 # Disable flashinfer's pos encoding and use vllm's rope.
505 pos_encoding_mode="NONE",
506 + window_left=self.window_left,
507 + logits_soft_cap=self.logits_soft_cap,
508 + sm_scale=self.sm_scale,
509 # kv-cache data type.
401 - data_type=self.data_type,
510 + kv_data_type=self.data_type,
511 # query data type.
512 q_data_type=self.q_data_type)
513
@@ -496,6 +605,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
605 self.sliding_window = input_builder.sliding_window
606 self.block_size = input_builder.block_size
607
608 + # Global hyperparameters shared by all attention layers
609 + self.global_hyperparameters: Optional[PerLayerParameters] = None
610 +
611 + self.vllm_config = get_current_vllm_config()
612 +
613 def prepare(self):
614 self.slot_mapping: List[int] = []
615 self.prefill_seq_lens: List[int] = []
@@ -528,6 +642,20 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
642 self.total_blocks = 0
643 self.is_profile_run: bool = False
644
645 + if self.global_hyperparameters is None:
646 + # Infer global hyperparameters, since currently we only support
647 + # models in which all layers share the same values for the
648 + # following hyperparameters:
649 + # - `window_left`
650 + # - `logits_soft_cap`
651 + # - `sm_scale`
652 + inferred_params = infer_global_hyperparameters(
653 + get_per_layer_parameters(self.vllm_config))
654 + self.global_hyperparameters = inferred_params
655 + self.window_left = inferred_params.window_left
656 + self.logits_soft_cap = inferred_params.logits_soft_cap
657 + self.sm_scale = inferred_params.sm_scale
658 +
659 def _add_seq_group(
660 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
661 chunked_prefill_enabled: bool):
@@ -756,7 +884,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
884 data_type=kv_cache_dtype,
885 q_data_type=self.runner.model_config.dtype,
886 use_cuda_graph=use_captured_graph,
759 - is_profile_run=self.is_profile_run)
887 + is_profile_run=self.is_profile_run,
888 + window_left=self.window_left,
889 + logits_soft_cap=self.logits_soft_cap,
890 + sm_scale=self.sm_scale,
891 + )
892
893
894 class FlashInferImpl(AttentionImpl):
@@ -885,25 +1017,34 @@ class FlashInferImpl(AttentionImpl):
1017 else:
1018 assert prefill_meta is not None
1019 assert prefill_meta.prefill_wrapper is not None
888 - prefill_output = prefill_meta.prefill_wrapper.forward(
1020 +
1021 + assert prefill_meta.prefill_wrapper._causal
1022 + assert prefill_meta.prefill_wrapper._window_left == window_left
1023 + assert prefill_meta.prefill_wrapper._logits_soft_cap == (
1024 + logits_soft_cap or 0.0)
1025 + assert prefill_meta.prefill_wrapper._sm_scale == softmax_scale
1026 +
1027 + prefill_output = prefill_meta.prefill_wrapper.run(
1028 query,
1029 kv_cache,
891 - logits_soft_cap=logits_soft_cap,
892 - causal=True,
1030 k_scale=layer._k_scale_float,
1031 v_scale=layer._v_scale_float,
895 - window_left=window_left)
1032 + )
1033 if decode_meta := attn_metadata.decode_metadata:
1034 assert decode_meta is not None
1035 assert decode_meta.decode_wrapper is not None
899 - decode_output = decode_meta.decode_wrapper.forward(
1036 +
1037 + assert decode_meta.decode_wrapper._window_left == window_left
1038 + assert decode_meta.decode_wrapper._logits_soft_cap == (
1039 + logits_soft_cap or 0.0)
1040 + assert decode_meta.decode_wrapper._sm_scale == softmax_scale
1041 +
1042 + decode_output = decode_meta.decode_wrapper.run(
1043 decode_query,
1044 kv_cache,
902 - sm_scale=softmax_scale,
903 - logits_soft_cap=logits_soft_cap,
1045 k_scale=layer._k_scale_float,
1046 v_scale=layer._v_scale_float,
906 - window_left=window_left)
1047 + )
1048
1049 if prefill_output is None and decode_output is not None:
1050 # Decode only batch.
vllm/config.py
+6 -4
@@ -310,14 +310,15 @@ class ModelConfig:
310 (self.hf_text_config.model_type in ["gemma2", "cohere2"]))
311
312 if (not self.disable_sliding_window and has_interleaved_attention):
313 - if envs.VLLM_ATTENTION_BACKEND == "XFORMERS":
313 + if (backend :=
314 + envs.VLLM_ATTENTION_BACKEND) in ("XFORMERS", "FLASHINFER"):
315 sliding_window_len_min = get_min_sliding_window(
316 self.hf_text_config.sliding_window)
317
318 logger.warning_once(
319 f"{self.hf_text_config.model_type} has interleaved "
320 "attention, which is currently not supported by the "
320 - "XFORMERS backend. Disabling sliding window and capping "
321 + f"{backend} backend. Disabling sliding window and capping "
322 "the max length to the sliding window size "
323 f"({sliding_window_len_min}).")
324 self.disable_sliding_window = True
@@ -3310,7 +3311,7 @@ _current_vllm_config: Optional[VllmConfig] = None
3311
3312
3313 @contextmanager
3313 -def set_current_vllm_config(vllm_config: VllmConfig):
3314 +def set_current_vllm_config(vllm_config: VllmConfig, check_compile=False):
3315 """
3316 Temporarily set the current VLLM config.
3317 Used during model initialization.
@@ -3330,7 +3331,8 @@ def set_current_vllm_config(vllm_config: VllmConfig):
3331 vllm_config.compilation_config.enabled_custom_ops)
3332 logger.debug("disabled custom ops: %s",
3333 vllm_config.compilation_config.disabled_custom_ops)
3333 - if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
3334 + if check_compile and \
3335 + vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
3336 and compilation_counter.num_models_seen == num_models_seen:
3337 # If the model supports compilation,
3338 # compilation_counter.num_models_seen should be increased
vllm/model_executor/model_loader/loader.py
+2 -2
@@ -114,7 +114,7 @@ def _initialize_model(
114 all_params = [param.name for param in signatures.parameters.values()]
115 if "vllm_config" in all_params and "prefix" in all_params:
116 # new-style model class
117 - with set_current_vllm_config(vllm_config):
117 + with set_current_vllm_config(vllm_config, check_compile=True):
118 return model_class(vllm_config=vllm_config, prefix=prefix)
119
120 msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
@@ -142,7 +142,7 @@ def _initialize_model(
142 kwargs["lora_config"] = vllm_config.lora_config
143 if "scheduler_config" in all_params:
144 kwargs["scheduler_config"] = vllm_config.scheduler_config
145 - with set_current_vllm_config(vllm_config):
145 + with set_current_vllm_config(vllm_config, check_compile=True):
146 return model_class(**kwargs)
147
148
vllm/model_executor/model_loader/tensorizer.py
+2 -1
@@ -288,7 +288,8 @@ class TensorizerAgent:
288 model_args.torch_dtype = self.tensorizer_config.dtype
289 assert self.tensorizer_config.model_class is not None
290 # TODO: Do we need to consider old-style model class?
291 - with no_init_or_tensor(), set_current_vllm_config(self.vllm_config):
291 + with no_init_or_tensor(), set_current_vllm_config(self.vllm_config,
292 + check_compile=True):
293 return self.tensorizer_config.model_class(
294 vllm_config=self.vllm_config, )
295
vllm/worker/worker_base.py
+13 -4
@@ -8,7 +8,8 @@ import cloudpickle
8 import torch
9 import torch.nn as nn
10
11 -from vllm.config import ObservabilityConfig, VllmConfig
11 +from vllm.config import (ObservabilityConfig, VllmConfig,
12 + set_current_vllm_config)
13 from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
14 from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
@@ -498,8 +499,11 @@ class WorkerWrapperBase:
499 group.
500 """
501 self.rpc_rank = rpc_rank
501 - self.vllm_config = vllm_config
502 self.worker: Optional[WorkerBase] = None
503 + # do not store this `vllm_config`, `init_worker` will set the final
504 + # one. TODO: investigate if we can remove this field in
505 + # `WorkerWrapperBase`, `init_cached_hf_modules` should be
506 + # unnecessary now.
507 if vllm_config.model_config is not None:
508 # it can be None in tests
509 trust_remote_code = vllm_config.model_config.trust_remote_code
@@ -533,6 +537,9 @@ class WorkerWrapperBase:
537 Arguments are passed to the worker class constructor.
538 """
539 kwargs = all_kwargs[self.rpc_rank]
540 + self.vllm_config = kwargs.get("vllm_config", None)
541 + assert self.vllm_config is not None, (
542 + "vllm_config is required to initialize the worker")
543 enable_trace_function_call_for_thread(self.vllm_config)
544
545 from vllm.plugins import load_general_plugins
@@ -546,8 +553,10 @@ class WorkerWrapperBase:
553 bytes)
554 worker_class = cloudpickle.loads(
555 self.vllm_config.parallel_config.worker_cls)
549 - self.worker = worker_class(**kwargs)
550 - assert self.worker is not None
556 + with set_current_vllm_config(self.vllm_config):
557 + # To make vLLM config available during worker initialization
558 + self.worker = worker_class(**kwargs)
559 + assert self.worker is not None
560
561 def execute_method(self, method: Union[str, bytes], *args, **kwargs):
562 try: