A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[bugfix] fix cpu tests (#10585)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 22, 2024 at 17:34 UTC
d559979c548c4bee6eca089d5e6dc318630bf465
3 files changed
+16
-10
vllm/worker/cpu_embedding_model_runner.py
+3
-1
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
6
+from vllm.forward_context import set_forward_context
7
from vllm.model_executor.pooling_metadata import PoolingMetadata
8
from vllm.multimodal import MultiModalKwargs
9
from vllm.pooling_params import PoolingParams
@@ -64,7 +65,8 @@ class CPUEmbeddingModelRunner(
65
intermediate_tensors,
66
}
67
67
- hidden_states = model_executable(**execute_model_kwargs)
68
+ with set_forward_context(model_input.attn_metadata, self.vllm_config):
69
+ hidden_states = model_executable(**execute_model_kwargs)
70
71
# Only perform pooling in the driver worker.
72
if not self.is_driver_worker:
vllm/worker/cpu_enc_dec_model_runner.py
+3
-1
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
4
import torch
5
6
from vllm.attention import AttentionMetadata
7
+from vllm.forward_context import set_forward_context
8
from vllm.model_executor import SamplingMetadata
9
from vllm.model_executor.layers.sampler import SamplerOutput
10
from vllm.multimodal import MultiModalKwargs
@@ -303,7 +304,8 @@ class CPUEncoderDecoderModelRunner(
304
intermediate_tensors,
305
}
306
306
- hidden_states = model_executable(**execute_model_kwargs)
307
+ with set_forward_context(model_input.attn_metadata, self.vllm_config):
308
+ hidden_states = model_executable(**execute_model_kwargs)
309
310
# Compute the logits.
311
logits = self.model.compute_logits(hidden_states,
vllm/worker/cpu_model_runner.py
+10
-8
@@ -10,6 +10,7 @@ from torch import nn
10
11
from vllm.attention import AttentionMetadata, get_attn_backend
12
from vllm.config import VllmConfig
13
+from vllm.forward_context import set_forward_context
14
from vllm.logger import init_logger
15
from vllm.model_executor import SamplingMetadata
16
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
@@ -487,14 +488,15 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
488
multimodal_kwargs = MultiModalKwargs.as_kwargs(
489
model_input.multi_modal_kwargs, device=self.device)
490
490
- hidden_states = model_executable(
491
- input_ids=model_input.input_tokens,
492
- positions=model_input.input_positions,
493
- kv_caches=kv_caches,
494
- attn_metadata=model_input.attn_metadata,
495
- intermediate_tensors=intermediate_tensors,
496
- **multimodal_kwargs,
497
- )
491
+ with set_forward_context(model_input.attn_metadata, self.vllm_config):
492
+ hidden_states = model_executable(
493
+ input_ids=model_input.input_tokens,
494
+ positions=model_input.input_positions,
495
+ kv_caches=kv_caches,
496
+ attn_metadata=model_input.attn_metadata,
497
+ intermediate_tensors=intermediate_tensors,
498
+ **multimodal_kwargs,
499
+ )
500
501
# Compute the logits.
502
logits = self.model.compute_logits(hidden_states,