@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d559979c54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[bugfix] fix cpu tests (#10585)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 22, 2024 at 17:34 UTC d559979c548c4bee6eca089d5e6dc318630bf465
3 files changed +16 -10
vllm/worker/cpu_embedding_model_runner.py
+3 -1
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union
3
4 import torch
5
6 +from vllm.forward_context import set_forward_context
7 from vllm.model_executor.pooling_metadata import PoolingMetadata
8 from vllm.multimodal import MultiModalKwargs
9 from vllm.pooling_params import PoolingParams
@@ -64,7 +65,8 @@ class CPUEmbeddingModelRunner(
65 intermediate_tensors,
66 }
67
67 - hidden_states = model_executable(**execute_model_kwargs)
68 + with set_forward_context(model_input.attn_metadata, self.vllm_config):
69 + hidden_states = model_executable(**execute_model_kwargs)
70
71 # Only perform pooling in the driver worker.
72 if not self.is_driver_worker:
vllm/worker/cpu_enc_dec_model_runner.py
+3 -1
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
4 import torch
5
6 from vllm.attention import AttentionMetadata
7 +from vllm.forward_context import set_forward_context
8 from vllm.model_executor import SamplingMetadata
9 from vllm.model_executor.layers.sampler import SamplerOutput
10 from vllm.multimodal import MultiModalKwargs
@@ -303,7 +304,8 @@ class CPUEncoderDecoderModelRunner(
304 intermediate_tensors,
305 }
306
306 - hidden_states = model_executable(**execute_model_kwargs)
307 + with set_forward_context(model_input.attn_metadata, self.vllm_config):
308 + hidden_states = model_executable(**execute_model_kwargs)
309
310 # Compute the logits.
311 logits = self.model.compute_logits(hidden_states,
vllm/worker/cpu_model_runner.py
+10 -8
@@ -10,6 +10,7 @@ from torch import nn
10
11 from vllm.attention import AttentionMetadata, get_attn_backend
12 from vllm.config import VllmConfig
13 +from vllm.forward_context import set_forward_context
14 from vllm.logger import init_logger
15 from vllm.model_executor import SamplingMetadata
16 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
@@ -487,14 +488,15 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
488 multimodal_kwargs = MultiModalKwargs.as_kwargs(
489 model_input.multi_modal_kwargs, device=self.device)
490
490 - hidden_states = model_executable(
491 - input_ids=model_input.input_tokens,
492 - positions=model_input.input_positions,
493 - kv_caches=kv_caches,
494 - attn_metadata=model_input.attn_metadata,
495 - intermediate_tensors=intermediate_tensors,
496 - **multimodal_kwargs,
497 - )
491 + with set_forward_context(model_input.attn_metadata, self.vllm_config):
492 + hidden_states = model_executable(
493 + input_ids=model_input.input_tokens,
494 + positions=model_input.input_positions,
495 + kv_caches=kv_caches,
496 + attn_metadata=model_input.attn_metadata,
497 + intermediate_tensors=intermediate_tensors,
498 + **multimodal_kwargs,
499 + )
500
501 # Compute the logits.
502 logits = self.model.compute_logits(hidden_states,