A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix][Hardware][CPU] Fix broken encoder-decoder CPU runner (#10218)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 11, 2024 at 20:37 UTC
2cebda42bb9f52a99e566b9b439fdcca2e9f950e
4 files changed
+16
.buildkite/run-cpu-test-ppc64le.sh
+2
@@ -18,6 +18,8 @@ source /etc/environment
18
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN="$HF_TOKEN" --name cpu-test cpu-test
19
20
function cpu_tests() {
21
+ set -e
22
+
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
set -e
.buildkite/run-cpu-test.sh
+2
@@ -20,6 +20,8 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
20
--cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21
22
function cpu_tests() {
23
+ set -e
24
+
25
# offline inference
26
docker exec cpu-test-avx2 bash -c "
27
set -e
vllm/worker/cpu_embedding_model_runner.py
+1
@@ -95,6 +95,7 @@ class CPUEmbeddingModelRunner(
95
model_input.seq_lens)
96
97
return dataclasses.replace(model_input,
98
+ virtual_engine=virtual_engine,
99
pooling_metadata=pooling_metadata)
100
101
def _prepare_pooling(
vllm/worker/cpu_enc_dec_model_runner.py
+11
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
4
import torch
5
6
from vllm.attention import AttentionMetadata
7
+from vllm.model_executor import SamplingMetadata
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.multimodal import MultiModalKwargs
10
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
@@ -96,11 +97,21 @@ class CPUEncoderDecoderModelRunner(
97
encoder_input_positions_tensor,
98
) = self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
99
model_input)
100
+ # Sampling metadata is only required for the final pp group
101
+ generators = self.get_generators(finished_requests_ids)
102
+ sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
103
+ model_input.seq_lens,
104
+ model_input.query_lens,
105
+ self.device,
106
+ pin_memory=False,
107
+ generators=generators)
108
return dataclasses.replace(
109
model_input,
110
+ sampling_metadata=sampling_metadata,
111
attn_metadata=attn_metadata,
112
encoder_input_tokens=encoder_input_tokens_tensor,
113
encoder_input_positions=encoder_input_positions_tensor,
114
+ virtual_engine=virtual_engine,
115
)
116
117
def _prepare_encoder_model_input_tensors(