@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2cebda42bb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix][Hardware][CPU] Fix broken encoder-decoder CPU runner (#10218)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 11, 2024 at 20:37 UTC 2cebda42bb9f52a99e566b9b439fdcca2e9f950e
4 files changed +16
.buildkite/run-cpu-test-ppc64le.sh
+2
@@ -18,6 +18,8 @@ source /etc/environment
18 docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN="$HF_TOKEN" --name cpu-test cpu-test
19
20 function cpu_tests() {
21 + set -e
22 +
23 # Run basic model test
24 docker exec cpu-test bash -c "
25 set -e
.buildkite/run-cpu-test.sh
+2
@@ -20,6 +20,8 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
20 --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21
22 function cpu_tests() {
23 + set -e
24 +
25 # offline inference
26 docker exec cpu-test-avx2 bash -c "
27 set -e
vllm/worker/cpu_embedding_model_runner.py
+1
@@ -95,6 +95,7 @@ class CPUEmbeddingModelRunner(
95 model_input.seq_lens)
96
97 return dataclasses.replace(model_input,
98 + virtual_engine=virtual_engine,
99 pooling_metadata=pooling_metadata)
100
101 def _prepare_pooling(
vllm/worker/cpu_enc_dec_model_runner.py
+11
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
4 import torch
5
6 from vllm.attention import AttentionMetadata
7 +from vllm.model_executor import SamplingMetadata
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.multimodal import MultiModalKwargs
10 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
@@ -96,11 +97,21 @@ class CPUEncoderDecoderModelRunner(
97 encoder_input_positions_tensor,
98 ) = self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
99 model_input)
100 + # Sampling metadata is only required for the final pp group
101 + generators = self.get_generators(finished_requests_ids)
102 + sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
103 + model_input.seq_lens,
104 + model_input.query_lens,
105 + self.device,
106 + pin_memory=False,
107 + generators=generators)
108 return dataclasses.replace(
109 model_input,
110 + sampling_metadata=sampling_metadata,
111 attn_metadata=attn_metadata,
112 encoder_input_tokens=encoder_input_tokens_tensor,
113 encoder_input_positions=encoder_input_positions_tensor,
114 + virtual_engine=virtual_engine,
115 )
116
117 def _prepare_encoder_model_input_tensors(