A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Intel GPU] Fix xpu decode input (#9145)
Kunshang Ji committed
Oct 8, 2024 at 11:51 UTC
80b57f00d554db8a2126d351bb5374c190b56699
1 file changed
+14
-7
vllm/worker/xpu_model_runner.py
+14
-7
@@ -15,6 +15,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
from vllm.distributed import get_pp_group
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry
17
from vllm.logger import init_logger
18
+from vllm.model_executor import SamplingMetadataCache
19
from vllm.model_executor.layers.sampler import SamplerOutput
20
from vllm.model_executor.model_loader import get_model
21
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
@@ -136,7 +137,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
137
(input_tokens, input_positions,
138
attn_metadata) = self._prepare_decode(
139
self.seq_group_metadata_list)
139
- seq_lens = []
140
+ seq_lens = None
141
multi_modal_kwargs = None
142
143
return self.model_input_cls(
@@ -390,6 +391,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
391
# Lazy initialization.
392
self.model: nn.Module # Set after init_Model
393
394
+ self.sampling_metadata_cache: SamplingMetadataCache = \
395
+ SamplingMetadataCache() \
396
+ if self.parallel_config.pipeline_parallel_size == 1 else None
397
+
398
def load_model(self) -> None:
399
with DeviceMemoryProfiler() as m:
400
self.model = get_model(
@@ -524,12 +529,14 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
529
seq_group_metadata_list, finished_requests_ids)
530
# Sampling metadata is only required for the final pp group
531
generators = self.get_generators(finished_requests_ids)
527
- sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
528
- model_input.seq_lens,
529
- model_input.query_lens,
530
- self.device,
531
- pin_memory=False,
532
- generators=generators)
532
+ sampling_metadata = SamplingMetadata.prepare(
533
+ seq_group_metadata_list,
534
+ model_input.seq_lens,
535
+ model_input.query_lens,
536
+ self.device,
537
+ pin_memory=False,
538
+ generators=generators,
539
+ cache=self.sampling_metadata_cache)
540
541
return dataclasses.replace(model_input,
542
sampling_metadata=sampling_metadata,