@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80b57f00d5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Intel GPU] Fix xpu decode input (#9145)

Kunshang Ji committed Oct 8, 2024 at 11:51 UTC 80b57f00d554db8a2126d351bb5374c190b56699
1 file changed +14 -7
vllm/worker/xpu_model_runner.py
+14 -7
@@ -15,6 +15,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 from vllm.distributed import get_pp_group
16 from vllm.inputs import INPUT_REGISTRY, InputRegistry
17 from vllm.logger import init_logger
18 +from vllm.model_executor import SamplingMetadataCache
19 from vllm.model_executor.layers.sampler import SamplerOutput
20 from vllm.model_executor.model_loader import get_model
21 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
@@ -136,7 +137,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
137 (input_tokens, input_positions,
138 attn_metadata) = self._prepare_decode(
139 self.seq_group_metadata_list)
139 - seq_lens = []
140 + seq_lens = None
141 multi_modal_kwargs = None
142
143 return self.model_input_cls(
@@ -390,6 +391,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
391 # Lazy initialization.
392 self.model: nn.Module # Set after init_Model
393
394 + self.sampling_metadata_cache: SamplingMetadataCache = \
395 + SamplingMetadataCache() \
396 + if self.parallel_config.pipeline_parallel_size == 1 else None
397 +
398 def load_model(self) -> None:
399 with DeviceMemoryProfiler() as m:
400 self.model = get_model(
@@ -524,12 +529,14 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
529 seq_group_metadata_list, finished_requests_ids)
530 # Sampling metadata is only required for the final pp group
531 generators = self.get_generators(finished_requests_ids)
527 - sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
528 - model_input.seq_lens,
529 - model_input.query_lens,
530 - self.device,
531 - pin_memory=False,
532 - generators=generators)
532 + sampling_metadata = SamplingMetadata.prepare(
533 + seq_group_metadata_list,
534 + model_input.seq_lens,
535 + model_input.query_lens,
536 + self.device,
537 + pin_memory=False,
538 + generators=generators,
539 + cache=self.sampling_metadata_cache)
540
541 return dataclasses.replace(model_input,
542 sampling_metadata=sampling_metadata,