@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 76aab90ab6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Hardware] [HPU]add `mark_step` for hpu (#10239)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>

Kunshang Ji committed Nov 17, 2024 at 16:44 UTC 76aab90ab68476c353ad58019fd51fd18622056a
1 file changed +14
vllm/worker/hpu_model_runner.py
+14
@@ -272,6 +272,19 @@ def precompute_indices_and_offsets(block_size, slot_mapping, is_prompt):
272 return indices, offsets
273
274
275 +def modify_decoder_layer(module: torch.nn.Module, suffix="DecoderLayer"):
276 + if module.__class__.__name__.endswith(suffix):
277 +
278 + def forward_hook(module, args, output):
279 + htorch.core.mark_step()
280 + return output
281 +
282 + module.register_forward_hook(forward_hook)
283 +
284 + for child_name, child_module in module.named_children():
285 + modify_decoder_layer(child_module)
286 +
287 +
288 class HpuModelAdapter:
289
290 def __init__(self, model, block_size, dtype, enforce_eager):
@@ -636,6 +649,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
649 else:
650 self.model = self.model.to("hpu")
651 htcore.mark_step()
652 + modify_decoder_layer(self.model)
653 torch.hpu.synchronize()
654
655 with HabanaMemoryProfiler() as m_wrap: