A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Hardware] [HPU]add `mark_step` for hpu (#10239)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
Nov 17, 2024 at 16:44 UTC
76aab90ab68476c353ad58019fd51fd18622056a
1 file changed
+14
vllm/worker/hpu_model_runner.py
+14
@@ -272,6 +272,19 @@ def precompute_indices_and_offsets(block_size, slot_mapping, is_prompt):
272
return indices, offsets
273
274
275
+def modify_decoder_layer(module: torch.nn.Module, suffix="DecoderLayer"):
276
+ if module.__class__.__name__.endswith(suffix):
277
+
278
+ def forward_hook(module, args, output):
279
+ htorch.core.mark_step()
280
+ return output
281
+
282
+ module.register_forward_hook(forward_hook)
283
+
284
+ for child_name, child_module in module.named_children():
285
+ modify_decoder_layer(child_module)
286
+
287
+
288
class HpuModelAdapter:
289
290
def __init__(self, model, block_size, dtype, enforce_eager):
@@ -636,6 +649,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
649
else:
650
self.model = self.model.to("hpu")
651
htcore.mark_step()
652
+ modify_decoder_layer(self.model)
653
torch.hpu.synchronize()
654
655
with HabanaMemoryProfiler() as m_wrap: