A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[HPU][Bugfix] set_forward_context and CI test execution (#12014)
Signed-off-by: Konrad Zawora <kzawora@habana.ai>
Konrad Zawora committed
Jan 14, 2025 at 04:04 UTC
078da319033a32304bb3297092c9ec34c52d598b
3 files changed
+23
-18
.buildkite/run-hpu-test.sh
+5
-2
@@ -8,9 +8,12 @@ set -ex
8
docker build -t hpu-test-env -f Dockerfile.hpu .
9
10
# Setup cleanup
11
+EXITCODE=1
12
remove_docker_container() { docker rm -f hpu-test || true; }
12
-trap remove_docker_container EXIT
13
+remove_docker_container_and_exit() { remove_docker_container; exit $EXITCODE; }
14
+trap remove_docker_container_and_exit EXIT
15
remove_docker_container
16
17
# Run the image and launch offline inference
16
-docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic.py
\ No newline at end of file
18
+docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic.py
19
+EXITCODE=$?
Dockerfile.hpu
+1
-1
@@ -1,4 +1,4 @@
1
-FROM vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest
1
+FROM vault.habana.ai/gaudi-docker/1.19.1/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest
2
3
COPY ./ /workspace/vllm
4
vllm/worker/hpu_model_runner.py
+17
-15
@@ -289,12 +289,14 @@ def modify_decoder_layer(module: torch.nn.Module, suffix="DecoderLayer"):
289
290
class HpuModelAdapter:
291
292
- def __init__(self, model, block_size, dtype, enforce_eager):
292
+ def __init__(self, model, vllm_config):
293
self.model = model
294
self.prefill_use_fusedsdpa = os.getenv('VLLM_PROMPT_USE_FUSEDSDPA',
295
'0').lower() in ['1', 'true']
296
- self.block_size = block_size
297
- self.dtype = dtype
296
+ self.vllm_config = vllm_config
297
+ self.block_size = vllm_config.cache_config.block_size
298
+ self.dtype = vllm_config.model_config.dtype
299
+ enforce_eager = vllm_config.model_config.enforce_eager
300
if not htorch.utils.internal.is_lazy() and not enforce_eager:
301
self.model = torch.compile(self.model,
302
backend='hpu_backend',
@@ -353,14 +355,20 @@ class HpuModelAdapter:
355
selected_token_indices = kwargs.pop('selected_token_indices')
356
if 'warmup_mode' in kwargs:
357
kwargs.pop('warmup_mode')
358
+ virtual_engine = 0
359
+ if 'virtual_engine' in kwargs:
360
+ virtual_engine = kwargs.pop('virtual_engine')
361
input_ids = kwargs['input_ids']
362
kwargs['attn_metadata'] = self._update_metadata(
363
kwargs['attn_metadata'], input_ids.size(0), input_ids.size(1),
364
input_ids.device, self.dtype)
365
LoraMask.setLoraMask(kwargs.pop('lora_mask'))
361
- hidden_states = self.model(*args, **kwargs)
362
- hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
363
- hidden_states = hidden_states.index_select(0, selected_token_indices)
366
+ with set_forward_context(kwargs['attn_metadata'], self.vllm_config,
367
+ virtual_engine):
368
+ hidden_states = self.model(*args, **kwargs)
369
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
370
+ hidden_states = hidden_states.index_select(0,
371
+ selected_token_indices)
372
return hidden_states
373
374
def compute_logits(self, *args, **kwargs):
@@ -660,10 +668,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
668
669
with HabanaMemoryProfiler() as m_wrap:
670
self.model = _maybe_wrap_in_hpu_graph(
663
- self.model,
664
- self.block_size,
665
- dtype=self.model_config.dtype,
666
- enforce_eager=self.enforce_eager)
671
+ self.model, vllm_config=self.vllm_config)
672
msg = f"Wrapping in HPU Graph took {m_wrap.get_summary_string()}"
673
logger.info(msg)
674
@@ -1934,6 +1939,7 @@ class HPUModelRunner(HPUModelRunnerBase[ModelInputForHPUWithSamplingMetadata]):
1939
"attn_metadata": self.trim_attn_metadata(attn_metadata),
1940
"intermediate_tensors": intermediate_tensors,
1941
"lora_mask": lora_mask,
1942
+ "virtual_engine": model_input.virtual_engine,
1943
**(model_input.multi_modal_kwargs or {}),
1944
}
1945
if htorch.utils.internal.is_lazy():
@@ -1948,11 +1954,7 @@ class HPUModelRunner(HPUModelRunnerBase[ModelInputForHPUWithSamplingMetadata]):
1954
f"graphs{'T' if use_graphs else 'F'}")
1955
else:
1956
model_event_name = 'model_executable'
1951
- with set_forward_context(
1952
- model_input.attn_metadata, self.vllm_config,
1953
- model_input.virtual_engine), \
1954
- self.profiler.record_event(
1955
- 'internal', model_event_name):
1957
+ with self.profiler.record_event('internal', model_event_name):
1958
hidden_states = self.model.forward(
1959
**execute_model_kwargs,
1960
selected_token_indices=sampling_metadata.selected_token_indices