@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b2496bb07f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[core] fix sleep mode and pytorch checkpoint compatibility (#13001)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 10, 2025 at 13:03 UTC b2496bb07fdf9318e7d9a8065356941fef380bac
2 files changed +8 -3
tests/basic_correctness/test_cumem.py
+8 -2
@@ -115,10 +115,16 @@ def test_cumem_with_cudagraph():
115
116
117 @fork_new_process_for_each_test
118 -def test_end_to_end():
118 +@pytest.mark.parametrize(
119 + "model",
120 + [
121 + "meta-llama/Llama-3.2-1B", # sleep mode with safetensors
122 + "facebook/opt-125m" # sleep mode with pytorch checkpoint
123 + ])
124 +def test_end_to_end(model):
125 free, total = torch.cuda.mem_get_info()
126 used_bytes_baseline = total - free # in case other process is running
121 - llm = LLM("meta-llama/Llama-3.2-1B", enable_sleep_mode=True)
127 + llm = LLM(model, enable_sleep_mode=True)
128 prompt = "How are you?"
129 sampling_params = SamplingParams(temperature=0, max_tokens=10)
130 output = llm.generate(prompt, sampling_params)
vllm/model_executor/model_loader/weight_utils.py
-1
@@ -462,7 +462,6 @@ def pt_weights_iterator(
462 state = torch.load(bin_file, map_location="cpu", weights_only=True)
463 yield from state.items()
464 del state
465 - torch.cuda.empty_cache()
465
466
467 def get_gguf_extra_tensor_names(