A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[core] fix sleep mode and pytorch checkpoint compatibility (#13001)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 10, 2025 at 13:03 UTC
b2496bb07fdf9318e7d9a8065356941fef380bac
2 files changed
+8
-3
tests/basic_correctness/test_cumem.py
+8
-2
@@ -115,10 +115,16 @@ def test_cumem_with_cudagraph():
115
116
117
@fork_new_process_for_each_test
118
-def test_end_to_end():
118
+@pytest.mark.parametrize(
119
+ "model",
120
+ [
121
+ "meta-llama/Llama-3.2-1B", # sleep mode with safetensors
122
+ "facebook/opt-125m" # sleep mode with pytorch checkpoint
123
+ ])
124
+def test_end_to_end(model):
125
free, total = torch.cuda.mem_get_info()
126
used_bytes_baseline = total - free # in case other process is running
121
- llm = LLM("meta-llama/Llama-3.2-1B", enable_sleep_mode=True)
127
+ llm = LLM(model, enable_sleep_mode=True)
128
prompt = "How are you?"
129
sampling_params = SamplingParams(temperature=0, max_tokens=10)
130
output = llm.generate(prompt, sampling_params)
vllm/model_executor/model_loader/weight_utils.py
-1
@@ -462,7 +462,6 @@ def pt_weights_iterator(
462
state = torch.load(bin_file, map_location="cpu", weights_only=True)
463
yield from state.items()
464
del state
465
- torch.cuda.empty_cache()
465
466
467
def get_gguf_extra_tensor_names(