@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0d62fe58db
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bug fix][Core] assert num_new_tokens == 1 fails when SamplingParams.n is not 1 and max_tokens is large & Add tests for preemption (#4451)

SangBin Cho committed May 2, 2024 at 11:24 UTC 0d62fe58dbb58cfe4132005ce7ff37319d66981d
6 files changed +172 -13
.buildkite/test-pipeline.yaml
+1
@@ -17,6 +17,7 @@ steps:
17 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
18 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
19 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
20 + - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
21
22 - label: Core Test
23 command: pytest -v -s core
tests/basic_correctness/test_chunked_prefill.py
-1
@@ -55,7 +55,6 @@ def test_models(
55 )
56 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
57 del vllm_model
58 - print(vllm_outputs[0])
58
59 for i in range(len(example_prompts)):
60 hf_output_ids, hf_output_str = hf_outputs[i]
tests/basic_correctness/test_preemption.py new
+138
@@ -0,0 +1,138 @@
1 +"""Compare the short outputs of HF and vLLM when using greedy sampling.
2 +
3 +VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 has to be set before running this test.
4 +
5 +Run `VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1
6 +pytest tests/basic_correctness/test_preemption.py`.
7 +"""
8 +import pytest
9 +
10 +from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
11 + ENABLE_ARTIFICIAL_PREEMPT)
12 +
13 +MODELS = [
14 + "facebook/opt-125m",
15 +]
16 +
17 +assert ENABLE_ARTIFICIAL_PREEMPT is True, (
18 + "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1. "
19 + "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest "
20 + "tests/basic_correctness/test_preemption.py`")
21 +
22 +
23 +@pytest.mark.parametrize("model", MODELS)
24 +@pytest.mark.parametrize("dtype", ["half"])
25 +@pytest.mark.parametrize("max_tokens", [96])
26 +@pytest.mark.parametrize("chunked_prefill_token_size", [16])
27 +def test_chunked_prefill_recompute(
28 + hf_runner,
29 + vllm_runner,
30 + example_prompts,
31 + model: str,
32 + dtype: str,
33 + max_tokens: int,
34 + chunked_prefill_token_size: int,
35 +) -> None:
36 + """Ensure that chunked prefill works with preemption."""
37 + max_num_seqs = min(chunked_prefill_token_size, 256)
38 + enable_chunked_prefill = False
39 + max_num_batched_tokens = None
40 + if chunked_prefill_token_size != -1:
41 + enable_chunked_prefill = True
42 + max_num_batched_tokens = chunked_prefill_token_size
43 +
44 + hf_model = hf_runner(model, dtype=dtype)
45 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
46 + del hf_model
47 +
48 + vllm_model = vllm_runner(
49 + model,
50 + dtype=dtype,
51 + max_num_batched_tokens=max_num_batched_tokens,
52 + enable_chunked_prefill=enable_chunked_prefill,
53 + max_num_seqs=max_num_seqs,
54 + )
55 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
56 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
57 + ARTIFICIAL_PREEMPTION_MAX_CNT)
58 + del vllm_model
59 +
60 + for i in range(len(example_prompts)):
61 + hf_output_ids, hf_output_str = hf_outputs[i]
62 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
63 + assert hf_output_str == vllm_output_str, (
64 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
65 + assert hf_output_ids == vllm_output_ids, (
66 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
67 +
68 +
69 +@pytest.mark.parametrize("model", MODELS)
70 +@pytest.mark.parametrize("dtype", ["float"])
71 +@pytest.mark.parametrize("max_tokens", [96])
72 +def test_preemption(
73 + hf_runner,
74 + vllm_runner,
75 + example_prompts,
76 + model: str,
77 + dtype: str,
78 + max_tokens: int,
79 +) -> None:
80 + """By default, recompute preemption is enabled"""
81 +
82 + hf_model = hf_runner(model, dtype=dtype)
83 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
84 + del hf_model
85 +
86 + vllm_model = vllm_runner(
87 + model,
88 + dtype=dtype,
89 + )
90 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
91 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
92 + ARTIFICIAL_PREEMPTION_MAX_CNT)
93 + del vllm_model
94 +
95 + for i in range(len(example_prompts)):
96 + hf_output_ids, hf_output_str = hf_outputs[i]
97 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
98 + assert hf_output_str == vllm_output_str, (
99 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
100 + assert hf_output_ids == vllm_output_ids, (
101 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
102 +
103 +
104 +@pytest.mark.parametrize("model", MODELS)
105 +@pytest.mark.parametrize("dtype", ["float"])
106 +@pytest.mark.parametrize("max_tokens", [96])
107 +@pytest.mark.parametrize("beam_width", [4])
108 +def test_swap(
109 + hf_runner,
110 + vllm_runner,
111 + example_prompts,
112 + model: str,
113 + dtype: str,
114 + max_tokens: int,
115 + beam_width: int,
116 +) -> None:
117 + """Use beam search enables swapping."""
118 + example_prompts = example_prompts[:1]
119 + hf_model = hf_runner(model, dtype=dtype)
120 + hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
121 + max_tokens)
122 + del hf_model
123 +
124 + vllm_model = vllm_runner(model, dtype=dtype, swap_space=10)
125 + vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
126 + max_tokens)
127 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
128 + ARTIFICIAL_PREEMPTION_MAX_CNT)
129 + del vllm_model
130 +
131 + for i in range(len(example_prompts)):
132 + hf_output_ids, _ = hf_outputs[i]
133 + vllm_output_ids, _ = vllm_outputs[i]
134 + assert len(hf_output_ids) == len(vllm_output_ids)
135 + for j in range(len(hf_output_ids)):
136 + assert hf_output_ids[j] == vllm_output_ids[j], (
137 + f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
138 + f"vLLM: {vllm_output_ids}")
tests/conftest.py
+2 -1
@@ -296,6 +296,7 @@ class VllmRunner:
296 tensor_parallel_size: int = 1,
297 block_size: int = 16,
298 enable_chunked_prefill: bool = False,
299 + swap_space=4,
300 **kwargs,
301 ) -> None:
302 self.model = LLM(
@@ -303,7 +304,7 @@ class VllmRunner:
304 tokenizer=tokenizer_name,
305 trust_remote_code=True,
306 dtype=dtype,
306 - swap_space=0,
307 + swap_space=swap_space,
308 disable_log_stats=disable_log_stats,
309 tensor_parallel_size=tensor_parallel_size,
310 max_model_len=max_model_len,
tests/spec_decode/test_spec_decode_worker.py
+3 -3
@@ -33,7 +33,7 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
33 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
34 metrics_collector)
35
36 - exception_secret = 'artifical stop'
36 + exception_secret = 'artificial stop'
37 draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
38
39 execute_model_data, _, _ = create_batch(batch_size, k)
@@ -101,7 +101,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
101 proposal_probs=proposal_probs,
102 proposal_lens=proposal_lens)
103
104 - exception_secret = 'artifical stop'
104 + exception_secret = 'artificial stop'
105 target_worker.execute_model.side_effect = ValueError(exception_secret)
106
107 with pytest.raises(ValueError, match=exception_secret):
@@ -197,7 +197,7 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
197
198 target_worker.execute_model.return_value = [target_output[0]]
199
200 - exception_secret = 'artifical stop'
200 + exception_secret = 'artificial stop'
201 rejection_sampler.side_effect = ValueError(exception_secret)
202
203 with pytest.raises(ValueError, match=exception_secret):
vllm/core/scheduler.py
+28 -8
@@ -1,4 +1,6 @@
1 import enum
2 +import os
3 +import random
4 import time
5 from collections import deque
6 from dataclasses import dataclass, field
@@ -15,6 +17,13 @@ from vllm.utils import merge_dicts
17
18 logger = init_logger(__name__)
19
20 +# Test-only. If configured, decode is preempted with
21 +# ARTIFICIAL_PREEMPTION_PROB% probability.
22 +ENABLE_ARTIFICIAL_PREEMPT = bool(
23 + os.getenv("VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT", False)) # noqa
24 +ARTIFICIAL_PREEMPTION_PROB = 0.5
25 +ARTIFICIAL_PREEMPTION_MAX_CNT = 500
26 +
27
28 class PreemptionMode(enum.Enum):
29 """Preemption modes.
@@ -286,6 +295,13 @@ class Scheduler:
295 # Latency of the last prompt step
296 self.last_prompt_latency = 0.0
297
298 + # The following field is test-only. It is used to inject artificial
299 + # preemption.
300 + self.enable_artificial_preemption = ENABLE_ARTIFICIAL_PREEMPT
301 + self.artificial_preempt_cnt = (ARTIFICIAL_PREEMPTION_MAX_CNT
302 + if self.enable_artificial_preemption
303 + else 0)
304 +
305 @property
306 def lora_enabled(self) -> bool:
307 return bool(self.lora_config)
@@ -386,15 +402,13 @@ class Scheduler:
402 # groups to preempt.
403 now = time.time()
404 running_queue = policy.sort_by_priority(now, running_queue)
389 -
405 while running_queue:
406 seq_group = running_queue[0]
407 num_running_tokens = self._get_num_new_tokens(
408 seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
409
395 - # We can have up to 1 running prefill at any given time in running
396 - # queue, which means we can guarantee chunk size is at least 1.
397 - assert num_running_tokens != 0
410 + if num_running_tokens == 0:
411 + break
412
413 running_queue.popleft()
414 while not self._can_append_slots(seq_group):
@@ -449,9 +463,6 @@ class Scheduler:
463 if curr_loras is not None and seq_group.lora_int_id > 0:
464 curr_loras.add(seq_group.lora_int_id)
465
452 - # Make sure all queues are updated.
453 - assert len(running_queue) == 0
454 -
466 return running_queue, SchedulerRunningOutputs(
467 decode_seq_groups=decode_seq_groups,
468 prefill_seq_groups=prefill_seq_groups,
@@ -545,7 +556,6 @@ class Scheduler:
556 ScheduledSequenceGroup(seq_group,
557 token_chunk_size=num_new_tokens))
558 else:
548 - assert num_new_tokens == 1
559 decode_seq_groups.append(
560 ScheduledSequenceGroup(seq_group, token_chunk_size=1))
561 budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
@@ -868,6 +878,13 @@ class Scheduler:
878 """Determine whether or not we have enough space in the KV cache to
879 continue generation of the sequence group.
880 """
881 + # It is True only for testing case to trigger artificial preemption.
882 + if (self.enable_artificial_preemption
883 + and random.uniform(0, 1) < ARTIFICIAL_PREEMPTION_PROB
884 + and self.artificial_preempt_cnt > 0):
885 + self.artificial_preempt_cnt -= 1
886 + return False
887 +
888 # Appending slots only occurs in decoding.
889 is_prefill = False
890
@@ -1116,11 +1133,14 @@ class Scheduler:
1133 if `enable_chunking` is True. If a sequence group has multiple
1134 sequences (e.g., running beam search), it means it is in decoding
1135 phase, so chunking doesn't happen.
1136 +
1137 + Returns 0 if the new token cannot be computed due to token budget.
1138 """
1139 num_new_tokens = 0
1140 seqs = seq_group.get_seqs(status=status)
1141 for seq in seqs:
1142 num_new_tokens += seq.get_num_new_tokens()
1143 + assert num_new_tokens > 0
1144 # Chunk if a running request cannot fit in.
1145 # If number of seq > 1, it means it is doing beam search in a
1146 # decode phase. Do not chunk in that case.