A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] H2O Danube3-4b (#6451)
Joe committed
Jul 26, 2024 at 20:47 UTC
14dbd5a7674e5de2862c18adb711d9feecd35063
10 files changed
+79
-7
.buildkite/run-cpu-test.sh
+1
-1
@@ -23,7 +23,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
pip install pytest Pillow protobuf
26
- pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py" # Mamba on CPU is not supported
26
+ pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
27
28
# online inference
29
docker exec cpu-test bash -c "
benchmarks/kernels/benchmark_paged_attention.py
+1
-1
@@ -175,7 +175,7 @@ if __name__ == '__main__':
175
parser.add_argument("--num-kv-heads", type=int, default=8)
176
parser.add_argument("--head-size",
177
type=int,
178
- choices=[64, 80, 96, 112, 128, 192, 256],
178
+ choices=[64, 80, 96, 112, 120, 128, 192, 256],
179
default=128)
180
parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
181
parser.add_argument("--use-alibi", action="store_true")
benchmarks/kernels/benchmark_rope.py
+1
-1
@@ -94,7 +94,7 @@ if __name__ == '__main__':
94
parser.add_argument("--num-heads", type=int, default=8)
95
parser.add_argument("--head-size",
96
type=int,
97
- choices=[64, 80, 96, 112, 128, 192, 256],
97
+ choices=[64, 80, 96, 112, 120, 128, 192, 256],
98
default=128)
99
parser.add_argument("--rotary-dim", type=int, choices=[16, 32], default=32)
100
parser.add_argument("--dtype",
csrc/attention/attention_kernels.cu
+6
@@ -751,6 +751,9 @@ void paged_attention_v1_launcher(
751
case 112:
752
LAUNCH_PAGED_ATTENTION_V1(112);
753
break;
754
+ case 120:
755
+ LAUNCH_PAGED_ATTENTION_V1(120);
756
+ break;
757
case 128:
758
LAUNCH_PAGED_ATTENTION_V1(128);
759
break;
@@ -912,6 +915,9 @@ void paged_attention_v2_launcher(
915
case 112:
916
LAUNCH_PAGED_ATTENTION_V2(112);
917
break;
918
+ case 120:
919
+ LAUNCH_PAGED_ATTENTION_V2(120);
920
+ break;
921
case 128:
922
LAUNCH_PAGED_ATTENTION_V2(128);
923
break;
tests/kernels/test_attention.py
+3
-1
@@ -28,7 +28,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28
29
# FlashAttention forward only supports head dimension at most 128
30
# https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
31
-HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256
31
+HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256
32
] if not is_hip() else [64, 80, 96, 112, 128]
33
34
BLOCK_SIZES = [16, 32]
@@ -134,6 +134,8 @@ def test_paged_attention(
134
seed: int,
135
device: str,
136
) -> None:
137
+ if kv_cache_dtype == "fp8" and head_size % 16:
138
+ pytest.skip()
139
random.seed(seed)
140
torch.random.manual_seed(seed)
141
if torch.cuda.is_available():
tests/kernels/test_cache.py
+7
-1
@@ -11,7 +11,7 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
11
NUM_TOKENS = [42] # Arbitrary values for testing
12
NUM_LAYERS = [1] # Arbitrary values for testing
13
NUM_HEADS = [8] # Arbitrary values for testing
14
-HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
14
+HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
15
BLOCK_SIZES = [8, 16, 32]
16
17
# Arbitrary values for testing
@@ -52,6 +52,8 @@ def test_copy_blocks(
52
kv_cache_dtype: str,
53
device: str,
54
) -> None:
55
+ if kv_cache_dtype == "fp8" and head_size % 16:
56
+ pytest.skip()
57
random.seed(seed)
58
torch.random.manual_seed(seed)
59
if torch.cuda.is_available():
@@ -124,6 +126,8 @@ def test_reshape_and_cache(
126
device: str,
127
kv_cache_dtype: str,
128
) -> None:
129
+ if kv_cache_dtype == "fp8" and head_size % 16:
130
+ pytest.skip()
131
random.seed(seed)
132
torch.random.manual_seed(seed)
133
if torch.cuda.is_available():
@@ -325,6 +329,8 @@ def test_swap_blocks(
329
) -> None:
330
if kv_cache_dtype == "fp8" and "cpu" in direction:
331
pytest.skip()
332
+ if kv_cache_dtype == "fp8" and head_size % 16:
333
+ pytest.skip()
334
random.seed(seed)
335
torch.random.manual_seed(seed)
336
if torch.cuda.is_available():
tests/kernels/test_pos_encoding.py
+1
-1
@@ -10,7 +10,7 @@ from .allclose_default import get_default_atol, get_default_rtol
10
11
IS_NEOX_STYLE = [True, False]
12
DTYPES = [torch.half, torch.bfloat16, torch.float]
13
-HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
13
+HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
14
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
15
NUM_HEADS = [7, 17] # Arbitrary values for testing
16
BATCH_SIZES = [1, 5] # Arbitrary values for testing
tests/models/test_danube3_4b.py
new
+52
@@ -0,0 +1,52 @@
1
+"""Compare the outputs of HF and vLLM when using greedy sampling.
2
+
3
+This tests danube3 separately because its head size isn't supported on CPU yet.
4
+
5
+Run `pytest tests/models/test_danube3_4b.py`.
6
+"""
7
+import pytest
8
+
9
+from .utils import check_outputs_equal
10
+
11
+MODELS = ["h2oai/h2o-danube3-4b-base"]
12
+
13
+target_dtype = "half"
14
+
15
+
16
+@pytest.mark.parametrize("model", MODELS)
17
+@pytest.mark.parametrize("dtype", [target_dtype])
18
+@pytest.mark.parametrize("max_tokens", [32])
19
+def test_models(
20
+ hf_runner,
21
+ vllm_runner,
22
+ example_prompts,
23
+ model: str,
24
+ dtype: str,
25
+ max_tokens: int,
26
+) -> None:
27
+ with hf_runner(model, dtype=dtype) as hf_model:
28
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
29
+
30
+ with vllm_runner(model, dtype=dtype) as vllm_model:
31
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
32
+
33
+ check_outputs_equal(
34
+ outputs_0_lst=hf_outputs,
35
+ outputs_1_lst=vllm_outputs,
36
+ name_0="hf",
37
+ name_1="vllm",
38
+ )
39
+
40
+
41
+@pytest.mark.parametrize("model", MODELS)
42
+@pytest.mark.parametrize("dtype", [target_dtype])
43
+def test_model_print(
44
+ vllm_runner,
45
+ model: str,
46
+ dtype: str,
47
+) -> None:
48
+ with vllm_runner(model, dtype=dtype) as vllm_model:
49
+ # This test is for verifying whether the model's extra_repr
50
+ # can be printed correctly.
51
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
52
+ model_runner.model)
vllm/attention/ops/paged_attn.py
+1
-1
@@ -31,7 +31,7 @@ class PagedAttention:
31
32
@staticmethod
33
def get_supported_head_sizes() -> List[int]:
34
- return [64, 80, 96, 112, 128, 192, 256]
34
+ return [64, 80, 96, 112, 120, 128, 192, 256]
35
36
@staticmethod
37
def get_kv_cache_shape(
vllm/utils.py
+6
@@ -508,6 +508,12 @@ def create_kv_caches_with_random(
508
seed: int = 0,
509
device: Optional[str] = "cuda",
510
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
511
+
512
+ if cache_dtype == "fp8" and head_size % 16:
513
+ raise ValueError(
514
+ f"Does not support key cache of type fp8 with head_size {head_size}"
515
+ )
516
+
517
torch.random.manual_seed(seed)
518
if torch.cuda.is_available():
519
torch.cuda.manual_seed(seed)