@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14dbd5a767
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] H2O Danube3-4b (#6451)

Joe committed Jul 26, 2024 at 20:47 UTC 14dbd5a7674e5de2862c18adb711d9feecd35063
10 files changed +79 -7
.buildkite/run-cpu-test.sh
+1 -1
@@ -23,7 +23,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23 # Run basic model test
24 docker exec cpu-test bash -c "
25 pip install pytest Pillow protobuf
26 - pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py" # Mamba on CPU is not supported
26 + pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
27
28 # online inference
29 docker exec cpu-test bash -c "
benchmarks/kernels/benchmark_paged_attention.py
+1 -1
@@ -175,7 +175,7 @@ if __name__ == '__main__':
175 parser.add_argument("--num-kv-heads", type=int, default=8)
176 parser.add_argument("--head-size",
177 type=int,
178 - choices=[64, 80, 96, 112, 128, 192, 256],
178 + choices=[64, 80, 96, 112, 120, 128, 192, 256],
179 default=128)
180 parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
181 parser.add_argument("--use-alibi", action="store_true")
benchmarks/kernels/benchmark_rope.py
+1 -1
@@ -94,7 +94,7 @@ if __name__ == '__main__':
94 parser.add_argument("--num-heads", type=int, default=8)
95 parser.add_argument("--head-size",
96 type=int,
97 - choices=[64, 80, 96, 112, 128, 192, 256],
97 + choices=[64, 80, 96, 112, 120, 128, 192, 256],
98 default=128)
99 parser.add_argument("--rotary-dim", type=int, choices=[16, 32], default=32)
100 parser.add_argument("--dtype",
csrc/attention/attention_kernels.cu
+6
@@ -751,6 +751,9 @@ void paged_attention_v1_launcher(
751 case 112:
752 LAUNCH_PAGED_ATTENTION_V1(112);
753 break;
754 + case 120:
755 + LAUNCH_PAGED_ATTENTION_V1(120);
756 + break;
757 case 128:
758 LAUNCH_PAGED_ATTENTION_V1(128);
759 break;
@@ -912,6 +915,9 @@ void paged_attention_v2_launcher(
915 case 112:
916 LAUNCH_PAGED_ATTENTION_V2(112);
917 break;
918 + case 120:
919 + LAUNCH_PAGED_ATTENTION_V2(120);
920 + break;
921 case 128:
922 LAUNCH_PAGED_ATTENTION_V2(128);
923 break;
tests/kernels/test_attention.py
+3 -1
@@ -28,7 +28,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28
29 # FlashAttention forward only supports head dimension at most 128
30 # https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
31 -HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256
31 +HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256
32 ] if not is_hip() else [64, 80, 96, 112, 128]
33
34 BLOCK_SIZES = [16, 32]
@@ -134,6 +134,8 @@ def test_paged_attention(
134 seed: int,
135 device: str,
136 ) -> None:
137 + if kv_cache_dtype == "fp8" and head_size % 16:
138 + pytest.skip()
139 random.seed(seed)
140 torch.random.manual_seed(seed)
141 if torch.cuda.is_available():
tests/kernels/test_cache.py
+7 -1
@@ -11,7 +11,7 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
11 NUM_TOKENS = [42] # Arbitrary values for testing
12 NUM_LAYERS = [1] # Arbitrary values for testing
13 NUM_HEADS = [8] # Arbitrary values for testing
14 -HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
14 +HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
15 BLOCK_SIZES = [8, 16, 32]
16
17 # Arbitrary values for testing
@@ -52,6 +52,8 @@ def test_copy_blocks(
52 kv_cache_dtype: str,
53 device: str,
54 ) -> None:
55 + if kv_cache_dtype == "fp8" and head_size % 16:
56 + pytest.skip()
57 random.seed(seed)
58 torch.random.manual_seed(seed)
59 if torch.cuda.is_available():
@@ -124,6 +126,8 @@ def test_reshape_and_cache(
126 device: str,
127 kv_cache_dtype: str,
128 ) -> None:
129 + if kv_cache_dtype == "fp8" and head_size % 16:
130 + pytest.skip()
131 random.seed(seed)
132 torch.random.manual_seed(seed)
133 if torch.cuda.is_available():
@@ -325,6 +329,8 @@ def test_swap_blocks(
329 ) -> None:
330 if kv_cache_dtype == "fp8" and "cpu" in direction:
331 pytest.skip()
332 + if kv_cache_dtype == "fp8" and head_size % 16:
333 + pytest.skip()
334 random.seed(seed)
335 torch.random.manual_seed(seed)
336 if torch.cuda.is_available():
tests/kernels/test_pos_encoding.py
+1 -1
@@ -10,7 +10,7 @@ from .allclose_default import get_default_atol, get_default_rtol
10
11 IS_NEOX_STYLE = [True, False]
12 DTYPES = [torch.half, torch.bfloat16, torch.float]
13 -HEAD_SIZES = [64, 80, 96, 112, 128, 192, 256]
13 +HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
14 ROTARY_DIMS = [None, 32] # None means rotary dim == head size
15 NUM_HEADS = [7, 17] # Arbitrary values for testing
16 BATCH_SIZES = [1, 5] # Arbitrary values for testing
tests/models/test_danube3_4b.py new
+52
@@ -0,0 +1,52 @@
1 +"""Compare the outputs of HF and vLLM when using greedy sampling.
2 +
3 +This tests danube3 separately because its head size isn't supported on CPU yet.
4 +
5 +Run `pytest tests/models/test_danube3_4b.py`.
6 +"""
7 +import pytest
8 +
9 +from .utils import check_outputs_equal
10 +
11 +MODELS = ["h2oai/h2o-danube3-4b-base"]
12 +
13 +target_dtype = "half"
14 +
15 +
16 +@pytest.mark.parametrize("model", MODELS)
17 +@pytest.mark.parametrize("dtype", [target_dtype])
18 +@pytest.mark.parametrize("max_tokens", [32])
19 +def test_models(
20 + hf_runner,
21 + vllm_runner,
22 + example_prompts,
23 + model: str,
24 + dtype: str,
25 + max_tokens: int,
26 +) -> None:
27 + with hf_runner(model, dtype=dtype) as hf_model:
28 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
29 +
30 + with vllm_runner(model, dtype=dtype) as vllm_model:
31 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
32 +
33 + check_outputs_equal(
34 + outputs_0_lst=hf_outputs,
35 + outputs_1_lst=vllm_outputs,
36 + name_0="hf",
37 + name_1="vllm",
38 + )
39 +
40 +
41 +@pytest.mark.parametrize("model", MODELS)
42 +@pytest.mark.parametrize("dtype", [target_dtype])
43 +def test_model_print(
44 + vllm_runner,
45 + model: str,
46 + dtype: str,
47 +) -> None:
48 + with vllm_runner(model, dtype=dtype) as vllm_model:
49 + # This test is for verifying whether the model's extra_repr
50 + # can be printed correctly.
51 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
52 + model_runner.model)
vllm/attention/ops/paged_attn.py
+1 -1
@@ -31,7 +31,7 @@ class PagedAttention:
31
32 @staticmethod
33 def get_supported_head_sizes() -> List[int]:
34 - return [64, 80, 96, 112, 128, 192, 256]
34 + return [64, 80, 96, 112, 120, 128, 192, 256]
35
36 @staticmethod
37 def get_kv_cache_shape(
vllm/utils.py
+6
@@ -508,6 +508,12 @@ def create_kv_caches_with_random(
508 seed: int = 0,
509 device: Optional[str] = "cuda",
510 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
511 +
512 + if cache_dtype == "fp8" and head_size % 16:
513 + raise ValueError(
514 + f"Does not support key cache of type fp8 with head_size {head_size}"
515 + )
516 +
517 torch.random.manual_seed(seed)
518 if torch.cuda.is_available():
519 torch.cuda.manual_seed(seed)