@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ef95b0f06
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] use float32 precision in samplers/test_logprobs.py for comparing with HF (#6409)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jul 15, 2024 at 19:14 UTC 4ef95b0f0677f95d8181837bbeebca7fca5a2bb2
2 files changed +8 -1
tests/samplers/test_logprobs.py
+2 -1
@@ -11,7 +11,8 @@ MODELS = ["facebook/opt-125m"]
11
12
13 @pytest.mark.parametrize("model", MODELS)
14 -@pytest.mark.parametrize("dtype", ["half"])
14 +@pytest.mark.parametrize("dtype",
15 + ["float"]) # needed for comparing logprobs with HF
16 @pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
17 @pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
18 @pytest.mark.parametrize("detokenize", [True, False])
vllm/attention/ops/prefix_prefill.py
+6
@@ -687,6 +687,12 @@ if triton.__version__ >= "2.1.0":
687
688 cap = current_platform.get_device_capability()
689 BLOCK = 128 if cap[0] >= 8 else 64
690 +
691 + # need to reduce num. blocks when using fp32
692 + # due to increased use of GPU shared memory
693 + if q.dtype is torch.float32:
694 + BLOCK = BLOCK // 2
695 +
696 # shape constraints
697 Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
698 assert Lq == Lk and Lk == Lv