A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] use float32 precision in samplers/test_logprobs.py for comparing with HF (#6409)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 15, 2024 at 19:14 UTC
4ef95b0f0677f95d8181837bbeebca7fca5a2bb2
2 files changed
+8
-1
tests/samplers/test_logprobs.py
+2
-1
@@ -11,7 +11,8 @@ MODELS = ["facebook/opt-125m"]
11
12
13
@pytest.mark.parametrize("model", MODELS)
14
-@pytest.mark.parametrize("dtype", ["half"])
14
+@pytest.mark.parametrize("dtype",
15
+ ["float"]) # needed for comparing logprobs with HF
16
@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
17
@pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
18
@pytest.mark.parametrize("detokenize", [True, False])
vllm/attention/ops/prefix_prefill.py
+6
@@ -687,6 +687,12 @@ if triton.__version__ >= "2.1.0":
687
688
cap = current_platform.get_device_capability()
689
BLOCK = 128 if cap[0] >= 8 else 64
690
+
691
+ # need to reduce num. blocks when using fp32
692
+ # due to increased use of GPU shared memory
693
+ if q.dtype is torch.float32:
694
+ BLOCK = BLOCK // 2
695
+
696
# shape constraints
697
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
698
assert Lq == Lk and Lk == Lv