@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db9120cded
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel] Change interface to Mamba selective_state_update for continuous batching (#8039)

Tyler Michael Smith committed Sep 18, 2024 at 16:05 UTC db9120cdedba5033037432775417df0b6117495d
2 files changed +174 -3
tests/kernels/test_mamba_ssm.py
+146
@@ -323,3 +323,149 @@ def test_selective_state_update(dim, dstate, has_z, itype):
323
324 assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
325 assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
326 +
327 +
328 +@pytest.mark.parametrize("itype",
329 + [torch.float32, torch.float16, torch.bfloat16])
330 +@pytest.mark.parametrize("has_z", [False, True])
331 +@pytest.mark.parametrize("dstate", [16, 32, 64])
332 +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
333 +def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
334 + device = "cuda"
335 + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
336 + if itype == torch.bfloat16:
337 + rtol, atol = 7e-2, 7e-2
338 + if torch.version.hip:
339 + atol *= 2
340 + # set seed
341 + torch.random.manual_seed(0)
342 + batch_size = 16
343 +
344 + total_entries = 10 * batch_size
345 + state = torch.randn(total_entries, dim, dstate, dtype=itype, device=device)
346 + state_indices = torch.randperm(total_entries)[:batch_size].to(
347 + dtype=torch.int32, device=device)
348 +
349 + x = torch.randn(batch_size, dim, device=device, dtype=itype)
350 + dt = torch.randn(batch_size, dim, device=device, dtype=itype)
351 + dt_bias = torch.rand(dim, device=device) - 4.0
352 + A = -torch.rand(dim, dstate, device=device) - 1.0
353 + B = torch.randn(batch_size, dstate, device=device)
354 + C = torch.randn(batch_size, dstate, device=device)
355 + D = torch.randn(dim, device=device)
356 + z = torch.randn_like(x) if has_z else None
357 + state_ref = state[state_indices, :].detach().clone()
358 + out = selective_state_update(state,
359 + x,
360 + dt,
361 + A,
362 + B,
363 + C,
364 + D=D,
365 + z=z,
366 + dt_bias=dt_bias,
367 + dt_softplus=True,
368 + state_batch_indices=state_indices)
369 + out_ref = selective_state_update_ref(state_ref,
370 + x,
371 + dt,
372 + A,
373 + B,
374 + C,
375 + D=D,
376 + z=z,
377 + dt_bias=dt_bias,
378 + dt_softplus=True)
379 +
380 + assert torch.allclose(state[state_indices, :],
381 + state_ref,
382 + rtol=rtol,
383 + atol=atol)
384 + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
385 +
386 +
387 +@pytest.mark.parametrize("itype",
388 + [torch.float32, torch.float16, torch.bfloat16])
389 +@pytest.mark.parametrize("has_z", [False, True])
390 +@pytest.mark.parametrize("tie_hdim", [False, True])
391 +@pytest.mark.parametrize("ngroups", [1, 2, 4])
392 +@pytest.mark.parametrize("dstate", [16, 32, 64])
393 +@pytest.mark.parametrize("dim", [2048, 4096])
394 +def test_selective_state_update_with_heads_with_batch_indices(
395 + dim, dstate, ngroups, has_z, tie_hdim, itype):
396 + device = "cuda"
397 + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 3e-2)
398 + if itype == torch.bfloat16:
399 + rtol, atol = 1e-1, 1e-1
400 + # set seed
401 + torch.random.manual_seed(0)
402 + batch_size = 16
403 + headdim = 64
404 + nheads = dim // headdim
405 +
406 + total_entries = 10 * batch_size
407 + state = torch.randn(total_entries,
408 + nheads,
409 + headdim,
410 + dstate,
411 + dtype=itype,
412 + device=device)
413 + state_indices = torch.randperm(total_entries)[:batch_size].to(
414 + dtype=torch.int32, device=device)
415 +
416 + x = torch.randn(batch_size, nheads, headdim, device=device, dtype=itype)
417 + if not tie_hdim:
418 + dt = torch.randn(batch_size,
419 + nheads,
420 + headdim,
421 + device=device,
422 + dtype=itype)
423 + dt_bias = torch.rand(nheads, headdim, device=device) - 4.0
424 + A = -torch.rand(nheads, headdim, dstate, device=device) - 1.0
425 + D = torch.randn(nheads, headdim, device=device)
426 + else:
427 + dt = repeat(torch.randn(batch_size, nheads, device=device,
428 + dtype=itype),
429 + "b h -> b h p",
430 + p=headdim)
431 + dt_bias = repeat(torch.rand(nheads, device=device) - 4.0,
432 + "h -> h p",
433 + p=headdim)
434 + A = repeat(-torch.rand(nheads, device=device) - 1.0,
435 + "h -> h p n",
436 + p=headdim,
437 + n=dstate)
438 + D = repeat(torch.randn(nheads, device=device), "h -> h p", p=headdim)
439 + B = torch.randn(batch_size, ngroups, dstate, device=device)
440 + C = torch.randn(batch_size, ngroups, dstate, device=device)
441 + z = torch.randn_like(x) if has_z else None
442 + state_ref = state[state_indices, :].detach().clone()
443 + out = selective_state_update(state,
444 + x,
445 + dt,
446 + A,
447 + B,
448 + C,
449 + D=D,
450 + z=z,
451 + dt_bias=dt_bias,
452 + dt_softplus=True,
453 + state_batch_indices=state_indices)
454 + out_ref = selective_state_update_ref(state_ref,
455 + x,
456 + dt,
457 + A,
458 + B,
459 + C,
460 + D=D,
461 + z=z,
462 + dt_bias=dt_bias,
463 + dt_softplus=True)
464 +
465 + print(f"Output max diff: {(out - out_ref).abs().max().item()}")
466 + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
467 + assert torch.allclose(state[state_indices, :],
468 + state_ref,
469 + rtol=rtol,
470 + atol=atol)
471 + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
+28 -3
@@ -1,4 +1,5 @@
1 # Copyright (c) 2024, Tri Dao, Albert Gu.
2 +# Adapted from https://github.com/state-spaces/mamba/blob/main/mamba_ssm/ops/triton/selective_state_update.py
3
4 import torch
5 import triton
@@ -27,6 +28,10 @@ else:
28 {"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
29 @triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
30 @triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
31 +@triton.heuristics({
32 + "HAS_STATE_BATCH_INDICES":
33 + lambda args: args["state_batch_indices_ptr"] is not None
34 +})
35 @triton.heuristics(
36 {"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])})
37 @triton.jit
@@ -42,6 +47,7 @@ def _selective_scan_update_kernel(
47 D_ptr,
48 z_ptr,
49 out_ptr,
50 + state_batch_indices_ptr,
51 # Matrix dimensions
52 batch,
53 nheads,
@@ -85,12 +91,24 @@ def _selective_scan_update_kernel(
91 HAS_DT_BIAS: tl.constexpr,
92 HAS_D: tl.constexpr,
93 HAS_Z: tl.constexpr,
94 + HAS_STATE_BATCH_INDICES: tl.constexpr,
95 BLOCK_SIZE_DSTATE: tl.constexpr,
96 ):
97 pid_m = tl.program_id(axis=0)
98 pid_b = tl.program_id(axis=1)
99 pid_h = tl.program_id(axis=2)
93 - state_ptr += pid_b * stride_state_batch + pid_h * stride_state_head
100 +
101 + # If HAS_STATE_BATCH_INDICES is true, then the ssm state's batch coordinate
102 + # is taken from the state_batch_indices_ptr Otherwise, the state coordinate
103 + # is the same as the batch id.
104 + if HAS_STATE_BATCH_INDICES:
105 + state_batch_indices_ptr += pid_b
106 + state_batch_idx = tl.load(state_batch_indices_ptr)
107 + state_ptr += (state_batch_idx * stride_state_batch +
108 + pid_h * stride_state_head)
109 + else:
110 + state_ptr += pid_b * stride_state_batch + pid_h * stride_state_head
111 +
112 x_ptr += pid_b * stride_x_batch + pid_h * stride_x_head
113 dt_ptr += pid_b * stride_dt_batch + pid_h * stride_dt_head
114 if HAS_DT_BIAS:
@@ -177,7 +195,8 @@ def selective_state_update(state,
195 D=None,
196 z=None,
197 dt_bias=None,
180 - dt_softplus=False):
198 + dt_softplus=False,
199 + state_batch_indices=None):
200 """
201 Argument:
202 state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
@@ -211,7 +230,10 @@ def selective_state_update(state,
230 z = z.unsqueeze(1)
231 if dt_bias is not None and dt_bias.dim() == 1:
232 dt_bias = dt_bias.unsqueeze(0)
214 - batch, nheads, dim, dstate = state.shape
233 +
234 + _, nheads, dim, dstate = state.shape
235 + batch = x.shape[0]
236 +
237 assert x.shape == (batch, nheads, dim)
238 assert dt.shape == x.shape
239 assert A.shape == (nheads, dim, dstate)
@@ -225,6 +247,8 @@ def selective_state_update(state,
247 assert z.shape == x.shape
248 if dt_bias is not None:
249 assert dt_bias.shape == (nheads, dim)
250 + if state_batch_indices is not None:
251 + assert state_batch_indices.shape == (batch, )
252 out = torch.empty_like(x)
253 grid = lambda META: (triton.cdiv(dim, META['BLOCK_SIZE_M']), batch, nheads)
254 z_strides = ((z.stride(0), z.stride(1), z.stride(2)) if z is not None else
@@ -249,6 +273,7 @@ def selective_state_update(state,
273 D,
274 z,
275 out,
276 + state_batch_indices,
277 batch,
278 nheads,
279 dim,