@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e8ca973eb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] fix flashinfer cudagraph capture for PP (#6708)

William Lin committed Jul 23, 2024 at 18:49 UTC 5e8ca973ebd5584582923b8ed1d3d823769a80a5
2 files changed +31 -7
tests/distributed/test_pipeline_parallel.py
+24
@@ -61,3 +61,27 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
61 tp_args.append("--enforce-eager")
62
63 compare_two_settings(MODEL_NAME, pp_args, tp_args)
64 +
65 +
66 +@pytest.mark.parametrize("PP_SIZE, MODEL_NAME", [
67 + (2, "JackFram/llama-160m"),
68 +])
69 +@pytest.mark.parametrize("ATTN_BACKEND", [
70 + "FLASH_ATTN",
71 + "FLASHINFER",
72 +])
73 +def test_pp_cudagraph(PP_SIZE, MODEL_NAME, ATTN_BACKEND):
74 + cudagraph_args = [
75 + # use half precision for speed and memory savings in CI environment
76 + "--dtype",
77 + "float16",
78 + "--pipeline-parallel-size",
79 + str(PP_SIZE),
80 + "--distributed-executor-backend",
81 + "ray",
82 + ]
83 + os.environ["VLLM_ATTENTION_BACKEND"] = ATTN_BACKEND
84 +
85 + eager_args = cudagraph_args + ["--enforce-eager"]
86 +
87 + compare_two_settings(MODEL_NAME, eager_args, cudagraph_args)
vllm/worker/model_runner.py
+7 -7
@@ -1040,9 +1040,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1040 self.parallel_config.pipeline_parallel_size):
1041 for batch_size in reversed(batch_size_capture_list):
1042 if self.attn_backend.get_name() == "flashinfer":
1043 - indptr_buffer = indptr_buffer[:batch_size + 1]
1044 - last_page_len_buffer = last_page_len_buffer[:
1045 - batch_size]
1043 + _indptr_buffer = indptr_buffer[:batch_size + 1]
1044 + _last_page_len_buffer = last_page_len_buffer[:
1045 + batch_size]
1046
1047 num_qo_heads = (
1048 self.model_config.get_num_attention_heads(
@@ -1055,8 +1055,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1055 use_tensor_cores = False
1056 decode_wrapper = \
1057 CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
1058 - decode_workspace_buffer, indptr_buffer,
1059 - indices_buffer, last_page_len_buffer, "NHD",
1058 + decode_workspace_buffer, _indptr_buffer,
1059 + indices_buffer, _last_page_len_buffer, "NHD",
1060 use_tensor_cores)
1061 kv_cache_dtype = get_kv_cache_torch_dtype(
1062 self.kv_cache_dtype, self.model_config.dtype)
@@ -1131,10 +1131,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1131 self.model, self.attn_backend.get_name())
1132
1133 if self.attn_backend.get_name() == "flashinfer":
1134 - graph_runner.flashinfer_indptr_buffer = indptr_buffer
1134 + graph_runner.flashinfer_indptr_buffer = _indptr_buffer
1135 graph_runner.flashinfer_indices_buffer = indices_buffer
1136 graph_runner.flashinfer_last_page_len_buffer = \
1137 - last_page_len_buffer
1137 + _last_page_len_buffer
1138 graph_runner.flashinfer_decode_workspace_buffer = \
1139 decode_workspace_buffer
1140 graph_runner.flashinfer_decode_wrapper = \