@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cfae35b861
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add miscellaneous updates (#8)

Woosuk Kwon committed Mar 13, 2023 at 13:48 UTC cfae35b861c5fc0c9f3689f99c7aba2e4501beb8
7 files changed +44 -22
cacheflow/master/scheduler.py
+8 -7
@@ -158,8 +158,8 @@ class Scheduler:
158 # 3. Join new sequences if possible.
159 # NOTE: Here we implicitly assume FCFS scheduling.
160 # TODO(woosuk): Add a batching policy to control the batch size.
161 + self._fetch_inputs()
162 if not self.swapped:
162 - self._fetch_inputs()
163 for i, seq_group in enumerate(self.pending):
164 num_prompt_tokens = seq_group.seqs[0].get_len()
165 if self.block_manager.can_allocate(seq_group):
@@ -211,12 +211,13 @@ class Scheduler:
211 input_seq_groups.append(input_seq_group)
212
213 # 5. Execute the first stage of the pipeline.
214 - self.controllers[0].execute_stage(
215 - input_seq_groups,
216 - blocks_to_swap_in,
217 - blocks_to_swap_out,
218 - blocks_to_copy,
219 - )
214 + if (input_seq_groups or blocks_to_swap_in or blocks_to_swap_out):
215 + self.controllers[0].execute_stage(
216 + input_seq_groups,
217 + blocks_to_swap_in,
218 + blocks_to_swap_out,
219 + blocks_to_copy,
220 + )
221
222 def post_step(
223 self,
cacheflow/models/attention.py
+5 -6
@@ -12,7 +12,7 @@ from cacheflow.models import InputMetadata
12 class OPTCacheFlowAttention(nn.Module):
13
14 def __init__(self, scale: float) -> None:
15 - super().__init__()
15 + super(OPTCacheFlowAttention, self).__init__()
16 self.scale = float(scale)
17
18 self.flash_attn = FlashAttention(softmax_scale=self.scale)
@@ -106,8 +106,8 @@ class OPTCacheFlowAttention(nn.Module):
106 output = output.view(-1, num_heads, head_size)
107
108 # Compute the attention op for prompts.
109 - if input_metadata.num_prompts > 0:
110 - num_prompt_tokens = sum(input_metadata.prompt_lens)
109 + num_prompt_tokens = input_metadata.num_prompt_tokens
110 + if num_prompt_tokens > 0:
111 self.multi_query_kv_attention(
112 output[:num_prompt_tokens],
113 query[:num_prompt_tokens],
@@ -126,10 +126,9 @@ class OPTCacheFlowAttention(nn.Module):
126
127 if input_metadata.num_generation_tokens > 0:
128 # Compute the attention op for generation tokens.
129 - start_idx = sum(input_metadata.prompt_lens)
129 self.single_query_cached_kv_attention(
131 - output[start_idx:],
132 - query[start_idx:],
130 + output[num_prompt_tokens:],
131 + query[num_prompt_tokens:],
132 key_cache,
133 value_cache,
134 input_metadata)
cacheflow/models/memory_analyzer.py
+14 -4
@@ -5,7 +5,7 @@ from cacheflow.models.utils import get_cpu_memory
5 from cacheflow.models.utils import get_dtype_size
6 from cacheflow.models.utils import get_gpu_memory
7
8 -_GiB = 1 << 30
8 +_GiB = 1 << 30
9
10
11 class CacheFlowMemoryAnalyzer:
@@ -117,9 +117,19 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
117
118 def get_max_num_cpu_blocks(
119 self,
120 - memory_utilization: float = 0.25,
120 + swap_space: int,
121 ) -> int:
122 + swap_space = swap_space * _GiB
123 cpu_memory = get_cpu_memory()
123 - usable_memory = int(memory_utilization * cpu_memory)
124 - max_num_blocks = usable_memory // self._get_cache_block_size()
124 + if swap_space > 0.8 * cpu_memory:
125 + raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
126 + 'takes more than 80% of the available memory '
127 + f'({cpu_memory / _GiB:.2f} GiB).'
128 + 'Please check the swap space size.')
129 + if swap_space > 0.5 * cpu_memory:
130 + print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
131 + 'takes more than 50% of the available memory '
132 + f'({cpu_memory / _GiB:.2f} GiB).'
133 + 'This may slow the system performance.')
134 + max_num_blocks = swap_space // self._get_cache_block_size()
135 return max_num_blocks
cacheflow/models/sample.py
+1 -1
@@ -11,7 +11,7 @@ from cacheflow.sequence import SequenceOutputs
11 class Sampler(nn.Module):
12
13 def __init__(self) -> None:
14 - super().__init__()
14 + super(Sampler, self).__init__()
15
16 def forward(
17 self,
cacheflow/worker/worker.py
+7
@@ -191,6 +191,13 @@ class Worker:
191 else:
192 cache_events = None
193
194 + # If there is no input, we don't need to execute the model.
195 + if not input_seq_groups:
196 + if cache_events is not None:
197 + for event in cache_events:
198 + event.wait()
199 + return {}
200 +
201 # Prepare input tensors.
202 input_tokens, input_positions, input_metadata = self.prepare_inputs(
203 input_seq_groups)
csrc/cache_kernels.cu
+5 -2
@@ -1,5 +1,4 @@
1 #include <torch/extension.h>
2 -
2 #include <ATen/cuda/CUDAContext.h>
3
4 #include <algorithm>
@@ -73,6 +72,8 @@ void copy_blocks(
72 }
73 }
74
75 +namespace cacheflow {
76 +
77 template<typename scalar_t>
78 __global__ void reshape_and_cache_kernel(
79 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
@@ -112,6 +113,8 @@ __global__ void reshape_and_cache_kernel(
113 }
114 }
115
116 +} // namespace cacheflow
117 +
118 void reshape_and_cache(
119 torch::Tensor& key,
120 torch::Tensor& value,
@@ -131,7 +134,7 @@ void reshape_and_cache(
134 key.scalar_type(),
135 "reshape_and_cache_kernel",
136 [&] {
134 - reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
137 + cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
138 key.data_ptr<scalar_t>(),
139 value.data_ptr<scalar_t>(),
140 key_cache.data_ptr<scalar_t>(),
server.py
+4 -2
@@ -15,7 +15,8 @@ parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='
15 parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
16 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
17 parser.add_argument('--seed', type=int, default=0, help='random seed')
18 -parser.add_argument('--max-batch-size', type=int, default=2048, help='maximum number of batched tokens')
18 +parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
19 +parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
20 args = parser.parse_args()
21
22
@@ -27,7 +28,8 @@ def main():
28 )
29 num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
30 max_num_batched_tokens=args.max_batch_size)
30 - num_cpu_blocks = memory_analyzer.get_max_num_cpu_blocks()
31 + num_cpu_blocks = memory_analyzer.get_max_num_cpu_blocks(
32 + swap_space=args.swap_space)
33 print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
34
35 # Create a controller for each node.