@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25ebed2f8c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][Minor] Cache np arange to reduce input preparation overhead (#11214)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 15, 2024 at 13:33 UTC 25ebed2f8ca6d747d63f2be9ede023c561851ac8
1 file changed +10 -2
vllm/v1/worker/gpu_model_runner.py
+10 -2
@@ -118,6 +118,12 @@ class GPUModelRunner:
118 dtype=self.dtype,
119 device=self.device)
120
121 + # OPTIMIZATION: Cache the tensors rather than creating them every step.
122 + self.arange_np = np.arange(max(self.max_num_reqs, self.max_model_len),
123 + dtype=np.int32)
124 + # NOTE(woosuk): These tensors are "stateless", i.e., they are literally
125 + # a faster version of creating a new tensor every time. Thus, we should
126 + # not make any assumptions about the values in these tensors.
127 self.input_ids_cpu = torch.zeros(self.max_num_tokens,
128 dtype=torch.int32,
129 device="cpu",
@@ -269,11 +275,13 @@ class GPUModelRunner:
275
276 # Get request indices.
277 # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
272 - req_indices = np.repeat(np.arange(num_reqs), num_scheduled_tokens)
278 + req_indices = np.repeat(self.arange_np[:num_reqs],
279 + num_scheduled_tokens)
280
281 # Get batched arange.
282 # E.g., [2, 5, 3] -> [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
276 - arange = np.concatenate([np.arange(n) for n in num_scheduled_tokens])
283 + arange = np.concatenate(
284 + [self.arange_np[:n] for n in num_scheduled_tokens])
285
286 # Get positions.
287 positions_np = self.positions_np[:total_num_scheduled_tokens]