@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b55ed6ef8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1][Minor] Optimize token_ids_cpu copy (#11692)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 3, 2025 at 04:04 UTC b55ed6ef8ab0dce7fb0f79ff292dafdb4d22610c
2 files changed +9 -5
vllm/v1/worker/gpu_input_batch.py
+8 -5
@@ -66,8 +66,9 @@ class InputBatch:
66 pin_memory=False,
67 )
68 self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
69 - self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
69 + self.num_tokens = np.zeros(max_num_reqs, dtype=np.int32)
70 self.num_prompt_tokens = np.zeros(max_num_reqs, dtype=np.int32)
71 + self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
72
73 # Attention-related.
74 self.block_table = torch.zeros(
@@ -189,6 +190,7 @@ class InputBatch:
190 end_idx = start_idx + len(request.output_token_ids)
191 self.token_ids_cpu[req_index,
192 start_idx:end_idx] = request.output_token_ids
193 + self.num_tokens[req_index] = request.num_tokens
194
195 self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
196 num_blocks = len(request.block_ids)
@@ -290,14 +292,15 @@ class InputBatch:
292 self.req_ids[last_req_index] = None
293 self.req_id_to_index[req_id] = empty_index
294
293 - # TODO(woosuk): Optimize the copy of token_ids_cpu and
294 - # block_table_cpu.
295 - self.token_ids_cpu[empty_index] = self.token_ids_cpu[
296 - last_req_index]
295 + num_tokens = self.num_tokens[last_req_index]
296 + self.token_ids_cpu[empty_index, :num_tokens] = self.token_ids_cpu[
297 + last_req_index, :num_tokens]
298 + self.num_tokens[empty_index] = num_tokens
299 self.num_prompt_tokens[empty_index] = \
300 self.num_prompt_tokens[last_req_index]
301 self.num_computed_tokens_cpu[
302 empty_index] = self.num_computed_tokens_cpu[last_req_index]
303 + # TODO(woosuk): Optimize the copy of block_table_cpu.
304 self.block_table_cpu[empty_index] = self.block_table_cpu[
305 last_req_index]
306 self.temperature_cpu[empty_index] = self.temperature_cpu[
vllm/v1/worker/gpu_model_runner.py
+1
@@ -644,6 +644,7 @@ class GPUModelRunner:
644 # Append the sampled token to the output token ids.
645 token_id = sampled_token_ids[i]
646 self.input_batch.token_ids_cpu[i, seq_len] = token_id
647 + self.input_batch.num_tokens[i] += 1
648 req_state.output_token_ids.append(token_id)
649 else:
650 # Ignore the sampled token from the partial request.