A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1][Minor] Optimize token_ids_cpu copy (#11692)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jan 3, 2025 at 04:04 UTC
b55ed6ef8ab0dce7fb0f79ff292dafdb4d22610c
2 files changed
+9
-5
vllm/v1/worker/gpu_input_batch.py
+8
-5
@@ -66,8 +66,9 @@ class InputBatch:
66
pin_memory=False,
67
)
68
self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
69
- self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
69
+ self.num_tokens = np.zeros(max_num_reqs, dtype=np.int32)
70
self.num_prompt_tokens = np.zeros(max_num_reqs, dtype=np.int32)
71
+ self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
72
73
# Attention-related.
74
self.block_table = torch.zeros(
@@ -189,6 +190,7 @@ class InputBatch:
190
end_idx = start_idx + len(request.output_token_ids)
191
self.token_ids_cpu[req_index,
192
start_idx:end_idx] = request.output_token_ids
193
+ self.num_tokens[req_index] = request.num_tokens
194
195
self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
196
num_blocks = len(request.block_ids)
@@ -290,14 +292,15 @@ class InputBatch:
292
self.req_ids[last_req_index] = None
293
self.req_id_to_index[req_id] = empty_index
294
293
- # TODO(woosuk): Optimize the copy of token_ids_cpu and
294
- # block_table_cpu.
295
- self.token_ids_cpu[empty_index] = self.token_ids_cpu[
296
- last_req_index]
295
+ num_tokens = self.num_tokens[last_req_index]
296
+ self.token_ids_cpu[empty_index, :num_tokens] = self.token_ids_cpu[
297
+ last_req_index, :num_tokens]
298
+ self.num_tokens[empty_index] = num_tokens
299
self.num_prompt_tokens[empty_index] = \
300
self.num_prompt_tokens[last_req_index]
301
self.num_computed_tokens_cpu[
302
empty_index] = self.num_computed_tokens_cpu[last_req_index]
303
+ # TODO(woosuk): Optimize the copy of block_table_cpu.
304
self.block_table_cpu[empty_index] = self.block_table_cpu[
305
last_req_index]
306
self.temperature_cpu[empty_index] = self.temperature_cpu[
vllm/v1/worker/gpu_model_runner.py
+1
@@ -644,6 +644,7 @@ class GPUModelRunner:
644
# Append the sampled token to the output token ids.
645
token_id = sampled_token_ids[i]
646
self.input_batch.token_ids_cpu[i, seq_len] = token_id
647
+ self.input_batch.num_tokens[i] += 1
648
req_state.output_token_ids.append(token_id)
649
else:
650
# Ignore the sampled token from the partial request.