A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Change LoRA embedding sharding to support loading methods (#5038)
Antoni Baum committed
Jun 6, 2024 at 19:07 UTC
ccdc490dda3f534c63c1faf29a638e65594d0dc3
11 files changed
+661
-129
.buildkite/test-pipeline.yaml
+2
-8
@@ -46,6 +46,7 @@ steps:
46
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
47
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
48
- pytest -v -s spec_decode/e2e/test_integration_dist.py
49
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
50
51
- label: Distributed Tests (Multiple Groups)
52
#mirror_hardwares: [amd]
@@ -138,14 +139,7 @@ steps:
139
num_gpus: 4
140
# This test runs llama 13B, so it is required to run on 4 GPUs.
141
commands:
141
- # Temporarily run this way because we cannot clean up GPU mem usage
142
- # for multi GPU tests.
143
- # TODO(sang): Fix it.
144
- - pytest -v -s lora/test_long_context.py::test_rotary_emb_replaced
145
- - pytest -v -s lora/test_long_context.py::test_batched_rope_kernel
146
- - pytest -v -s lora/test_long_context.py::test_self_consistency
147
- - pytest -v -s lora/test_long_context.py::test_quality
148
- - pytest -v -s lora/test_long_context.py::test_max_len
142
+ - pytest -v -s -x lora/test_long_context.py
143
144
- label: Tensorizer Test
145
#mirror_hardwares: [amd]
tests/conftest.py
+21
@@ -1,6 +1,8 @@
1
import contextlib
2
import gc
3
import os
4
+import subprocess
5
+import sys
6
from typing import Any, Dict, List, Optional, Tuple, TypeVar
7
8
import pytest
@@ -522,3 +524,22 @@ def caplog_vllm(temporary_enable_log_propagate, caplog):
524
# To capture vllm log, we should enable propagate=True temporarily
525
# because caplog depends on logs propagated to the root logger.
526
yield caplog
527
+
528
+
529
+@pytest.fixture(scope="session")
530
+def num_gpus_available():
531
+ """Get number of GPUs without initializing the CUDA context
532
+ in current process."""
533
+
534
+ try:
535
+ out = subprocess.run([
536
+ sys.executable, "-c",
537
+ "import torch; print(torch.cuda.device_count())"
538
+ ],
539
+ capture_output=True,
540
+ check=True,
541
+ text=True)
542
+ except subprocess.CalledProcessError as e:
543
+ logger.warning("Failed to get number of GPUs.", exc_info=e)
544
+ return 0
545
+ return int(out.stdout.strip())
tests/lora/conftest.py
+16
-2
@@ -42,10 +42,24 @@ def cleanup():
42
ray.shutdown()
43
44
45
+@pytest.fixture()
46
+def should_do_global_cleanup_after_test(request) -> bool:
47
+ """Allow subdirectories to skip global cleanup by overriding this fixture.
48
+ This can provide a ~10x speedup for non-GPU unit tests since they don't need
49
+ to initialize torch.
50
+ """
51
+
52
+ if request.node.get_closest_marker("skip_global_cleanup"):
53
+ return False
54
+
55
+ return True
56
+
57
+
58
@pytest.fixture(autouse=True)
46
-def cleanup_fixture():
59
+def cleanup_fixture(should_do_global_cleanup_after_test: bool):
60
yield
48
- cleanup()
61
+ if should_do_global_cleanup_after_test:
62
+ cleanup()
63
64
65
@pytest.fixture
tests/lora/test_layers.py
+217
-2
@@ -2,6 +2,7 @@ import random
2
from copy import deepcopy
3
from dataclasses import dataclass
4
from typing import Dict, List, Optional, Tuple
5
+from unittest.mock import patch
6
7
import pytest
8
import torch
@@ -32,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
- ParallelLMHead, VocabParallelEmbedding)
36
+ ParallelLMHead, VocabParallelEmbedding, get_masked_input_and_mask)
37
from vllm.model_executor.utils import set_random_seed
38
39
from .utils import DummyLoRAManager
@@ -427,7 +428,8 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
428
logits_processor = LogitsProcessor(
429
vocab_size + lora_config.lora_extra_vocab_size, vocab_size)
430
lora_logits_processor = LogitsProcessorWithLoRA(
430
- logits_processor, 1024, linear.weight.dtype, linear.weight.device)
431
+ logits_processor, 1024, linear.weight.dtype, linear.weight.device,
432
+ None)
433
lora_logits_processor.create_lora_weights(max_loras, lora_config)
434
435
return linear, logits_processor, lora_logits_processor
@@ -867,3 +869,216 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
869
870
torch.allclose(ref_q, actual_q)
871
torch.allclose(ref_k, actual_k)
872
+
873
+
874
+@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
875
+@pytest.mark.parametrize("seed", list(range(256)))
876
+def test_vocab_parallel_embedding_indices(tp_size, seed):
877
+ random.seed(seed)
878
+ vocab_size = random.randint(4000, 64000)
879
+ added_vocab_size = random.randint(0, 1024)
880
+ org_vocab_size = vocab_size - added_vocab_size
881
+ last_org_vocab_end_index = 0
882
+ last_added_vocab_end_index = org_vocab_size
883
+ computed_vocab_size = 0
884
+ computed_org_vocab_size = 0
885
+ computed_added_vocab_size = 0
886
+ vocab_size_padded = -1
887
+
888
+ all_org_tokens = []
889
+ all_added_tokens = []
890
+ token_ids = []
891
+
892
+ for tp_rank in range(tp_size):
893
+ with patch(
894
+ "vllm.model_executor.layers.vocab_parallel_embedding.get_tensor_model_parallel_rank",
895
+ return_value=tp_rank
896
+ ), patch(
897
+ "vllm.model_executor.layers.vocab_parallel_embedding.get_tensor_model_parallel_world_size",
898
+ return_value=tp_size):
899
+ vocab_embedding = VocabParallelEmbedding(
900
+ vocab_size, 1, org_num_embeddings=org_vocab_size)
901
+ vocab_size_padded = vocab_embedding.num_embeddings_padded
902
+ shard_indices = vocab_embedding.shard_indices
903
+ # Assert that the ranges are contiguous
904
+ assert shard_indices.org_vocab_start_index == last_org_vocab_end_index
905
+ assert (shard_indices.added_vocab_start_index ==
906
+ last_added_vocab_end_index)
907
+
908
+ # Ensure that we are not exceeding the vocab size
909
+ computed_vocab_size += shard_indices.num_elements_padded
910
+ computed_org_vocab_size += shard_indices.num_org_elements
911
+ computed_added_vocab_size += shard_indices.num_added_elements
912
+
913
+ # Ensure that the ranges are not overlapping
914
+ all_org_tokens.extend(
915
+ range(shard_indices.org_vocab_start_index,
916
+ shard_indices.org_vocab_end_index))
917
+ all_added_tokens.extend(
918
+ range(shard_indices.added_vocab_start_index,
919
+ shard_indices.added_vocab_end_index))
920
+
921
+ token_ids.extend(
922
+ range(shard_indices.org_vocab_start_index,
923
+ shard_indices.org_vocab_end_index))
924
+ token_ids.extend([-1] * (shard_indices.num_org_elements_padded -
925
+ shard_indices.num_org_elements))
926
+ token_ids.extend(
927
+ range(shard_indices.added_vocab_start_index,
928
+ shard_indices.added_vocab_end_index))
929
+ token_ids.extend([-1] * (shard_indices.num_added_elements_padded -
930
+ shard_indices.num_added_elements))
931
+
932
+ last_org_vocab_end_index = shard_indices.org_vocab_end_index
933
+ last_added_vocab_end_index = shard_indices.added_vocab_end_index
934
+
935
+ assert computed_vocab_size == vocab_size_padded
936
+ assert computed_org_vocab_size == org_vocab_size
937
+ assert computed_added_vocab_size == added_vocab_size
938
+
939
+ # Ensure that the ranges are not overlapping
940
+ assert len(all_org_tokens) == len(set(all_org_tokens))
941
+ assert len(all_added_tokens) == len(set(all_added_tokens))
942
+ assert not set(all_org_tokens).intersection(set(all_added_tokens))
943
+
944
+ token_ids_tensor = torch.tensor(token_ids, dtype=torch.long)
945
+ reindex_mapping = vocab_embedding.get_sharded_to_full_mapping()
946
+ assert reindex_mapping is not None or tp_size == 1
947
+ if reindex_mapping is not None:
948
+ reindexed_token_ids = token_ids_tensor[reindex_mapping]
949
+ expected = torch.tensor(list(range(0, vocab_size)))
950
+ assert reindexed_token_ids[:vocab_size].equal(expected)
951
+ assert torch.all(reindexed_token_ids[vocab_size:] == -1)
952
+
953
+
954
+def test_get_masked_input_and_mask():
955
+ x = torch.tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
956
+
957
+ # base tp 1 case, no padding
958
+ modified_x, _ = get_masked_input_and_mask(x,
959
+ org_vocab_start_index=0,
960
+ org_vocab_end_index=8,
961
+ added_vocab_start_index=8,
962
+ added_vocab_end_index=12,
963
+ num_org_vocab_padding=0)
964
+ assert torch.equal(x, modified_x)
965
+
966
+ # tp 2 case, no padding
967
+ modified_x_rank_0, _ = get_masked_input_and_mask(x,
968
+ org_vocab_start_index=0,
969
+ org_vocab_end_index=4,
970
+ added_vocab_start_index=8,
971
+ added_vocab_end_index=10,
972
+ num_org_vocab_padding=0)
973
+ modified_x_rank_1, _ = get_masked_input_and_mask(
974
+ x,
975
+ org_vocab_start_index=4,
976
+ org_vocab_end_index=8,
977
+ added_vocab_start_index=10,
978
+ added_vocab_end_index=12,
979
+ num_org_vocab_padding=0)
980
+ assert torch.equal(modified_x_rank_0,
981
+ torch.tensor([0, 1, 2, 3, 0, 0, 0, 0, 4, 5, 0, 0]))
982
+ assert torch.equal(modified_x_rank_1,
983
+ torch.tensor([0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 4, 5]))
984
+
985
+ # tp 4 case, no padding
986
+ modified_x_rank_0, _ = get_masked_input_and_mask(x,
987
+ org_vocab_start_index=0,
988
+ org_vocab_end_index=2,
989
+ added_vocab_start_index=8,
990
+ added_vocab_end_index=9,
991
+ num_org_vocab_padding=0)
992
+ modified_x_rank_1, _ = get_masked_input_and_mask(x,
993
+ org_vocab_start_index=2,
994
+ org_vocab_end_index=4,
995
+ added_vocab_start_index=9,
996
+ added_vocab_end_index=10,
997
+ num_org_vocab_padding=0)
998
+ modified_x_rank_2, _ = get_masked_input_and_mask(
999
+ x,
1000
+ org_vocab_start_index=4,
1001
+ org_vocab_end_index=6,
1002
+ added_vocab_start_index=10,
1003
+ added_vocab_end_index=11,
1004
+ num_org_vocab_padding=0)
1005
+ modified_x_rank_3, _ = get_masked_input_and_mask(
1006
+ x,
1007
+ org_vocab_start_index=6,
1008
+ org_vocab_end_index=8,
1009
+ added_vocab_start_index=11,
1010
+ added_vocab_end_index=12,
1011
+ num_org_vocab_padding=0)
1012
+ assert torch.equal(modified_x_rank_0,
1013
+ torch.tensor([0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0]))
1014
+ assert torch.equal(modified_x_rank_1,
1015
+ torch.tensor([0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0]))
1016
+ assert torch.equal(modified_x_rank_2,
1017
+ torch.tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0]))
1018
+ assert torch.equal(modified_x_rank_3,
1019
+ torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2]))
1020
+
1021
+ # base tp 1 case, with padding
1022
+ modified_x, _ = get_masked_input_and_mask(x,
1023
+ org_vocab_start_index=0,
1024
+ org_vocab_end_index=8,
1025
+ added_vocab_start_index=8,
1026
+ added_vocab_end_index=12,
1027
+ num_org_vocab_padding=2)
1028
+ assert torch.equal(modified_x,
1029
+ torch.tensor([0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13]))
1030
+
1031
+ # tp 2 case, with padding
1032
+ modified_x_rank_0, _ = get_masked_input_and_mask(x,
1033
+ org_vocab_start_index=0,
1034
+ org_vocab_end_index=4,
1035
+ added_vocab_start_index=8,
1036
+ added_vocab_end_index=10,
1037
+ num_org_vocab_padding=2)
1038
+ modified_x_rank_1, _ = get_masked_input_and_mask(
1039
+ x,
1040
+ org_vocab_start_index=4,
1041
+ org_vocab_end_index=8,
1042
+ added_vocab_start_index=10,
1043
+ added_vocab_end_index=12,
1044
+ num_org_vocab_padding=2)
1045
+ assert torch.equal(modified_x_rank_0,
1046
+ torch.tensor([0, 1, 2, 3, 0, 0, 0, 0, 6, 7, 0, 0]))
1047
+ assert torch.equal(modified_x_rank_1,
1048
+ torch.tensor([0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 6, 7]))
1049
+
1050
+ # tp 4 case, with padding
1051
+ modified_x_rank_0, _ = get_masked_input_and_mask(x,
1052
+ org_vocab_start_index=0,
1053
+ org_vocab_end_index=2,
1054
+ added_vocab_start_index=8,
1055
+ added_vocab_end_index=9,
1056
+ num_org_vocab_padding=2)
1057
+ modified_x_rank_1, _ = get_masked_input_and_mask(x,
1058
+ org_vocab_start_index=2,
1059
+ org_vocab_end_index=4,
1060
+ added_vocab_start_index=9,
1061
+ added_vocab_end_index=10,
1062
+ num_org_vocab_padding=2)
1063
+ modified_x_rank_2, _ = get_masked_input_and_mask(
1064
+ x,
1065
+ org_vocab_start_index=4,
1066
+ org_vocab_end_index=6,
1067
+ added_vocab_start_index=10,
1068
+ added_vocab_end_index=11,
1069
+ num_org_vocab_padding=2)
1070
+ modified_x_rank_3, _ = get_masked_input_and_mask(
1071
+ x,
1072
+ org_vocab_start_index=6,
1073
+ org_vocab_end_index=8,
1074
+ added_vocab_start_index=11,
1075
+ added_vocab_end_index=12,
1076
+ num_org_vocab_padding=2)
1077
+ assert torch.equal(modified_x_rank_0,
1078
+ torch.tensor([0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0]))
1079
+ assert torch.equal(modified_x_rank_1,
1080
+ torch.tensor([0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0]))
1081
+ assert torch.equal(modified_x_rank_2,
1082
+ torch.tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0]))
1083
+ assert torch.equal(modified_x_rank_3,
1084
+ torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4]))
tests/lora/test_llama.py
+7
-10
@@ -36,11 +36,10 @@ def do_sample(llm, lora_path: str, lora_id: int):
36
return generated_texts
37
38
39
-@pytest.mark.parametrize("tp_size", [1])
40
-def test_llama_lora(sql_lora_files, tp_size):
41
- # Cannot use as it will initialize torch.cuda too early...
42
- # if torch.cuda.device_count() < tp_size:
43
- # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
39
+@pytest.mark.parametrize("tp_size", [1, 2, 4])
40
+def test_llama_lora(sql_lora_files, tp_size, num_gpus_available):
41
+ if num_gpus_available < tp_size:
42
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
43
44
llm = vllm.LLM(MODEL_PATH,
45
enable_lora=True,
@@ -80,11 +79,9 @@ def test_llama_lora(sql_lora_files, tp_size):
79
print("removing lora")
80
81
83
-@pytest.mark.skip("Requires multiple GPUs")
84
-def test_llama_tensor_parallel_equality(sql_lora_files):
85
- # Cannot use as it will initialize torch.cuda too early...
86
- # if torch.cuda.device_count() < 4:
87
- # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
82
+def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
83
+ if num_gpus_available < 4:
84
+ pytest.skip("Not enough GPUs for tensor parallelism 4")
85
86
llm_tp1 = vllm.LLM(MODEL_PATH,
87
enable_lora=True,
tests/lora/test_long_context.py
+13
-10
@@ -102,22 +102,21 @@ def batched_generate(
102
return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
103
104
105
-@pytest.fixture
105
+@pytest.fixture(scope="module")
106
def lora_llm(long_context_infos):
107
scaling_factors = [
108
context_len_to_scaling_factor[info["context_length"]]
109
for info in long_context_infos.values()
110
]
111
112
- llm = vllm.LLM(
113
- "meta-llama/Llama-2-13b-chat-hf",
114
- enable_lora=True,
115
- max_num_seqs=16,
116
- max_loras=2,
117
- long_lora_scaling_factors=tuple(scaling_factors),
118
- max_num_batched_tokens=4096 * 8,
119
- tensor_parallel_size=4,
120
- )
112
+ llm = vllm.LLM("meta-llama/Llama-2-13b-chat-hf",
113
+ enable_lora=True,
114
+ max_num_seqs=16,
115
+ max_loras=2,
116
+ long_lora_scaling_factors=tuple(scaling_factors),
117
+ max_num_batched_tokens=4096 * 8,
118
+ tensor_parallel_size=4,
119
+ distributed_executor_backend="mp")
120
yield llm
121
del llm
122
@@ -154,6 +153,7 @@ def test_rotary_emb_replaced(dist_init):
153
assert rotary_emb_count == 32
154
155
156
+@pytest.mark.skip_global_cleanup
157
def test_batched_rope_kernel(lora_llm, long_context_infos):
158
"""We test the batched kernel by comparing the results of batched an
159
non-batched generation.
@@ -188,6 +188,7 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):
188
f"same:\n{batched}\n{non_batched}")
189
190
191
+@pytest.mark.skip_global_cleanup
192
def test_self_consistency(lora_llm, long_context_infos):
193
"""We test consistency of the batched kernel by permuting batched
194
inputs and comparing the results to the non-permuted batched results.
@@ -227,6 +228,7 @@ def test_self_consistency(lora_llm, long_context_infos):
228
f"\n{permutated_batched_results[permutation[i]]}")
229
230
231
+@pytest.mark.skip_global_cleanup
232
def test_quality(lora_llm, long_context_infos):
233
"""We test the quality of the answers given by the LoRA model by
234
comparing the generated text to the merged model's outputs.
@@ -257,6 +259,7 @@ def test_quality(lora_llm, long_context_infos):
259
assert np.mean(scores) > 0.5
260
261
262
+@pytest.mark.skip_global_cleanup
263
def test_max_len(lora_llm, long_context_infos):
264
"""Test that we raise an ValueError when the input of a given LoRA
265
model exceeds the maximum length."""
tests/test_sharded_state_loader.py
+80
-44
@@ -1,3 +1,4 @@
1
+import multiprocessing as mp
2
import os
3
import shutil
4
from tempfile import TemporaryDirectory
@@ -18,9 +19,7 @@ prompts = [
19
20
# Create a sampling params object.
21
sampling_params = SamplingParams(
21
- temperature=0.8,
22
- top_p=0.95,
23
- seed=0,
22
+ temperature=0,
23
max_tokens=256,
24
ignore_eos=True,
25
)
@@ -43,48 +42,85 @@ def test_filter_subtensors():
42
assert tensor.equal(state_dict[key])
43
44
45
+@pytest.fixture(scope="module")
46
+def llama_2_7b_files():
47
+ with TemporaryDirectory() as cache_dir:
48
+ input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
49
+ cache_dir=cache_dir,
50
+ ignore_patterns="*.bin*")
51
+ yield input_dir
52
+
53
+
54
+def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
55
+ llm_sharded_writer = LLM(model=input_dir, **kwargs)
56
+
57
+ # Dump worker states to output directory
58
+ llm_sharded_writer.llm_engine.model_executor.save_sharded_state(
59
+ path=output_dir)
60
+ # Copy metadata files to output directory
61
+ for file in os.listdir(input_dir):
62
+ if not any(file.endswith(ext) for ext in weights_patterns):
63
+ shutil.copy(f"{input_dir}/{file}", output_dir)
64
+
65
+
66
+def _run_generate(input_dir, queue: mp.Queue, **kwargs):
67
+ llm = LLM(model=input_dir, **kwargs)
68
+ gen = llm.generate(prompts, sampling_params)
69
+ queue.put([g.outputs[0].__dict__ for g in gen])
70
+ queue.close()
71
+ queue.join_thread()
72
+
73
+
74
@pytest.mark.parametrize("enable_lora", [False, True])
47
-def test_sharded_state_loader(enable_lora):
48
- weights_patterns = ("*.bin", "*.pt", "*.safetensors")
75
+@pytest.mark.parametrize("tp_size", [1, 2])
76
+def test_sharded_state_loader(enable_lora, tp_size, num_gpus_available,
77
+ llama_2_7b_files):
78
+ if num_gpus_available < tp_size:
79
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
80
50
- with TemporaryDirectory() as cache_dir, TemporaryDirectory() as output_dir:
51
- input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
52
- cache_dir=cache_dir)
53
-
54
- llm = LLM(
55
- model=input_dir,
56
- worker_use_ray=True,
57
- gpu_memory_utilization=0.3,
58
- )
59
-
60
- # Dump worker states to output directory
61
- model_executor = llm.llm_engine.model_executor
62
- model_executor.save_sharded_state(path=output_dir)
63
- # Copy metadata files to output directory
64
- for file in os.listdir(input_dir):
65
- if not any(file.endswith(ext) for ext in weights_patterns):
66
- shutil.copy(f"{input_dir}/{file}", output_dir)
67
- del llm.llm_engine.model_executor
68
-
69
- llm_before = LLM(
70
- model=input_dir,
71
- worker_use_ray=True,
72
- enable_lora=enable_lora,
73
- gpu_memory_utilization=0.3,
74
- )
75
- gen_before = llm_before.generate(prompts, sampling_params)
76
- out_before = [gen.outputs[0].__dict__ for gen in gen_before]
77
- del llm_before.llm_engine.model_executor
78
-
79
- llm_after = LLM(
80
- model=output_dir,
81
- worker_use_ray=True,
82
- enable_lora=enable_lora,
83
- gpu_memory_utilization=0.3,
84
- load_format="sharded_state",
85
- )
86
- gen_after = llm_after.generate(prompts, sampling_params)
87
- out_after = [gen.outputs[0].__dict__ for gen in gen_after]
88
- del llm_after.llm_engine.model_executor
81
+ weights_patterns = ("*.safetensors", )
82
+ gpu_memory_utilization = 0.8
83
+ input_dir = llama_2_7b_files
84
+ ctx = mp.get_context("spawn")
85
+
86
+ # Run in separate processes for memory & CUDA isolation
87
+ with TemporaryDirectory() as output_dir:
88
+ p = ctx.Process(target=_run_writer,
89
+ args=(input_dir, output_dir, weights_patterns),
90
+ kwargs=dict(
91
+ tensor_parallel_size=tp_size,
92
+ distributed_executor_backend="mp",
93
+ gpu_memory_utilization=gpu_memory_utilization,
94
+ enforce_eager=True,
95
+ ))
96
+ p.start()
97
+ p.join()
98
+
99
+ queue = ctx.Queue()
100
+
101
+ p = ctx.Process(target=_run_generate,
102
+ args=(input_dir, queue),
103
+ kwargs=dict(
104
+ distributed_executor_backend="mp",
105
+ enable_lora=enable_lora,
106
+ gpu_memory_utilization=gpu_memory_utilization,
107
+ tensor_parallel_size=tp_size,
108
+ ))
109
+ p.start()
110
+ p.join()
111
+ out_before = queue.get()
112
+
113
+ p = ctx.Process(target=_run_generate,
114
+ args=(output_dir, queue),
115
+ kwargs=dict(
116
+ distributed_executor_backend="mp",
117
+ enable_lora=enable_lora,
118
+ gpu_memory_utilization=gpu_memory_utilization,
119
+ tensor_parallel_size=tp_size,
120
+ load_format="sharded_state",
121
+ ))
122
+ p.start()
123
+ p.join()
124
+ out_after = queue.get()
125
126
assert out_before == out_after
vllm/lora/layers.py
+57
-19
@@ -215,19 +215,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
215
lora_config: LoRAConfig,
216
model_config: Optional[PretrainedConfig] = None) -> None:
217
218
- lora_vocab_start_idx = self.base_layer.org_vocab_size
219
- weights_idx = None
220
- if self.base_layer.vocab_end_index > lora_vocab_start_idx:
218
+ if self.base_layer.num_added_embeddings_per_partition > 0:
219
# We can start adding lora weights
222
- weights_idx = max(
223
- lora_vocab_start_idx - self.base_layer.vocab_start_index, 0)
224
- self.embeddings_slice = (self.base_layer.vocab_start_index -
225
- self.base_layer.org_vocab_size +
226
- weights_idx,
227
- self.base_layer.vocab_end_index -
228
- self.base_layer.org_vocab_size)
229
- self.embeddings_weights = self.base_layer.weight.data[weights_idx:]
230
- self.embeddings_weights.fill_(0)
220
+ self.embeddings_weights = self.base_layer.weight.data[
221
+ self.base_layer.num_org_embeddings_per_partition:self.
222
+ base_layer.num_org_embeddings_per_partition +
223
+ self.base_layer.num_added_embeddings_per_partition]
224
+ self.embeddings_slice = (
225
+ self.base_layer.shard_indices.added_vocab_start_index -
226
+ self.base_layer.org_vocab_size,
227
+ self.base_layer.shard_indices.added_vocab_end_index -
228
+ self.base_layer.org_vocab_size)
229
+ self.base_layer.weight.data[
230
+ self.base_layer.num_org_embeddings_per_partition:].fill_(0)
231
else:
232
self.embeddings_slice = None
233
self.embeddings_weights = None
@@ -1025,19 +1025,31 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1025
1026
1027
class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1028
+ """
1029
+ LoRA wrapper for LogitsProcessor, with extra logic to handle the
1030
+ application of the LoRA adapter and added LoRA vocabulary.
1031
+
1032
+ Args:
1033
+ base_layer: LogitsProcessor layer
1034
+ hidden_size: hidden size of the model
1035
+ dtype: data type of the model
1036
+ device: device of the model
1037
+ sharded_to_full_mapping: index mapping from sharded vocab to full vocab
1038
+ received from base_layer.get_sharded_to_full_mapping(). If None,
1039
+ no reindexing will be done.
1040
+ """
1041
1029
- def __init__(
1030
- self,
1031
- base_layer: LogitsProcessor,
1032
- hidden_size: int,
1033
- dtype: torch.dtype,
1034
- device: torch.device,
1035
- ) -> None:
1042
+ def __init__(self, base_layer: LogitsProcessor, hidden_size: int,
1043
+ dtype: torch.dtype, device: torch.device,
1044
+ sharded_to_full_mapping: Optional[List[int]]) -> None:
1045
super().__init__()
1046
self.base_layer = base_layer
1047
self.hidden_size = hidden_size
1048
self.dtype = dtype
1049
self.device = device
1050
+ self.tp_size = get_tensor_model_parallel_world_size()
1051
+ self.tp_rank = get_tensor_model_parallel_rank()
1052
+ self.sharded_to_full_mapping = sharded_to_full_mapping
1053
1054
@property
1055
def logits_as_input(self):
@@ -1098,6 +1110,13 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1110
dtype=self.dtype,
1111
device=self.device,
1112
)
1113
+ if self.sharded_to_full_mapping is not None:
1114
+ self.sharded_to_full_mapping_gpu = torch.tensor(
1115
+ self.sharded_to_full_mapping,
1116
+ device=self.device,
1117
+ dtype=torch.long)
1118
+ else:
1119
+ self.sharded_to_full_mapping_gpu = None
1120
# Lazily initialized.
1121
self.indices: torch.Tensor
1122
self.indices_len: List[int]
@@ -1154,6 +1173,25 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1173
if logits is None:
1174
return None
1175
1176
+ if self.sharded_to_full_mapping_gpu is not None:
1177
+ # Reindex full logits tensor to ensure 1:1 mapping between
1178
+ # index and token_id
1179
+ # Example for:
1180
+ # org_vocab_size = 4
1181
+ # added_vocab_size = 2
1182
+ # pad_to_size = 8
1183
+ # tp_size = 2
1184
+
1185
+ # indices: [0, 1, 2, 3, 4, 5, 6, 7]
1186
+ # token_id: [0, 1, 4, -1, 2, 3, 5, -1]
1187
+
1188
+ # Therefore, the mapping is expected to be:
1189
+ # [0, 1, 4, 6, 2, 3, 5, 7] so that when we reindex,
1190
+ # we get:
1191
+ # indices: [0, 1, 2, 3, 4, 5, 6, 7]
1192
+ # token_id: [0, 1, 2, 3, 4, 5, -1, -1]
1193
+ logits = logits[:, self.sharded_to_full_mapping_gpu]
1194
+
1195
lora_logits = torch.empty(
1196
self.embeddings_tensors.shape[0] + 1,
1197
self.embeddings_tensors.shape[1],
vllm/lora/utils.py
+2
-1
@@ -67,7 +67,8 @@ def from_layer_logits_processor(
67
model_config: Optional[PretrainedConfig] = None,
68
) -> LogitsProcessorWithLoRA:
69
ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
70
- lm_head.weight.dtype, lm_head.weight.device)
70
+ lm_head.weight.dtype, lm_head.weight.device,
71
+ lm_head.get_sharded_to_full_mapping())
72
ret.create_lora_weights(max_loras, lora_config, model_config)
73
return ret
74
vllm/model_executor/layers/vocab_parallel_embedding.py
+235
-25
@@ -1,4 +1,5 @@
1
-from typing import Optional, Sequence
1
+from dataclasses import dataclass
2
+from typing import List, Optional, Sequence, Tuple
3
4
import torch
5
import torch.nn.functional as F
@@ -18,18 +19,107 @@ def pad_vocab_size(vocab_size: int,
19
return ((vocab_size + pad_to - 1) // pad_to) * pad_to
20
21
21
-def vocab_range_from_per_partition_vocab_size(per_partition_vocab_size: int,
22
- rank: int) -> Sequence[int]:
22
+def vocab_range_from_per_partition_vocab_size(
23
+ per_partition_vocab_size: int,
24
+ rank: int,
25
+ offset: int = 0) -> Sequence[int]:
26
index_f = rank * per_partition_vocab_size
27
index_l = index_f + per_partition_vocab_size
25
- return index_f, index_l
28
+ return index_f + offset, index_l + offset
29
30
28
-def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
29
- world_size: int) -> Sequence[int]:
31
+def vocab_range_from_global_vocab_size(global_vocab_size: int,
32
+ rank: int,
33
+ world_size: int,
34
+ offset: int = 0) -> Sequence[int]:
35
per_partition_vocab_size = divide(global_vocab_size, world_size)
36
return vocab_range_from_per_partition_vocab_size(per_partition_vocab_size,
32
- rank)
37
+ rank,
38
+ offset=offset)
39
+
40
+
41
+@dataclass
42
+class VocabParallelEmbeddingShardIndices:
43
+ """Indices for a shard of a vocab parallel embedding."""
44
+ padded_org_vocab_start_index: int
45
+ padded_org_vocab_end_index: int
46
+ padded_added_vocab_start_index: int
47
+ padded_added_vocab_end_index: int
48
+
49
+ org_vocab_start_index: int
50
+ org_vocab_end_index: int
51
+ added_vocab_start_index: int
52
+ added_vocab_end_index: int
53
+
54
+ @property
55
+ def num_org_elements(self) -> int:
56
+ return self.org_vocab_end_index - self.org_vocab_start_index
57
+
58
+ @property
59
+ def num_added_elements(self) -> int:
60
+ return self.added_vocab_end_index - self.added_vocab_start_index
61
+
62
+ @property
63
+ def num_org_elements_padded(self) -> int:
64
+ return (self.padded_org_vocab_end_index -
65
+ self.padded_org_vocab_start_index)
66
+
67
+ @property
68
+ def num_added_elements_padded(self) -> int:
69
+ return (self.padded_added_vocab_end_index -
70
+ self.padded_added_vocab_start_index)
71
+
72
+ @property
73
+ def num_org_vocab_padding(self) -> int:
74
+ return self.num_org_elements_padded - self.num_org_elements
75
+
76
+ @property
77
+ def num_added_vocab_padding(self) -> int:
78
+ return self.num_added_elements_padded - self.num_added_elements
79
+
80
+ @property
81
+ def num_elements_padded(self) -> int:
82
+ return self.num_org_elements_padded + self.num_added_elements_padded
83
+
84
+ def __post_init__(self):
85
+ # sanity checks
86
+ assert (self.padded_org_vocab_start_index <=
87
+ self.padded_org_vocab_end_index)
88
+ assert (self.padded_added_vocab_start_index <=
89
+ self.padded_added_vocab_end_index)
90
+
91
+ assert self.org_vocab_start_index <= self.org_vocab_end_index
92
+ assert self.added_vocab_start_index <= self.added_vocab_end_index
93
+
94
+ assert self.org_vocab_start_index <= self.padded_org_vocab_start_index
95
+ assert (self.added_vocab_start_index <=
96
+ self.padded_added_vocab_start_index)
97
+ assert self.org_vocab_end_index <= self.padded_org_vocab_end_index
98
+ assert self.added_vocab_end_index <= self.padded_added_vocab_end_index
99
+
100
+ assert self.num_org_elements <= self.num_org_elements_padded
101
+ assert self.num_added_elements <= self.num_added_elements_padded
102
+
103
+
104
+@torch.jit.script
105
+def get_masked_input_and_mask(
106
+ input_: torch.Tensor, org_vocab_start_index: int,
107
+ org_vocab_end_index: int, num_org_vocab_padding: int,
108
+ added_vocab_start_index: int,
109
+ added_vocab_end_index: int) -> Tuple[torch.Tensor, torch.Tensor]:
110
+ # torch.jit.script will fuse all of the pointwise ops below
111
+ # into a single kernel, making it very fast
112
+ org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ <
113
+ org_vocab_end_index)
114
+ added_vocab_mask = (input_ >= added_vocab_start_index) & (
115
+ input_ < added_vocab_end_index)
116
+ added_offset = added_vocab_start_index - (
117
+ org_vocab_end_index - org_vocab_start_index) - num_org_vocab_padding
118
+ valid_offset = (org_vocab_start_index *
119
+ org_vocab_mask) + (added_offset * added_vocab_mask)
120
+ vocab_mask = org_vocab_mask | added_vocab_mask
121
+ input_ = vocab_mask * (input_ - valid_offset)
122
+ return input_, ~vocab_mask
123
124
125
class VocabParallelEmbedding(torch.nn.Module):
@@ -38,13 +128,36 @@ class VocabParallelEmbedding(torch.nn.Module):
128
Adapted from torch.nn.Embedding, note that we pad the vocabulary size to
129
make sure it is divisible by the number of model parallel GPUs.
130
131
+ In order to support various loading methods, we ensure that LoRA-added
132
+ embeddings are always at the end of TP-sharded tensors. In other words,
133
+ we shard base embeddings and LoRA embeddings separately (both padded),
134
+ and place them in the same tensor.
135
+ In this example, we will have the original vocab size = 1010,
136
+ added vocab size = 16 and padding to 64. Therefore, the total
137
+ vocab size with padding will be 1088 (because we first pad 1010 to
138
+ 1024, add 16, and then pad to 1088).
139
+ Therefore, the tensor format looks like the following:
140
+ TP1, rank 0 (no sharding):
141
+ |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >|
142
+ corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1015 | -1 | ... | -1 |
143
+ index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 |
144
+
145
+ TP2, rank 0:
146
+ |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >|
147
+ corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1000 | ... | 1015 | -1 | ... | -1 |
148
+ index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 520 | ... | 543 |
149
+ TP2, rank 1:
150
+ |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >|
151
+ corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 |
152
+ index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 519 | 520 | ... | 543 |
153
+
154
Args:
155
num_embeddings: vocabulary size.
156
embedding_dim: size of hidden state.
157
params_dtype: type of the parameters.
158
org_num_embeddings: original vocabulary size (without LoRA).
159
padding_size: padding size for the vocabulary.
47
- """
160
+ """ # noqa: E501
161
162
def __init__(self,
163
num_embeddings: int,
@@ -55,21 +168,39 @@ class VocabParallelEmbedding(torch.nn.Module):
168
super().__init__()
169
170
# Keep the input dimensions.
171
+ tp_rank = get_tensor_model_parallel_rank()
172
+ self.tp_size = get_tensor_model_parallel_world_size()
173
self.num_embeddings = num_embeddings
174
+ self.padding_size = padding_size
175
self.org_vocab_size = org_num_embeddings or num_embeddings
60
- self.num_embeddings_padded = pad_vocab_size(num_embeddings,
61
- padding_size)
176
+ num_added_embeddings = num_embeddings - self.org_vocab_size
177
+ self.org_vocab_size_padded = pad_vocab_size(self.org_vocab_size,
178
+ self.padding_size)
179
+ self.num_embeddings_padded = pad_vocab_size(
180
+ self.org_vocab_size_padded + num_added_embeddings,
181
+ self.padding_size)
182
+ assert self.org_vocab_size_padded <= self.num_embeddings_padded
183
+
184
+ self.shard_indices = self._get_indices(self.num_embeddings_padded,
185
+ self.org_vocab_size_padded,
186
+ self.num_embeddings,
187
+ self.org_vocab_size, tp_rank,
188
+ self.tp_size)
189
self.embedding_dim = embedding_dim
190
if params_dtype is None:
191
params_dtype = torch.get_default_dtype()
65
- self.tp_size = get_tensor_model_parallel_world_size()
192
# Divide the weight matrix along the vocaburaly dimension.
67
- self.vocab_start_index, self.vocab_end_index = (
68
- vocab_range_from_global_vocab_size(
69
- self.num_embeddings_padded, get_tensor_model_parallel_rank(),
70
- self.tp_size))
71
- self.num_embeddings_per_partition = (self.vocab_end_index -
72
- self.vocab_start_index)
193
+ self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
194
+ self.num_embeddings_per_partition = divide(self.num_embeddings_padded,
195
+ self.tp_size)
196
+ assert (self.shard_indices.num_elements_padded ==
197
+ self.num_embeddings_per_partition)
198
+ self.num_org_embeddings_per_partition = (
199
+ self.shard_indices.org_vocab_end_index -
200
+ self.shard_indices.org_vocab_start_index)
201
+ self.num_added_embeddings_per_partition = (
202
+ self.shard_indices.added_vocab_end_index -
203
+ self.shard_indices.added_vocab_start_index)
204
self.weight = Parameter(
205
torch.empty(self.num_embeddings_per_partition,
206
self.embedding_dim,
@@ -79,28 +210,107 @@ class VocabParallelEmbedding(torch.nn.Module):
210
"weight_loader": self.weight_loader
211
})
212
213
+ @classmethod
214
+ def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int,
215
+ vocab_size: int, org_vocab_size: int, tp_rank: int,
216
+ tp_size: int) -> VocabParallelEmbeddingShardIndices:
217
+ """Get start and end indices for vocab parallel embedding, following the
218
+ layout outlined in the class docstring, based on the given tp_rank and
219
+ tp_size."""
220
+ num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded
221
+ padded_org_vocab_start_index, padded_org_vocab_end_index = (
222
+ vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank,
223
+ tp_size))
224
+ padded_added_vocab_start_index, padded_added_vocab_end_index = (
225
+ vocab_range_from_global_vocab_size(num_added_embeddings_padded,
226
+ tp_rank,
227
+ tp_size,
228
+ offset=org_vocab_size))
229
+ # remove padding
230
+ org_vocab_start_index = min(padded_org_vocab_start_index,
231
+ org_vocab_size)
232
+ org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size)
233
+ added_vocab_start_index = min(padded_added_vocab_start_index,
234
+ vocab_size)
235
+ added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size)
236
+ return VocabParallelEmbeddingShardIndices(
237
+ padded_org_vocab_start_index, padded_org_vocab_end_index,
238
+ padded_added_vocab_start_index, padded_added_vocab_end_index,
239
+ org_vocab_start_index, org_vocab_end_index,
240
+ added_vocab_start_index, added_vocab_end_index)
241
+
242
+ def get_sharded_to_full_mapping(self) -> Optional[List[int]]:
243
+ """Get a mapping that can be used to reindex the gathered
244
+ logits for sampling.
245
+
246
+ During sampling, we gather logits from all ranks. The relationship
247
+ of index->token_id will follow the same format as outlined in the class
248
+ docstring. However, after the gather, we want to reindex the final
249
+ logits tensor to map index->token_id one-to-one (the index is always
250
+ equal the token_id it corresponds to). The indices returned by this
251
+ method allow us to do that.
252
+ """
253
+ if self.tp_size < 2:
254
+ return None
255
+
256
+ base_embeddings: List[int] = []
257
+ added_embeddings: List[int] = []
258
+ padding: List[int] = []
259
+ for tp_rank in range(self.tp_size):
260
+ shard_indices = self._get_indices(self.num_embeddings_padded,
261
+ self.org_vocab_size_padded,
262
+ self.num_embeddings,
263
+ self.org_vocab_size, tp_rank,
264
+ self.tp_size)
265
+ range_start = self.num_embeddings_per_partition * tp_rank
266
+ range_end = self.num_embeddings_per_partition * (tp_rank + 1)
267
+ base_embeddings.extend(
268
+ range(range_start,
269
+ range_start + shard_indices.num_org_elements))
270
+ padding.extend(
271
+ range(range_start + shard_indices.num_org_elements,
272
+ range_start + shard_indices.num_org_elements_padded))
273
+ added_embeddings.extend(
274
+ range(
275
+ range_start + shard_indices.num_org_elements_padded,
276
+ range_start + shard_indices.num_org_elements_padded +
277
+ shard_indices.num_added_elements))
278
+ padding.extend(
279
+ range(
280
+ range_start + shard_indices.num_org_elements_padded +
281
+ shard_indices.num_added_elements,
282
+ range_start + shard_indices.num_org_elements_padded +
283
+ shard_indices.num_added_elements_padded))
284
+ assert (range_start + shard_indices.num_org_elements_padded +
285
+ shard_indices.num_added_elements_padded == range_end)
286
+ ret = base_embeddings + added_embeddings + padding
287
+ assert len(ret) == self.num_embeddings_padded
288
+ return ret
289
+
290
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
291
parallel_dim = param.parallel_dim
292
assert loaded_weight.shape[parallel_dim] == self.org_vocab_size
85
- loaded_weight = loaded_weight[self.vocab_start_index:self.
86
- vocab_end_index]
293
+ loaded_weight = loaded_weight[self.shard_indices.org_vocab_start_index:
294
+ self.shard_indices.org_vocab_end_index]
295
param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
296
+ param[loaded_weight.shape[0]:].data.fill_(0)
297
298
def forward(self, input_):
299
if self.tp_size > 1:
300
# Build the mask.
92
- input_mask = ((input_ < self.vocab_start_index) |
93
- (input_ >= self.vocab_end_index))
94
- # Mask the input.
95
- masked_input = input_.clone() - self.vocab_start_index
96
- masked_input[input_mask] = 0
301
+ masked_input, input_mask = get_masked_input_and_mask(
302
+ input_, self.shard_indices.org_vocab_start_index,
303
+ self.shard_indices.org_vocab_end_index,
304
+ self.shard_indices.num_org_vocab_padding,
305
+ self.shard_indices.added_vocab_start_index,
306
+ self.shard_indices.added_vocab_end_index)
307
else:
308
masked_input = input_
309
# Get the embeddings.
310
output_parallel = F.embedding(masked_input, self.weight)
311
# Mask the output embedding.
312
if self.tp_size > 1:
103
- output_parallel[input_mask, :] = 0.0
313
+ output_parallel.masked_fill_(input_mask.unsqueeze(1), 0)
314
# Reduce across all the model parallel GPUs.
315
output = tensor_model_parallel_all_reduce(output_parallel)
316
return output
vllm/worker/model_runner.py
+11
-8
@@ -35,6 +35,7 @@ _BATCH_SIZE_ALIGNMENT = 8
35
_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
36
_BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
37
]
38
+_NUM_WARMUP_ITERS = 2
39
40
41
class ModelInput(NamedTuple):
@@ -975,16 +976,18 @@ class CUDAGraphRunner:
976
**kwargs,
977
) -> None:
978
assert self._graph is None
978
- # Run the model once without capturing the graph.
979
+ # Run the model a few times without capturing the graph.
980
# This is to make sure that the captured graph does not include the
981
# kernel launches for initial benchmarking (e.g., Triton autotune).
981
- self.model(
982
- input_ids,
983
- positions,
984
- kv_caches,
985
- attn_metadata,
986
- **kwargs,
987
- )
982
+ # Note one iteration is not enough for torch.jit.script
983
+ for _ in range(_NUM_WARMUP_ITERS):
984
+ self.model(
985
+ input_ids,
986
+ positions,
987
+ kv_caches,
988
+ attn_metadata,
989
+ **kwargs,
990
+ )
991
torch.cuda.synchronize()
992
993
# Capture the graph.