@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ccdc490dda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Change LoRA embedding sharding to support loading methods (#5038)

Antoni Baum committed Jun 6, 2024 at 19:07 UTC ccdc490dda3f534c63c1faf29a638e65594d0dc3
11 files changed +661 -129
.buildkite/test-pipeline.yaml
+2 -8
@@ -46,6 +46,7 @@ steps:
46 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
47 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
48 - pytest -v -s spec_decode/e2e/test_integration_dist.py
49 + - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
50
51 - label: Distributed Tests (Multiple Groups)
52 #mirror_hardwares: [amd]
@@ -138,14 +139,7 @@ steps:
139 num_gpus: 4
140 # This test runs llama 13B, so it is required to run on 4 GPUs.
141 commands:
141 - # Temporarily run this way because we cannot clean up GPU mem usage
142 - # for multi GPU tests.
143 - # TODO(sang): Fix it.
144 - - pytest -v -s lora/test_long_context.py::test_rotary_emb_replaced
145 - - pytest -v -s lora/test_long_context.py::test_batched_rope_kernel
146 - - pytest -v -s lora/test_long_context.py::test_self_consistency
147 - - pytest -v -s lora/test_long_context.py::test_quality
148 - - pytest -v -s lora/test_long_context.py::test_max_len
142 + - pytest -v -s -x lora/test_long_context.py
143
144 - label: Tensorizer Test
145 #mirror_hardwares: [amd]
tests/conftest.py
+21
@@ -1,6 +1,8 @@
1 import contextlib
2 import gc
3 import os
4 +import subprocess
5 +import sys
6 from typing import Any, Dict, List, Optional, Tuple, TypeVar
7
8 import pytest
@@ -522,3 +524,22 @@ def caplog_vllm(temporary_enable_log_propagate, caplog):
524 # To capture vllm log, we should enable propagate=True temporarily
525 # because caplog depends on logs propagated to the root logger.
526 yield caplog
527 +
528 +
529 +@pytest.fixture(scope="session")
530 +def num_gpus_available():
531 + """Get number of GPUs without initializing the CUDA context
532 + in current process."""
533 +
534 + try:
535 + out = subprocess.run([
536 + sys.executable, "-c",
537 + "import torch; print(torch.cuda.device_count())"
538 + ],
539 + capture_output=True,
540 + check=True,
541 + text=True)
542 + except subprocess.CalledProcessError as e:
543 + logger.warning("Failed to get number of GPUs.", exc_info=e)
544 + return 0
545 + return int(out.stdout.strip())
tests/lora/conftest.py
+16 -2
@@ -42,10 +42,24 @@ def cleanup():
42 ray.shutdown()
43
44
45 +@pytest.fixture()
46 +def should_do_global_cleanup_after_test(request) -> bool:
47 + """Allow subdirectories to skip global cleanup by overriding this fixture.
48 + This can provide a ~10x speedup for non-GPU unit tests since they don't need
49 + to initialize torch.
50 + """
51 +
52 + if request.node.get_closest_marker("skip_global_cleanup"):
53 + return False
54 +
55 + return True
56 +
57 +
58 @pytest.fixture(autouse=True)
46 -def cleanup_fixture():
59 +def cleanup_fixture(should_do_global_cleanup_after_test: bool):
60 yield
48 - cleanup()
61 + if should_do_global_cleanup_after_test:
62 + cleanup()
63
64
65 @pytest.fixture
tests/lora/test_layers.py
+217 -2
@@ -2,6 +2,7 @@ import random
2 from copy import deepcopy
3 from dataclasses import dataclass
4 from typing import Dict, List, Optional, Tuple
5 +from unittest.mock import patch
6
7 import pytest
8 import torch
@@ -32,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 - ParallelLMHead, VocabParallelEmbedding)
36 + ParallelLMHead, VocabParallelEmbedding, get_masked_input_and_mask)
37 from vllm.model_executor.utils import set_random_seed
38
39 from .utils import DummyLoRAManager
@@ -427,7 +428,8 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
428 logits_processor = LogitsProcessor(
429 vocab_size + lora_config.lora_extra_vocab_size, vocab_size)
430 lora_logits_processor = LogitsProcessorWithLoRA(
430 - logits_processor, 1024, linear.weight.dtype, linear.weight.device)
431 + logits_processor, 1024, linear.weight.dtype, linear.weight.device,
432 + None)
433 lora_logits_processor.create_lora_weights(max_loras, lora_config)
434
435 return linear, logits_processor, lora_logits_processor
@@ -867,3 +869,216 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
869
870 torch.allclose(ref_q, actual_q)
871 torch.allclose(ref_k, actual_k)
872 +
873 +
874 +@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
875 +@pytest.mark.parametrize("seed", list(range(256)))
876 +def test_vocab_parallel_embedding_indices(tp_size, seed):
877 + random.seed(seed)
878 + vocab_size = random.randint(4000, 64000)
879 + added_vocab_size = random.randint(0, 1024)
880 + org_vocab_size = vocab_size - added_vocab_size
881 + last_org_vocab_end_index = 0
882 + last_added_vocab_end_index = org_vocab_size
883 + computed_vocab_size = 0
884 + computed_org_vocab_size = 0
885 + computed_added_vocab_size = 0
886 + vocab_size_padded = -1
887 +
888 + all_org_tokens = []
889 + all_added_tokens = []
890 + token_ids = []
891 +
892 + for tp_rank in range(tp_size):
893 + with patch(
894 + "vllm.model_executor.layers.vocab_parallel_embedding.get_tensor_model_parallel_rank",
895 + return_value=tp_rank
896 + ), patch(
897 + "vllm.model_executor.layers.vocab_parallel_embedding.get_tensor_model_parallel_world_size",
898 + return_value=tp_size):
899 + vocab_embedding = VocabParallelEmbedding(
900 + vocab_size, 1, org_num_embeddings=org_vocab_size)
901 + vocab_size_padded = vocab_embedding.num_embeddings_padded
902 + shard_indices = vocab_embedding.shard_indices
903 + # Assert that the ranges are contiguous
904 + assert shard_indices.org_vocab_start_index == last_org_vocab_end_index
905 + assert (shard_indices.added_vocab_start_index ==
906 + last_added_vocab_end_index)
907 +
908 + # Ensure that we are not exceeding the vocab size
909 + computed_vocab_size += shard_indices.num_elements_padded
910 + computed_org_vocab_size += shard_indices.num_org_elements
911 + computed_added_vocab_size += shard_indices.num_added_elements
912 +
913 + # Ensure that the ranges are not overlapping
914 + all_org_tokens.extend(
915 + range(shard_indices.org_vocab_start_index,
916 + shard_indices.org_vocab_end_index))
917 + all_added_tokens.extend(
918 + range(shard_indices.added_vocab_start_index,
919 + shard_indices.added_vocab_end_index))
920 +
921 + token_ids.extend(
922 + range(shard_indices.org_vocab_start_index,
923 + shard_indices.org_vocab_end_index))
924 + token_ids.extend([-1] * (shard_indices.num_org_elements_padded -
925 + shard_indices.num_org_elements))
926 + token_ids.extend(
927 + range(shard_indices.added_vocab_start_index,
928 + shard_indices.added_vocab_end_index))
929 + token_ids.extend([-1] * (shard_indices.num_added_elements_padded -
930 + shard_indices.num_added_elements))
931 +
932 + last_org_vocab_end_index = shard_indices.org_vocab_end_index
933 + last_added_vocab_end_index = shard_indices.added_vocab_end_index
934 +
935 + assert computed_vocab_size == vocab_size_padded
936 + assert computed_org_vocab_size == org_vocab_size
937 + assert computed_added_vocab_size == added_vocab_size
938 +
939 + # Ensure that the ranges are not overlapping
940 + assert len(all_org_tokens) == len(set(all_org_tokens))
941 + assert len(all_added_tokens) == len(set(all_added_tokens))
942 + assert not set(all_org_tokens).intersection(set(all_added_tokens))
943 +
944 + token_ids_tensor = torch.tensor(token_ids, dtype=torch.long)
945 + reindex_mapping = vocab_embedding.get_sharded_to_full_mapping()
946 + assert reindex_mapping is not None or tp_size == 1
947 + if reindex_mapping is not None:
948 + reindexed_token_ids = token_ids_tensor[reindex_mapping]
949 + expected = torch.tensor(list(range(0, vocab_size)))
950 + assert reindexed_token_ids[:vocab_size].equal(expected)
951 + assert torch.all(reindexed_token_ids[vocab_size:] == -1)
952 +
953 +
954 +def test_get_masked_input_and_mask():
955 + x = torch.tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
956 +
957 + # base tp 1 case, no padding
958 + modified_x, _ = get_masked_input_and_mask(x,
959 + org_vocab_start_index=0,
960 + org_vocab_end_index=8,
961 + added_vocab_start_index=8,
962 + added_vocab_end_index=12,
963 + num_org_vocab_padding=0)
964 + assert torch.equal(x, modified_x)
965 +
966 + # tp 2 case, no padding
967 + modified_x_rank_0, _ = get_masked_input_and_mask(x,
968 + org_vocab_start_index=0,
969 + org_vocab_end_index=4,
970 + added_vocab_start_index=8,
971 + added_vocab_end_index=10,
972 + num_org_vocab_padding=0)
973 + modified_x_rank_1, _ = get_masked_input_and_mask(
974 + x,
975 + org_vocab_start_index=4,
976 + org_vocab_end_index=8,
977 + added_vocab_start_index=10,
978 + added_vocab_end_index=12,
979 + num_org_vocab_padding=0)
980 + assert torch.equal(modified_x_rank_0,
981 + torch.tensor([0, 1, 2, 3, 0, 0, 0, 0, 4, 5, 0, 0]))
982 + assert torch.equal(modified_x_rank_1,
983 + torch.tensor([0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 4, 5]))
984 +
985 + # tp 4 case, no padding
986 + modified_x_rank_0, _ = get_masked_input_and_mask(x,
987 + org_vocab_start_index=0,
988 + org_vocab_end_index=2,
989 + added_vocab_start_index=8,
990 + added_vocab_end_index=9,
991 + num_org_vocab_padding=0)
992 + modified_x_rank_1, _ = get_masked_input_and_mask(x,
993 + org_vocab_start_index=2,
994 + org_vocab_end_index=4,
995 + added_vocab_start_index=9,
996 + added_vocab_end_index=10,
997 + num_org_vocab_padding=0)
998 + modified_x_rank_2, _ = get_masked_input_and_mask(
999 + x,
1000 + org_vocab_start_index=4,
1001 + org_vocab_end_index=6,
1002 + added_vocab_start_index=10,
1003 + added_vocab_end_index=11,
1004 + num_org_vocab_padding=0)
1005 + modified_x_rank_3, _ = get_masked_input_and_mask(
1006 + x,
1007 + org_vocab_start_index=6,
1008 + org_vocab_end_index=8,
1009 + added_vocab_start_index=11,
1010 + added_vocab_end_index=12,
1011 + num_org_vocab_padding=0)
1012 + assert torch.equal(modified_x_rank_0,
1013 + torch.tensor([0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0]))
1014 + assert torch.equal(modified_x_rank_1,
1015 + torch.tensor([0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0]))
1016 + assert torch.equal(modified_x_rank_2,
1017 + torch.tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0]))
1018 + assert torch.equal(modified_x_rank_3,
1019 + torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2]))
1020 +
1021 + # base tp 1 case, with padding
1022 + modified_x, _ = get_masked_input_and_mask(x,
1023 + org_vocab_start_index=0,
1024 + org_vocab_end_index=8,
1025 + added_vocab_start_index=8,
1026 + added_vocab_end_index=12,
1027 + num_org_vocab_padding=2)
1028 + assert torch.equal(modified_x,
1029 + torch.tensor([0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13]))
1030 +
1031 + # tp 2 case, with padding
1032 + modified_x_rank_0, _ = get_masked_input_and_mask(x,
1033 + org_vocab_start_index=0,
1034 + org_vocab_end_index=4,
1035 + added_vocab_start_index=8,
1036 + added_vocab_end_index=10,
1037 + num_org_vocab_padding=2)
1038 + modified_x_rank_1, _ = get_masked_input_and_mask(
1039 + x,
1040 + org_vocab_start_index=4,
1041 + org_vocab_end_index=8,
1042 + added_vocab_start_index=10,
1043 + added_vocab_end_index=12,
1044 + num_org_vocab_padding=2)
1045 + assert torch.equal(modified_x_rank_0,
1046 + torch.tensor([0, 1, 2, 3, 0, 0, 0, 0, 6, 7, 0, 0]))
1047 + assert torch.equal(modified_x_rank_1,
1048 + torch.tensor([0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 6, 7]))
1049 +
1050 + # tp 4 case, with padding
1051 + modified_x_rank_0, _ = get_masked_input_and_mask(x,
1052 + org_vocab_start_index=0,
1053 + org_vocab_end_index=2,
1054 + added_vocab_start_index=8,
1055 + added_vocab_end_index=9,
1056 + num_org_vocab_padding=2)
1057 + modified_x_rank_1, _ = get_masked_input_and_mask(x,
1058 + org_vocab_start_index=2,
1059 + org_vocab_end_index=4,
1060 + added_vocab_start_index=9,
1061 + added_vocab_end_index=10,
1062 + num_org_vocab_padding=2)
1063 + modified_x_rank_2, _ = get_masked_input_and_mask(
1064 + x,
1065 + org_vocab_start_index=4,
1066 + org_vocab_end_index=6,
1067 + added_vocab_start_index=10,
1068 + added_vocab_end_index=11,
1069 + num_org_vocab_padding=2)
1070 + modified_x_rank_3, _ = get_masked_input_and_mask(
1071 + x,
1072 + org_vocab_start_index=6,
1073 + org_vocab_end_index=8,
1074 + added_vocab_start_index=11,
1075 + added_vocab_end_index=12,
1076 + num_org_vocab_padding=2)
1077 + assert torch.equal(modified_x_rank_0,
1078 + torch.tensor([0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0]))
1079 + assert torch.equal(modified_x_rank_1,
1080 + torch.tensor([0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0]))
1081 + assert torch.equal(modified_x_rank_2,
1082 + torch.tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0]))
1083 + assert torch.equal(modified_x_rank_3,
1084 + torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4]))
tests/lora/test_llama.py
+7 -10
@@ -36,11 +36,10 @@ def do_sample(llm, lora_path: str, lora_id: int):
36 return generated_texts
37
38
39 -@pytest.mark.parametrize("tp_size", [1])
40 -def test_llama_lora(sql_lora_files, tp_size):
41 - # Cannot use as it will initialize torch.cuda too early...
42 - # if torch.cuda.device_count() < tp_size:
43 - # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
39 +@pytest.mark.parametrize("tp_size", [1, 2, 4])
40 +def test_llama_lora(sql_lora_files, tp_size, num_gpus_available):
41 + if num_gpus_available < tp_size:
42 + pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
43
44 llm = vllm.LLM(MODEL_PATH,
45 enable_lora=True,
@@ -80,11 +79,9 @@ def test_llama_lora(sql_lora_files, tp_size):
79 print("removing lora")
80
81
83 -@pytest.mark.skip("Requires multiple GPUs")
84 -def test_llama_tensor_parallel_equality(sql_lora_files):
85 - # Cannot use as it will initialize torch.cuda too early...
86 - # if torch.cuda.device_count() < 4:
87 - # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
82 +def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
83 + if num_gpus_available < 4:
84 + pytest.skip("Not enough GPUs for tensor parallelism 4")
85
86 llm_tp1 = vllm.LLM(MODEL_PATH,
87 enable_lora=True,
tests/lora/test_long_context.py
+13 -10
@@ -102,22 +102,21 @@ def batched_generate(
102 return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
103
104
105 -@pytest.fixture
105 +@pytest.fixture(scope="module")
106 def lora_llm(long_context_infos):
107 scaling_factors = [
108 context_len_to_scaling_factor[info["context_length"]]
109 for info in long_context_infos.values()
110 ]
111
112 - llm = vllm.LLM(
113 - "meta-llama/Llama-2-13b-chat-hf",
114 - enable_lora=True,
115 - max_num_seqs=16,
116 - max_loras=2,
117 - long_lora_scaling_factors=tuple(scaling_factors),
118 - max_num_batched_tokens=4096 * 8,
119 - tensor_parallel_size=4,
120 - )
112 + llm = vllm.LLM("meta-llama/Llama-2-13b-chat-hf",
113 + enable_lora=True,
114 + max_num_seqs=16,
115 + max_loras=2,
116 + long_lora_scaling_factors=tuple(scaling_factors),
117 + max_num_batched_tokens=4096 * 8,
118 + tensor_parallel_size=4,
119 + distributed_executor_backend="mp")
120 yield llm
121 del llm
122
@@ -154,6 +153,7 @@ def test_rotary_emb_replaced(dist_init):
153 assert rotary_emb_count == 32
154
155
156 +@pytest.mark.skip_global_cleanup
157 def test_batched_rope_kernel(lora_llm, long_context_infos):
158 """We test the batched kernel by comparing the results of batched an
159 non-batched generation.
@@ -188,6 +188,7 @@ def test_batched_rope_kernel(lora_llm, long_context_infos):
188 f"same:\n{batched}\n{non_batched}")
189
190
191 +@pytest.mark.skip_global_cleanup
192 def test_self_consistency(lora_llm, long_context_infos):
193 """We test consistency of the batched kernel by permuting batched
194 inputs and comparing the results to the non-permuted batched results.
@@ -227,6 +228,7 @@ def test_self_consistency(lora_llm, long_context_infos):
228 f"\n{permutated_batched_results[permutation[i]]}")
229
230
231 +@pytest.mark.skip_global_cleanup
232 def test_quality(lora_llm, long_context_infos):
233 """We test the quality of the answers given by the LoRA model by
234 comparing the generated text to the merged model's outputs.
@@ -257,6 +259,7 @@ def test_quality(lora_llm, long_context_infos):
259 assert np.mean(scores) > 0.5
260
261
262 +@pytest.mark.skip_global_cleanup
263 def test_max_len(lora_llm, long_context_infos):
264 """Test that we raise an ValueError when the input of a given LoRA
265 model exceeds the maximum length."""
tests/test_sharded_state_loader.py
+80 -44
@@ -1,3 +1,4 @@
1 +import multiprocessing as mp
2 import os
3 import shutil
4 from tempfile import TemporaryDirectory
@@ -18,9 +19,7 @@ prompts = [
19
20 # Create a sampling params object.
21 sampling_params = SamplingParams(
21 - temperature=0.8,
22 - top_p=0.95,
23 - seed=0,
22 + temperature=0,
23 max_tokens=256,
24 ignore_eos=True,
25 )
@@ -43,48 +42,85 @@ def test_filter_subtensors():
42 assert tensor.equal(state_dict[key])
43
44
45 +@pytest.fixture(scope="module")
46 +def llama_2_7b_files():
47 + with TemporaryDirectory() as cache_dir:
48 + input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
49 + cache_dir=cache_dir,
50 + ignore_patterns="*.bin*")
51 + yield input_dir
52 +
53 +
54 +def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
55 + llm_sharded_writer = LLM(model=input_dir, **kwargs)
56 +
57 + # Dump worker states to output directory
58 + llm_sharded_writer.llm_engine.model_executor.save_sharded_state(
59 + path=output_dir)
60 + # Copy metadata files to output directory
61 + for file in os.listdir(input_dir):
62 + if not any(file.endswith(ext) for ext in weights_patterns):
63 + shutil.copy(f"{input_dir}/{file}", output_dir)
64 +
65 +
66 +def _run_generate(input_dir, queue: mp.Queue, **kwargs):
67 + llm = LLM(model=input_dir, **kwargs)
68 + gen = llm.generate(prompts, sampling_params)
69 + queue.put([g.outputs[0].__dict__ for g in gen])
70 + queue.close()
71 + queue.join_thread()
72 +
73 +
74 @pytest.mark.parametrize("enable_lora", [False, True])
47 -def test_sharded_state_loader(enable_lora):
48 - weights_patterns = ("*.bin", "*.pt", "*.safetensors")
75 +@pytest.mark.parametrize("tp_size", [1, 2])
76 +def test_sharded_state_loader(enable_lora, tp_size, num_gpus_available,
77 + llama_2_7b_files):
78 + if num_gpus_available < tp_size:
79 + pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
80
50 - with TemporaryDirectory() as cache_dir, TemporaryDirectory() as output_dir:
51 - input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
52 - cache_dir=cache_dir)
53 -
54 - llm = LLM(
55 - model=input_dir,
56 - worker_use_ray=True,
57 - gpu_memory_utilization=0.3,
58 - )
59 -
60 - # Dump worker states to output directory
61 - model_executor = llm.llm_engine.model_executor
62 - model_executor.save_sharded_state(path=output_dir)
63 - # Copy metadata files to output directory
64 - for file in os.listdir(input_dir):
65 - if not any(file.endswith(ext) for ext in weights_patterns):
66 - shutil.copy(f"{input_dir}/{file}", output_dir)
67 - del llm.llm_engine.model_executor
68 -
69 - llm_before = LLM(
70 - model=input_dir,
71 - worker_use_ray=True,
72 - enable_lora=enable_lora,
73 - gpu_memory_utilization=0.3,
74 - )
75 - gen_before = llm_before.generate(prompts, sampling_params)
76 - out_before = [gen.outputs[0].__dict__ for gen in gen_before]
77 - del llm_before.llm_engine.model_executor
78 -
79 - llm_after = LLM(
80 - model=output_dir,
81 - worker_use_ray=True,
82 - enable_lora=enable_lora,
83 - gpu_memory_utilization=0.3,
84 - load_format="sharded_state",
85 - )
86 - gen_after = llm_after.generate(prompts, sampling_params)
87 - out_after = [gen.outputs[0].__dict__ for gen in gen_after]
88 - del llm_after.llm_engine.model_executor
81 + weights_patterns = ("*.safetensors", )
82 + gpu_memory_utilization = 0.8
83 + input_dir = llama_2_7b_files
84 + ctx = mp.get_context("spawn")
85 +
86 + # Run in separate processes for memory & CUDA isolation
87 + with TemporaryDirectory() as output_dir:
88 + p = ctx.Process(target=_run_writer,
89 + args=(input_dir, output_dir, weights_patterns),
90 + kwargs=dict(
91 + tensor_parallel_size=tp_size,
92 + distributed_executor_backend="mp",
93 + gpu_memory_utilization=gpu_memory_utilization,
94 + enforce_eager=True,
95 + ))
96 + p.start()
97 + p.join()
98 +
99 + queue = ctx.Queue()
100 +
101 + p = ctx.Process(target=_run_generate,
102 + args=(input_dir, queue),
103 + kwargs=dict(
104 + distributed_executor_backend="mp",
105 + enable_lora=enable_lora,
106 + gpu_memory_utilization=gpu_memory_utilization,
107 + tensor_parallel_size=tp_size,
108 + ))
109 + p.start()
110 + p.join()
111 + out_before = queue.get()
112 +
113 + p = ctx.Process(target=_run_generate,
114 + args=(output_dir, queue),
115 + kwargs=dict(
116 + distributed_executor_backend="mp",
117 + enable_lora=enable_lora,
118 + gpu_memory_utilization=gpu_memory_utilization,
119 + tensor_parallel_size=tp_size,
120 + load_format="sharded_state",
121 + ))
122 + p.start()
123 + p.join()
124 + out_after = queue.get()
125
126 assert out_before == out_after
vllm/lora/layers.py
+57 -19
@@ -215,19 +215,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
215 lora_config: LoRAConfig,
216 model_config: Optional[PretrainedConfig] = None) -> None:
217
218 - lora_vocab_start_idx = self.base_layer.org_vocab_size
219 - weights_idx = None
220 - if self.base_layer.vocab_end_index > lora_vocab_start_idx:
218 + if self.base_layer.num_added_embeddings_per_partition > 0:
219 # We can start adding lora weights
222 - weights_idx = max(
223 - lora_vocab_start_idx - self.base_layer.vocab_start_index, 0)
224 - self.embeddings_slice = (self.base_layer.vocab_start_index -
225 - self.base_layer.org_vocab_size +
226 - weights_idx,
227 - self.base_layer.vocab_end_index -
228 - self.base_layer.org_vocab_size)
229 - self.embeddings_weights = self.base_layer.weight.data[weights_idx:]
230 - self.embeddings_weights.fill_(0)
220 + self.embeddings_weights = self.base_layer.weight.data[
221 + self.base_layer.num_org_embeddings_per_partition:self.
222 + base_layer.num_org_embeddings_per_partition +
223 + self.base_layer.num_added_embeddings_per_partition]
224 + self.embeddings_slice = (
225 + self.base_layer.shard_indices.added_vocab_start_index -
226 + self.base_layer.org_vocab_size,
227 + self.base_layer.shard_indices.added_vocab_end_index -
228 + self.base_layer.org_vocab_size)
229 + self.base_layer.weight.data[
230 + self.base_layer.num_org_embeddings_per_partition:].fill_(0)
231 else:
232 self.embeddings_slice = None
233 self.embeddings_weights = None
@@ -1025,19 +1025,31 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1025
1026
1027 class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1028 + """
1029 + LoRA wrapper for LogitsProcessor, with extra logic to handle the
1030 + application of the LoRA adapter and added LoRA vocabulary.
1031 +
1032 + Args:
1033 + base_layer: LogitsProcessor layer
1034 + hidden_size: hidden size of the model
1035 + dtype: data type of the model
1036 + device: device of the model
1037 + sharded_to_full_mapping: index mapping from sharded vocab to full vocab
1038 + received from base_layer.get_sharded_to_full_mapping(). If None,
1039 + no reindexing will be done.
1040 + """
1041
1029 - def __init__(
1030 - self,
1031 - base_layer: LogitsProcessor,
1032 - hidden_size: int,
1033 - dtype: torch.dtype,
1034 - device: torch.device,
1035 - ) -> None:
1042 + def __init__(self, base_layer: LogitsProcessor, hidden_size: int,
1043 + dtype: torch.dtype, device: torch.device,
1044 + sharded_to_full_mapping: Optional[List[int]]) -> None:
1045 super().__init__()
1046 self.base_layer = base_layer
1047 self.hidden_size = hidden_size
1048 self.dtype = dtype
1049 self.device = device
1050 + self.tp_size = get_tensor_model_parallel_world_size()
1051 + self.tp_rank = get_tensor_model_parallel_rank()
1052 + self.sharded_to_full_mapping = sharded_to_full_mapping
1053
1054 @property
1055 def logits_as_input(self):
@@ -1098,6 +1110,13 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1110 dtype=self.dtype,
1111 device=self.device,
1112 )
1113 + if self.sharded_to_full_mapping is not None:
1114 + self.sharded_to_full_mapping_gpu = torch.tensor(
1115 + self.sharded_to_full_mapping,
1116 + device=self.device,
1117 + dtype=torch.long)
1118 + else:
1119 + self.sharded_to_full_mapping_gpu = None
1120 # Lazily initialized.
1121 self.indices: torch.Tensor
1122 self.indices_len: List[int]
@@ -1154,6 +1173,25 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1173 if logits is None:
1174 return None
1175
1176 + if self.sharded_to_full_mapping_gpu is not None:
1177 + # Reindex full logits tensor to ensure 1:1 mapping between
1178 + # index and token_id
1179 + # Example for:
1180 + # org_vocab_size = 4
1181 + # added_vocab_size = 2
1182 + # pad_to_size = 8
1183 + # tp_size = 2
1184 +
1185 + # indices: [0, 1, 2, 3, 4, 5, 6, 7]
1186 + # token_id: [0, 1, 4, -1, 2, 3, 5, -1]
1187 +
1188 + # Therefore, the mapping is expected to be:
1189 + # [0, 1, 4, 6, 2, 3, 5, 7] so that when we reindex,
1190 + # we get:
1191 + # indices: [0, 1, 2, 3, 4, 5, 6, 7]
1192 + # token_id: [0, 1, 2, 3, 4, 5, -1, -1]
1193 + logits = logits[:, self.sharded_to_full_mapping_gpu]
1194 +
1195 lora_logits = torch.empty(
1196 self.embeddings_tensors.shape[0] + 1,
1197 self.embeddings_tensors.shape[1],
vllm/lora/utils.py
+2 -1
@@ -67,7 +67,8 @@ def from_layer_logits_processor(
67 model_config: Optional[PretrainedConfig] = None,
68 ) -> LogitsProcessorWithLoRA:
69 ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
70 - lm_head.weight.dtype, lm_head.weight.device)
70 + lm_head.weight.dtype, lm_head.weight.device,
71 + lm_head.get_sharded_to_full_mapping())
72 ret.create_lora_weights(max_loras, lora_config, model_config)
73 return ret
74
vllm/model_executor/layers/vocab_parallel_embedding.py
+235 -25
@@ -1,4 +1,5 @@
1 -from typing import Optional, Sequence
1 +from dataclasses import dataclass
2 +from typing import List, Optional, Sequence, Tuple
3
4 import torch
5 import torch.nn.functional as F
@@ -18,18 +19,107 @@ def pad_vocab_size(vocab_size: int,
19 return ((vocab_size + pad_to - 1) // pad_to) * pad_to
20
21
21 -def vocab_range_from_per_partition_vocab_size(per_partition_vocab_size: int,
22 - rank: int) -> Sequence[int]:
22 +def vocab_range_from_per_partition_vocab_size(
23 + per_partition_vocab_size: int,
24 + rank: int,
25 + offset: int = 0) -> Sequence[int]:
26 index_f = rank * per_partition_vocab_size
27 index_l = index_f + per_partition_vocab_size
25 - return index_f, index_l
28 + return index_f + offset, index_l + offset
29
30
28 -def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
29 - world_size: int) -> Sequence[int]:
31 +def vocab_range_from_global_vocab_size(global_vocab_size: int,
32 + rank: int,
33 + world_size: int,
34 + offset: int = 0) -> Sequence[int]:
35 per_partition_vocab_size = divide(global_vocab_size, world_size)
36 return vocab_range_from_per_partition_vocab_size(per_partition_vocab_size,
32 - rank)
37 + rank,
38 + offset=offset)
39 +
40 +
41 +@dataclass
42 +class VocabParallelEmbeddingShardIndices:
43 + """Indices for a shard of a vocab parallel embedding."""
44 + padded_org_vocab_start_index: int
45 + padded_org_vocab_end_index: int
46 + padded_added_vocab_start_index: int
47 + padded_added_vocab_end_index: int
48 +
49 + org_vocab_start_index: int
50 + org_vocab_end_index: int
51 + added_vocab_start_index: int
52 + added_vocab_end_index: int
53 +
54 + @property
55 + def num_org_elements(self) -> int:
56 + return self.org_vocab_end_index - self.org_vocab_start_index
57 +
58 + @property
59 + def num_added_elements(self) -> int:
60 + return self.added_vocab_end_index - self.added_vocab_start_index
61 +
62 + @property
63 + def num_org_elements_padded(self) -> int:
64 + return (self.padded_org_vocab_end_index -
65 + self.padded_org_vocab_start_index)
66 +
67 + @property
68 + def num_added_elements_padded(self) -> int:
69 + return (self.padded_added_vocab_end_index -
70 + self.padded_added_vocab_start_index)
71 +
72 + @property
73 + def num_org_vocab_padding(self) -> int:
74 + return self.num_org_elements_padded - self.num_org_elements
75 +
76 + @property
77 + def num_added_vocab_padding(self) -> int:
78 + return self.num_added_elements_padded - self.num_added_elements
79 +
80 + @property
81 + def num_elements_padded(self) -> int:
82 + return self.num_org_elements_padded + self.num_added_elements_padded
83 +
84 + def __post_init__(self):
85 + # sanity checks
86 + assert (self.padded_org_vocab_start_index <=
87 + self.padded_org_vocab_end_index)
88 + assert (self.padded_added_vocab_start_index <=
89 + self.padded_added_vocab_end_index)
90 +
91 + assert self.org_vocab_start_index <= self.org_vocab_end_index
92 + assert self.added_vocab_start_index <= self.added_vocab_end_index
93 +
94 + assert self.org_vocab_start_index <= self.padded_org_vocab_start_index
95 + assert (self.added_vocab_start_index <=
96 + self.padded_added_vocab_start_index)
97 + assert self.org_vocab_end_index <= self.padded_org_vocab_end_index
98 + assert self.added_vocab_end_index <= self.padded_added_vocab_end_index
99 +
100 + assert self.num_org_elements <= self.num_org_elements_padded
101 + assert self.num_added_elements <= self.num_added_elements_padded
102 +
103 +
104 +@torch.jit.script
105 +def get_masked_input_and_mask(
106 + input_: torch.Tensor, org_vocab_start_index: int,
107 + org_vocab_end_index: int, num_org_vocab_padding: int,
108 + added_vocab_start_index: int,
109 + added_vocab_end_index: int) -> Tuple[torch.Tensor, torch.Tensor]:
110 + # torch.jit.script will fuse all of the pointwise ops below
111 + # into a single kernel, making it very fast
112 + org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ <
113 + org_vocab_end_index)
114 + added_vocab_mask = (input_ >= added_vocab_start_index) & (
115 + input_ < added_vocab_end_index)
116 + added_offset = added_vocab_start_index - (
117 + org_vocab_end_index - org_vocab_start_index) - num_org_vocab_padding
118 + valid_offset = (org_vocab_start_index *
119 + org_vocab_mask) + (added_offset * added_vocab_mask)
120 + vocab_mask = org_vocab_mask | added_vocab_mask
121 + input_ = vocab_mask * (input_ - valid_offset)
122 + return input_, ~vocab_mask
123
124
125 class VocabParallelEmbedding(torch.nn.Module):
@@ -38,13 +128,36 @@ class VocabParallelEmbedding(torch.nn.Module):
128 Adapted from torch.nn.Embedding, note that we pad the vocabulary size to
129 make sure it is divisible by the number of model parallel GPUs.
130
131 + In order to support various loading methods, we ensure that LoRA-added
132 + embeddings are always at the end of TP-sharded tensors. In other words,
133 + we shard base embeddings and LoRA embeddings separately (both padded),
134 + and place them in the same tensor.
135 + In this example, we will have the original vocab size = 1010,
136 + added vocab size = 16 and padding to 64. Therefore, the total
137 + vocab size with padding will be 1088 (because we first pad 1010 to
138 + 1024, add 16, and then pad to 1088).
139 + Therefore, the tensor format looks like the following:
140 + TP1, rank 0 (no sharding):
141 + |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >|
142 + corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1015 | -1 | ... | -1 |
143 + index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 |
144 +
145 + TP2, rank 0:
146 + |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >|
147 + corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1000 | ... | 1015 | -1 | ... | -1 |
148 + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 520 | ... | 543 |
149 + TP2, rank 1:
150 + |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >|
151 + corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 |
152 + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 519 | 520 | ... | 543 |
153 +
154 Args:
155 num_embeddings: vocabulary size.
156 embedding_dim: size of hidden state.
157 params_dtype: type of the parameters.
158 org_num_embeddings: original vocabulary size (without LoRA).
159 padding_size: padding size for the vocabulary.
47 - """
160 + """ # noqa: E501
161
162 def __init__(self,
163 num_embeddings: int,
@@ -55,21 +168,39 @@ class VocabParallelEmbedding(torch.nn.Module):
168 super().__init__()
169
170 # Keep the input dimensions.
171 + tp_rank = get_tensor_model_parallel_rank()
172 + self.tp_size = get_tensor_model_parallel_world_size()
173 self.num_embeddings = num_embeddings
174 + self.padding_size = padding_size
175 self.org_vocab_size = org_num_embeddings or num_embeddings
60 - self.num_embeddings_padded = pad_vocab_size(num_embeddings,
61 - padding_size)
176 + num_added_embeddings = num_embeddings - self.org_vocab_size
177 + self.org_vocab_size_padded = pad_vocab_size(self.org_vocab_size,
178 + self.padding_size)
179 + self.num_embeddings_padded = pad_vocab_size(
180 + self.org_vocab_size_padded + num_added_embeddings,
181 + self.padding_size)
182 + assert self.org_vocab_size_padded <= self.num_embeddings_padded
183 +
184 + self.shard_indices = self._get_indices(self.num_embeddings_padded,
185 + self.org_vocab_size_padded,
186 + self.num_embeddings,
187 + self.org_vocab_size, tp_rank,
188 + self.tp_size)
189 self.embedding_dim = embedding_dim
190 if params_dtype is None:
191 params_dtype = torch.get_default_dtype()
65 - self.tp_size = get_tensor_model_parallel_world_size()
192 # Divide the weight matrix along the vocaburaly dimension.
67 - self.vocab_start_index, self.vocab_end_index = (
68 - vocab_range_from_global_vocab_size(
69 - self.num_embeddings_padded, get_tensor_model_parallel_rank(),
70 - self.tp_size))
71 - self.num_embeddings_per_partition = (self.vocab_end_index -
72 - self.vocab_start_index)
193 + self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
194 + self.num_embeddings_per_partition = divide(self.num_embeddings_padded,
195 + self.tp_size)
196 + assert (self.shard_indices.num_elements_padded ==
197 + self.num_embeddings_per_partition)
198 + self.num_org_embeddings_per_partition = (
199 + self.shard_indices.org_vocab_end_index -
200 + self.shard_indices.org_vocab_start_index)
201 + self.num_added_embeddings_per_partition = (
202 + self.shard_indices.added_vocab_end_index -
203 + self.shard_indices.added_vocab_start_index)
204 self.weight = Parameter(
205 torch.empty(self.num_embeddings_per_partition,
206 self.embedding_dim,
@@ -79,28 +210,107 @@ class VocabParallelEmbedding(torch.nn.Module):
210 "weight_loader": self.weight_loader
211 })
212
213 + @classmethod
214 + def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int,
215 + vocab_size: int, org_vocab_size: int, tp_rank: int,
216 + tp_size: int) -> VocabParallelEmbeddingShardIndices:
217 + """Get start and end indices for vocab parallel embedding, following the
218 + layout outlined in the class docstring, based on the given tp_rank and
219 + tp_size."""
220 + num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded
221 + padded_org_vocab_start_index, padded_org_vocab_end_index = (
222 + vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank,
223 + tp_size))
224 + padded_added_vocab_start_index, padded_added_vocab_end_index = (
225 + vocab_range_from_global_vocab_size(num_added_embeddings_padded,
226 + tp_rank,
227 + tp_size,
228 + offset=org_vocab_size))
229 + # remove padding
230 + org_vocab_start_index = min(padded_org_vocab_start_index,
231 + org_vocab_size)
232 + org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size)
233 + added_vocab_start_index = min(padded_added_vocab_start_index,
234 + vocab_size)
235 + added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size)
236 + return VocabParallelEmbeddingShardIndices(
237 + padded_org_vocab_start_index, padded_org_vocab_end_index,
238 + padded_added_vocab_start_index, padded_added_vocab_end_index,
239 + org_vocab_start_index, org_vocab_end_index,
240 + added_vocab_start_index, added_vocab_end_index)
241 +
242 + def get_sharded_to_full_mapping(self) -> Optional[List[int]]:
243 + """Get a mapping that can be used to reindex the gathered
244 + logits for sampling.
245 +
246 + During sampling, we gather logits from all ranks. The relationship
247 + of index->token_id will follow the same format as outlined in the class
248 + docstring. However, after the gather, we want to reindex the final
249 + logits tensor to map index->token_id one-to-one (the index is always
250 + equal the token_id it corresponds to). The indices returned by this
251 + method allow us to do that.
252 + """
253 + if self.tp_size < 2:
254 + return None
255 +
256 + base_embeddings: List[int] = []
257 + added_embeddings: List[int] = []
258 + padding: List[int] = []
259 + for tp_rank in range(self.tp_size):
260 + shard_indices = self._get_indices(self.num_embeddings_padded,
261 + self.org_vocab_size_padded,
262 + self.num_embeddings,
263 + self.org_vocab_size, tp_rank,
264 + self.tp_size)
265 + range_start = self.num_embeddings_per_partition * tp_rank
266 + range_end = self.num_embeddings_per_partition * (tp_rank + 1)
267 + base_embeddings.extend(
268 + range(range_start,
269 + range_start + shard_indices.num_org_elements))
270 + padding.extend(
271 + range(range_start + shard_indices.num_org_elements,
272 + range_start + shard_indices.num_org_elements_padded))
273 + added_embeddings.extend(
274 + range(
275 + range_start + shard_indices.num_org_elements_padded,
276 + range_start + shard_indices.num_org_elements_padded +
277 + shard_indices.num_added_elements))
278 + padding.extend(
279 + range(
280 + range_start + shard_indices.num_org_elements_padded +
281 + shard_indices.num_added_elements,
282 + range_start + shard_indices.num_org_elements_padded +
283 + shard_indices.num_added_elements_padded))
284 + assert (range_start + shard_indices.num_org_elements_padded +
285 + shard_indices.num_added_elements_padded == range_end)
286 + ret = base_embeddings + added_embeddings + padding
287 + assert len(ret) == self.num_embeddings_padded
288 + return ret
289 +
290 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
291 parallel_dim = param.parallel_dim
292 assert loaded_weight.shape[parallel_dim] == self.org_vocab_size
85 - loaded_weight = loaded_weight[self.vocab_start_index:self.
86 - vocab_end_index]
293 + loaded_weight = loaded_weight[self.shard_indices.org_vocab_start_index:
294 + self.shard_indices.org_vocab_end_index]
295 param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
296 + param[loaded_weight.shape[0]:].data.fill_(0)
297
298 def forward(self, input_):
299 if self.tp_size > 1:
300 # Build the mask.
92 - input_mask = ((input_ < self.vocab_start_index) |
93 - (input_ >= self.vocab_end_index))
94 - # Mask the input.
95 - masked_input = input_.clone() - self.vocab_start_index
96 - masked_input[input_mask] = 0
301 + masked_input, input_mask = get_masked_input_and_mask(
302 + input_, self.shard_indices.org_vocab_start_index,
303 + self.shard_indices.org_vocab_end_index,
304 + self.shard_indices.num_org_vocab_padding,
305 + self.shard_indices.added_vocab_start_index,
306 + self.shard_indices.added_vocab_end_index)
307 else:
308 masked_input = input_
309 # Get the embeddings.
310 output_parallel = F.embedding(masked_input, self.weight)
311 # Mask the output embedding.
312 if self.tp_size > 1:
103 - output_parallel[input_mask, :] = 0.0
313 + output_parallel.masked_fill_(input_mask.unsqueeze(1), 0)
314 # Reduce across all the model parallel GPUs.
315 output = tensor_model_parallel_all_reduce(output_parallel)
316 return output
vllm/worker/model_runner.py
+11 -8
@@ -35,6 +35,7 @@ _BATCH_SIZE_ALIGNMENT = 8
35 _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
36 _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
37 ]
38 +_NUM_WARMUP_ITERS = 2
39
40
41 class ModelInput(NamedTuple):
@@ -975,16 +976,18 @@ class CUDAGraphRunner:
976 **kwargs,
977 ) -> None:
978 assert self._graph is None
978 - # Run the model once without capturing the graph.
979 + # Run the model a few times without capturing the graph.
980 # This is to make sure that the captured graph does not include the
981 # kernel launches for initial benchmarking (e.g., Triton autotune).
981 - self.model(
982 - input_ids,
983 - positions,
984 - kv_caches,
985 - attn_metadata,
986 - **kwargs,
987 - )
982 + # Note one iteration is not enough for torch.jit.script
983 + for _ in range(_NUM_WARMUP_ITERS):
984 + self.model(
985 + input_ids,
986 + positions,
987 + kv_caches,
988 + attn_metadata,
989 + **kwargs,
990 + )
991 torch.cuda.synchronize()
992
993 # Capture the graph.