@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96b6f475dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Remove hardcoded `device="cuda" ` to support more devices (#2503)

Co-authored-by: Jiang Li <jiang1.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>

Kunshang Ji committed Feb 2, 2024 at 07:46 UTC 96b6f475dda40a0c7d557f73c36fe09c07be2e9c
32 files changed +346 -295
benchmarks/benchmark_latency.py
+7
@@ -25,6 +25,7 @@ def main(args: argparse.Namespace):
25 dtype=args.dtype,
26 enforce_eager=args.enforce_eager,
27 kv_cache_dtype=args.kv_cache_dtype,
28 + device=args.device,
29 )
30
31 sampling_params = SamplingParams(
@@ -135,5 +136,11 @@ if __name__ == '__main__':
136 default=None,
137 help=('path to save the pytorch profiler output. Can be visualized '
138 'with ui.perfetto.dev or Tensorboard.'))
139 + parser.add_argument(
140 + "--device",
141 + type=str,
142 + default="cuda",
143 + choices=["cuda"],
144 + help='device type for vLLM execution, supporting CUDA only currently.')
145 args = parser.parse_args()
146 main(args)
benchmarks/benchmark_throughput.py
+9 -1
@@ -72,6 +72,7 @@ def run_vllm(
72 max_model_len: Optional[int],
73 enforce_eager: bool,
74 kv_cache_dtype: str,
75 + device: str,
76 ) -> float:
77 from vllm import LLM, SamplingParams
78 llm = LLM(
@@ -85,6 +86,7 @@ def run_vllm(
86 max_model_len=max_model_len,
87 enforce_eager=enforce_eager,
88 kv_cache_dtype=kv_cache_dtype,
89 + device=device,
90 )
91
92 # Add the requests to the engine.
@@ -209,7 +211,7 @@ def main(args: argparse.Namespace):
211 args.seed, args.n, args.use_beam_search,
212 args.trust_remote_code, args.dtype,
213 args.max_model_len, args.enforce_eager,
212 - args.kv_cache_dtype)
214 + args.kv_cache_dtype, args.device)
215 elif args.backend == "hf":
216 assert args.tensor_parallel_size == 1
217 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -294,6 +296,12 @@ if __name__ == "__main__":
296 default="auto",
297 help=
298 'Data type for kv cache storage. If "auto", will use model data type.')
299 + parser.add_argument(
300 + "--device",
301 + type=str,
302 + default="cuda",
303 + choices=["cuda"],
304 + help='device type for vLLM execution, supporting CUDA only currently.')
305 args = parser.parse_args()
306 if args.tokenizer is None:
307 args.tokenizer = args.model
benchmarks/kernels/benchmark_paged_attention.py
+18 -9
@@ -25,18 +25,20 @@ def main(
25 dtype: torch.dtype,
26 seed: int,
27 do_profile: bool,
28 + device: str = "cuda",
29 kv_cache_dtype: Optional[str] = None,
30 ) -> None:
31 random.seed(seed)
32 torch.random.manual_seed(seed)
32 - torch.cuda.manual_seed(seed)
33 + if torch.cuda.is_available():
34 + torch.cuda.manual_seed(seed)
35
36 scale = float(1.0 / (head_size**0.5))
37 query = torch.empty(num_seqs,
38 num_query_heads,
39 head_size,
40 dtype=dtype,
39 - device="cuda")
41 + device=device)
42 query.uniform_(-scale, scale)
43
44 assert num_query_heads % num_kv_heads == 0
@@ -44,11 +46,11 @@ def main(
46 if use_alibi:
47 alibi_slopes = torch.randn(num_query_heads,
48 dtype=torch.float,
47 - device="cuda")
49 + device=device)
50
51 context_lens = [context_len for _ in range(num_seqs)]
52 max_context_len = max(context_lens)
51 - context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
53 + context_lens = torch.tensor(context_lens, dtype=torch.int, device=device)
54
55 # Create the block tables.
56 max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
@@ -59,12 +61,17 @@ def main(
61 for _ in range(max_num_blocks_per_seq)
62 ]
63 block_tables.append(block_table)
62 - block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
64 + block_tables = torch.tensor(block_tables, dtype=torch.int, device=device)
65
66 # Create the KV cache.
65 - key_caches, value_caches = create_kv_caches_with_random(
66 - NUM_BLOCKS, block_size, 1, num_kv_heads, head_size, kv_cache_dtype,
67 - dtype)
67 + key_caches, value_caches = create_kv_caches_with_random(NUM_BLOCKS,
68 + block_size,
69 + 1,
70 + num_kv_heads,
71 + head_size,
72 + kv_cache_dtype,
73 + dtype,
74 + device=device)
75 key_cache, value_cache = key_caches[0], value_caches[0]
76
77 # Prepare for the paged attention kernel.
@@ -84,7 +91,7 @@ def main(
91 )
92 max_logits = torch.empty_like(exp_sums)
93
87 - def run_benchmark(num_iters: int, profile: bool = False) -> float:
94 + def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float:
95 torch.cuda.synchronize()
96 if profile:
97 torch.cuda.cudart().cudaProfilerStart()
@@ -135,6 +142,7 @@ def main(
142
143 # Warmup.
144 print("Warming up...")
145 + run_benchmark = run_cuda_benchmark
146 run_benchmark(num_iters=3, profile=False)
147
148 # Benchmark.
@@ -175,6 +183,7 @@ if __name__ == '__main__':
183 default="auto",
184 help=
185 'Data type for kv cache storage. If "auto", will use model data type.')
186 + parser.add_argument("--device", type=str, choices=["cuda"], default="cuda")
187 args = parser.parse_args()
188 print(args)
189
tests/kernels/test_activation.py
+21 -16
@@ -7,26 +7,29 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
7 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
8 D = [512, 4096, 5120, 13824] # Arbitrary values for testing
9 SEEDS = [0]
10 -DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
10 +CUDA_DEVICES = [
11 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
12 +]
13
14
15 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
16 @pytest.mark.parametrize("d", D)
17 @pytest.mark.parametrize("dtype", DTYPES)
18 @pytest.mark.parametrize("seed", SEEDS)
17 -@pytest.mark.parametrize("device", DEVICES)
19 +@pytest.mark.parametrize("device", CUDA_DEVICES)
20 @torch.inference_mode()
21 def test_silu_and_mul(
22 num_tokens: int,
23 d: int,
24 dtype: torch.dtype,
25 seed: int,
24 - device: int,
26 + device: str,
27 ) -> None:
28 torch.random.manual_seed(seed)
27 - torch.cuda.manual_seed(seed)
28 - gpu_id = f"cuda:{device}"
29 - x = torch.randn(num_tokens, 2 * d, dtype=dtype, device=gpu_id)
29 + if torch.cuda.is_available():
30 + torch.cuda.manual_seed(seed)
31 + torch.set_default_device(device)
32 + x = torch.randn(num_tokens, 2 * d, dtype=dtype)
33 layer = SiluAndMul()
34 out = layer(x)
35 ref_out = layer._forward(x)
@@ -37,19 +40,20 @@ def test_silu_and_mul(
40 @pytest.mark.parametrize("d", D)
41 @pytest.mark.parametrize("dtype", DTYPES)
42 @pytest.mark.parametrize("seed", SEEDS)
40 -@pytest.mark.parametrize("device", DEVICES)
43 +@pytest.mark.parametrize("device", CUDA_DEVICES)
44 @torch.inference_mode()
45 def test_gelu_new(
46 num_tokens: int,
47 d: int,
48 dtype: torch.dtype,
49 seed: int,
47 - device: int,
50 + device: str,
51 ) -> None:
52 torch.random.manual_seed(seed)
50 - torch.cuda.manual_seed(seed)
51 - gpu_id = f"cuda:{device}"
52 - x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
53 + if torch.cuda.is_available():
54 + torch.cuda.manual_seed(seed)
55 + torch.set_default_device(device)
56 + x = torch.randn(num_tokens, d, dtype=dtype)
57 layer = NewGELU()
58 out = layer(x)
59 ref_out = layer._forward(x)
@@ -60,18 +64,19 @@ def test_gelu_new(
64 @pytest.mark.parametrize("d", D)
65 @pytest.mark.parametrize("dtype", DTYPES)
66 @pytest.mark.parametrize("seed", SEEDS)
63 -@pytest.mark.parametrize("device", DEVICES)
67 +@pytest.mark.parametrize("device", CUDA_DEVICES)
68 def test_gelu_fast(
69 num_tokens: int,
70 d: int,
71 dtype: torch.dtype,
72 seed: int,
69 - device: int,
73 + device: str,
74 ) -> None:
75 torch.random.manual_seed(seed)
72 - torch.cuda.manual_seed(seed)
73 - gpu_id = f"cuda:{device}"
74 - x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
76 + if torch.cuda.is_available():
77 + torch.cuda.manual_seed(seed)
78 + torch.set_default_device(device)
79 + x = torch.randn(num_tokens, d, dtype=dtype)
80 layer = FastGELU()
81 out = layer(x)
82 ref_out = layer._forward(x)
tests/kernels/test_attention.py
+23 -28
@@ -27,7 +27,9 @@ BLOCK_SIZES = [16, 32]
27 USE_ALIBI = [False, True]
28 KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
29 SEEDS = [0]
30 -DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
30 +CUDA_DEVICES = [
31 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
32 +]
33
34
35 def ref_masked_attention(
@@ -91,7 +93,7 @@ def ref_single_query_cached_kv_attention(
93 alibi_bias = None
94 if alibi_slopes is not None:
95 # Create the ALiBi bias used in the paged attention kernel.
94 - position_ids = torch.arange(context_len, device=query.device).int()
96 + position_ids = torch.arange(context_len).int()
97 alibi_bias = (position_ids - context_len + 1).float()
98 alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
99 1, 1, -1)
@@ -110,7 +112,7 @@ def ref_single_query_cached_kv_attention(
112 @pytest.mark.parametrize("dtype", DTYPES)
113 @pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
114 @pytest.mark.parametrize("seed", SEEDS)
113 -@pytest.mark.parametrize("device", DEVICES)
115 +@pytest.mark.parametrize("device", CUDA_DEVICES)
116 def test_paged_attention(
117 kv_cache_factory,
118 version: str,
@@ -122,33 +124,28 @@ def test_paged_attention(
124 dtype: torch.dtype,
125 kv_cache_dtype: str,
126 seed: int,
125 - device: int,
127 + device: str,
128 ) -> None:
129 random.seed(seed)
130 torch.random.manual_seed(seed)
129 - torch.cuda.manual_seed(seed)
130 - gpu_id = f"cuda:{device}"
131 + if torch.cuda.is_available():
132 + torch.cuda.manual_seed(seed)
133 + torch.set_default_device(device)
134 scale = float(1.0 / (head_size**0.5))
135 num_query_heads, num_kv_heads = num_heads
133 - query = torch.empty(num_seqs,
134 - num_query_heads,
135 - head_size,
136 - dtype=dtype,
137 - device=gpu_id)
136 + query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
137 query.uniform_(-scale, scale)
138
139 assert num_query_heads % num_kv_heads == 0
140 num_queries_per_kv = num_query_heads // num_kv_heads
141 alibi_slopes = None
142 if use_alibi:
144 - alibi_slopes = torch.randn(num_query_heads,
145 - dtype=torch.float,
146 - device=gpu_id)
143 + alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
144
145 context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
146 context_lens[-1] = MAX_SEQ_LEN
147 max_context_len = max(context_lens)
151 - context_lens = torch.tensor(context_lens, dtype=torch.int, device=gpu_id)
148 + context_lens = torch.tensor(context_lens, dtype=torch.int)
149
150 # Create the block tables.
151 max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
@@ -159,13 +156,13 @@ def test_paged_attention(
156 for _ in range(max_num_blocks_per_seq)
157 ]
158 block_tables.append(block_table)
162 - block_tables = torch.tensor(block_tables, dtype=torch.int, device=gpu_id)
159 + block_tables = torch.tensor(block_tables, dtype=torch.int)
160
161 # Create the KV caches.
162 key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
163 num_kv_heads, head_size,
164 kv_cache_dtype, dtype, seed,
168 - gpu_id)
165 + device)
166 key_cache, value_cache = key_caches[0], value_caches[0]
167
168 # Call the paged attention kernel.
@@ -193,12 +190,10 @@ def test_paged_attention(
190 tmp_output = torch.empty(
191 size=(num_seqs, num_heads, num_partitions, head_size),
192 dtype=output.dtype,
196 - device=output.device,
193 )
194 exp_sums = torch.empty(
195 size=(num_seqs, num_heads, num_partitions),
196 dtype=torch.float32,
201 - device=output.device,
197 )
198 max_logits = torch.empty_like(exp_sums)
199 ops.paged_attention_v2(
@@ -229,14 +224,14 @@ def test_paged_attention(
224 block_size, x)
225 dequantized_key_cache = torch.empty(size=key_cache_shape,
226 dtype=dtype,
232 - device=gpu_id)
227 + device=device)
228 cache_ops.convert_fp8_e5m2(key_cache, dequantized_key_cache)
229 key_cache = dequantized_key_cache
230
231 value_cache_shape = value_cache.shape
232 dequantized_value_cache = torch.empty(size=value_cache_shape,
233 dtype=dtype,
239 - device=gpu_id)
234 + device=device)
235 cache_ops.convert_fp8_e5m2(value_cache, dequantized_value_cache)
236 value_cache = dequantized_value_cache
237
@@ -283,7 +278,7 @@ def ref_multi_query_kv_attention(
278 attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
279 diagonal=1)
280 attn_mask = attn_mask * torch.finfo(dtype).min
286 - attn_mask = attn_mask.to(dtype=dtype, device=query.device)
281 + attn_mask = attn_mask.to(dtype=dtype)
282
283 ref_output = ref_masked_attention(
284 query[start_idx:end_idx],
@@ -303,7 +298,7 @@ def ref_multi_query_kv_attention(
298 @pytest.mark.parametrize("head_size", HEAD_SIZES)
299 @pytest.mark.parametrize("dtype", DTYPES)
300 @pytest.mark.parametrize("seed", SEEDS)
306 -@pytest.mark.parametrize("device", DEVICES)
301 +@pytest.mark.parametrize("device", CUDA_DEVICES)
302 @torch.inference_mode()
303 def test_multi_query_kv_attention(
304 num_seqs: int,
@@ -311,12 +306,13 @@ def test_multi_query_kv_attention(
306 head_size: int,
307 dtype: torch.dtype,
308 seed: int,
314 - device: int,
309 + device: str,
310 ) -> None:
311 random.seed(seed)
312 torch.random.manual_seed(seed)
318 - torch.cuda.manual_seed(seed)
319 - gpu_id = f"cuda:{device}"
313 + if torch.cuda.is_available():
314 + torch.cuda.manual_seed(seed)
315 + torch.set_default_device(device)
316 # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
317 # As the xformers library is already tested with its own tests, we can use
318 # a smaller MAX_SEQ_LEN here.
@@ -329,8 +325,7 @@ def test_multi_query_kv_attention(
325 qkv = torch.empty(num_tokens,
326 num_query_heads + 2 * num_kv_heads,
327 head_size,
332 - dtype=dtype,
333 - device=gpu_id)
328 + dtype=dtype)
329 qkv.uniform_(-scale, scale)
330 query, key, value = qkv.split(
331 [num_query_heads, num_kv_heads, num_kv_heads], dim=1)
tests/kernels/test_cache.py
+21 -21
@@ -17,7 +17,9 @@ BLOCK_SIZES = [8, 16, 32]
17 NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
18 NUM_MAPPINGS = [256] # Arbitrary values for testing
19 SEEDS = [0]
20 -DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
20 +CUDA_DEVICES = [
21 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
22 +]
23 KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
24
25
@@ -29,7 +31,7 @@ KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
31 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
32 @pytest.mark.parametrize("dtype", DTYPES)
33 @pytest.mark.parametrize("seed", SEEDS)
32 -@pytest.mark.parametrize("device", DEVICES)
34 +@pytest.mark.parametrize("device", CUDA_DEVICES)
35 @pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
36 @torch.inference_mode()
37 def test_copy_blocks(
@@ -42,13 +44,14 @@ def test_copy_blocks(
44 num_blocks: int,
45 dtype: torch.dtype,
46 seed: int,
45 - device: int,
47 kv_cache_dtype: str,
48 + device: str,
49 ) -> None:
50 random.seed(seed)
51 torch.random.manual_seed(seed)
50 - torch.cuda.manual_seed(seed)
51 - gpu_id = f"cuda:{device}"
52 + if torch.cuda.is_available():
53 + torch.cuda.manual_seed(seed)
54 + torch.set_default_device(device)
55 # Generate random block mappings where each source block is mapped to two
56 # destination blocks.
57 assert 2 * num_mappings <= num_blocks
@@ -66,7 +69,7 @@ def test_copy_blocks(
69 key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
70 num_layers, num_heads,
71 head_size, kv_cache_dtype,
69 - dtype, seed, gpu_id)
72 + dtype, seed, device)
73
74 # Clone the KV caches.
75 cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
@@ -98,7 +101,7 @@ def test_copy_blocks(
101 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
102 @pytest.mark.parametrize("dtype", DTYPES)
103 @pytest.mark.parametrize("seed", SEEDS)
101 -@pytest.mark.parametrize("device", DEVICES)
104 +@pytest.mark.parametrize("device", CUDA_DEVICES)
105 @torch.inference_mode()
106 def test_reshape_and_cache(
107 kv_cache_factory,
@@ -109,29 +112,25 @@ def test_reshape_and_cache(
112 num_blocks: int,
113 dtype: torch.dtype,
114 seed: int,
112 - device: int,
115 + device: str,
116 ) -> None:
117 random.seed(seed)
118 torch.random.manual_seed(seed)
116 - torch.cuda.manual_seed(seed)
117 - gpu_id = f"cuda:{device}"
119 + if torch.cuda.is_available():
120 + torch.cuda.manual_seed(seed)
121 + torch.set_default_device(device)
122 # Create a random slot mapping.
123 num_slots = block_size * num_blocks
124 slot_mapping = random.sample(range(num_slots), num_tokens)
121 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=gpu_id)
122 -
123 - qkv = torch.randn(num_tokens,
124 - 3,
125 - num_heads,
126 - head_size,
127 - dtype=dtype,
128 - device=gpu_id)
125 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.long)
126 +
127 + qkv = torch.randn(num_tokens, 3, num_heads, head_size, dtype=dtype)
128 _, key, value = qkv.unbind(dim=1)
129
130 # Create the KV caches.
131 key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
132 num_heads, head_size, dtype,
134 - None, seed, gpu_id)
133 + None, seed, device)
134 key_cache, value_cache = key_caches[0], value_caches[0]
135
136 # Clone the KV caches.
@@ -166,7 +165,7 @@ def test_reshape_and_cache(
165 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
166 @pytest.mark.parametrize("dtype", DTYPES)
167 @pytest.mark.parametrize("seed", SEEDS)
169 -@pytest.mark.parametrize("device", DEVICES)
168 +@pytest.mark.parametrize("device", CUDA_DEVICES)
169 @torch.inference_mode()
170 def test_swap_blocks(
171 kv_cache_factory,
@@ -182,7 +181,8 @@ def test_swap_blocks(
181 ) -> None:
182 random.seed(seed)
183 torch.random.manual_seed(seed)
185 - torch.cuda.manual_seed(seed)
184 + if torch.cuda.is_available():
185 + torch.cuda.manual_seed(seed)
186 src_device = f"{direction[0]}:{device}" if direction[
187 0] == "cuda" else direction[0]
188 dst_device = f"{direction[1]}:{device}" if direction[
tests/kernels/test_layernorm.py
+10 -7
@@ -8,7 +8,9 @@ NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8 HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
9 ADD_RESIDUAL = [False, True]
10 SEEDS = [0]
11 -DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
11 +CUDA_DEVICES = [
12 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
13 +]
14
15
16 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -16,7 +18,7 @@ DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
18 @pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
19 @pytest.mark.parametrize("dtype", DTYPES)
20 @pytest.mark.parametrize("seed", SEEDS)
19 -@pytest.mark.parametrize("device", DEVICES)
21 +@pytest.mark.parametrize("device", CUDA_DEVICES)
22 @torch.inference_mode()
23 def test_rms_norm(
24 num_tokens: int,
@@ -24,15 +26,16 @@ def test_rms_norm(
26 add_residual: bool,
27 dtype: torch.dtype,
28 seed: int,
27 - device: int,
29 + device: str,
30 ) -> None:
31 torch.random.manual_seed(seed)
30 - torch.cuda.manual_seed(seed)
31 - gpu_id = f"cuda:{device}"
32 - layer = RMSNorm(hidden_size).to(dtype=dtype, device=gpu_id)
32 + if torch.cuda.is_available():
33 + torch.cuda.manual_seed(seed)
34 + torch.set_default_device(device)
35 + layer = RMSNorm(hidden_size).to(dtype=dtype)
36 layer.weight.data.normal_(mean=1.0, std=0.1)
37 scale = 1 / (2 * hidden_size)
35 - x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=gpu_id)
38 + x = torch.randn(num_tokens, hidden_size, dtype=dtype)
39 x *= scale
40 residual = torch.randn_like(x) * scale if add_residual else None
41
tests/kernels/test_pos_encoding.py
+11 -11
@@ -13,7 +13,9 @@ NUM_HEADS = [7, 17] # Arbitrary values for testing
13 BATCH_SIZES = [1, 5] # Arbitrary values for testing
14 SEQ_LENS = [11, 8192] # Arbitrary values for testing
15 SEEDS = [0]
16 -DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
16 +CUDA_DEVICES = [
17 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
18 +]
19
20
21 @pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -24,7 +26,7 @@ DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
26 @pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
27 @pytest.mark.parametrize("dtype", DTYPES)
28 @pytest.mark.parametrize("seed", SEEDS)
27 -@pytest.mark.parametrize("device", DEVICES)
29 +@pytest.mark.parametrize("device", CUDA_DEVICES)
30 @torch.inference_mode()
31 def test_rotary_embedding(
32 is_neox_style: bool,
@@ -35,28 +37,26 @@ def test_rotary_embedding(
37 rotary_dim: Optional[int],
38 dtype: torch.dtype,
39 seed: int,
38 - device: int,
40 + device: str,
41 max_position: int = 8192,
42 base: int = 10000,
43 ) -> None:
44 if rotary_dim is None:
45 rotary_dim = head_size
46 torch.random.manual_seed(seed)
45 - torch.cuda.manual_seed(seed)
46 - gpu_id = f"cuda:{device}"
47 + if torch.cuda.is_available():
48 + torch.cuda.manual_seed(seed)
49 + torch.set_default_device(device)
50 if rotary_dim is None:
51 rotary_dim = head_size
52 rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
50 - rope = rope.to(dtype=dtype, device=gpu_id)
53 + rope = rope.to(dtype=dtype)
54
52 - positions = torch.randint(0,
53 - max_position, (batch_size, seq_len),
54 - device=gpu_id)
55 + positions = torch.randint(0, max_position, (batch_size, seq_len))
56 query = torch.randn(batch_size,
57 seq_len,
58 num_heads * head_size,
58 - dtype=dtype,
59 - device=gpu_id)
59 + dtype=dtype)
60 key = torch.randn_like(query)
61
62 # NOTE(woosuk): The reference implementation should be executed first
tests/kernels/test_prefix_prefill.py
+20 -37
@@ -11,19 +11,27 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
11 NUM_HEADS = [12]
12 HEAD_SIZES = [128]
13 DTYPES = [torch.float16]
14 +CUDA_DEVICES = [
15 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
16 +]
17
18
19 @pytest.mark.parametrize("num_heads", NUM_HEADS)
20 @pytest.mark.parametrize("head_size", HEAD_SIZES)
21 @pytest.mark.parametrize("dtype", DTYPES)
22 +@pytest.mark.parametrize("device", CUDA_DEVICES)
23 @torch.inference_mode()
24 def test_contexted_kv_attention(
25 num_heads: int,
26 head_size: int,
27 dtype: torch.dtype,
28 + device: str,
29 ) -> None:
30 random.seed(0)
31 torch.manual_seed(0)
32 + if torch.cuda.is_available():
33 + torch.cuda.manual_seed(0)
34 + torch.set_default_device(device)
35 MAX_SEQ_LEN = 1024
36 MAX_CTX_LEN = 1024
37 BS = 10
@@ -35,24 +43,11 @@ def test_contexted_kv_attention(
43 seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
44
45 num_tokens = sum(subquery_lens)
38 - query = torch.empty(num_tokens,
39 - num_heads,
40 - head_size,
41 - dtype=dtype,
42 - device='cuda')
46 + query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
47 query.uniform_(-1e-3, 1e-3)
44 - output = torch.empty(num_tokens,
45 - num_heads,
46 - head_size,
47 - dtype=dtype,
48 - device='cuda')
48 + output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
49
50 - kv = torch.empty(sum(seq_lens),
51 - 2,
52 - num_heads,
53 - head_size,
54 - dtype=dtype,
55 - device='cuda')
50 + kv = torch.empty(sum(seq_lens), 2, num_heads, head_size, dtype=dtype)
51 kv.uniform_(-1e-3, 1e-3)
52 key, value = kv.unbind(dim=1)
53
@@ -60,39 +55,27 @@ def test_contexted_kv_attention(
55 block_size,
56 num_heads,
57 head_size,
63 - dtype=dtype,
64 - device='cuda')
58 + dtype=dtype)
59 v_cache = torch.zeros(cache_size,
60 block_size,
61 num_heads,
62 head_size,
69 - dtype=dtype,
70 - device='cuda')
71 - k = torch.zeros(sum(subquery_lens),
72 - num_heads,
73 - head_size,
74 - dtype=dtype,
75 - device='cuda')
76 - v = torch.zeros(sum(subquery_lens),
77 - num_heads,
78 - head_size,
79 - dtype=dtype,
80 - device='cuda')
81 - values = torch.arange(0, cache_size, dtype=torch.long, device='cuda')
63 + dtype=dtype)
64 + k = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
65 + v = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
66 + values = torch.arange(0, cache_size, dtype=torch.long)
67 values = values[torch.randperm(cache_size)]
68 block_table = values[:BS * max_block_per_request].view(
69 BS, max_block_per_request)
85 - b_seq_len = torch.tensor(seq_lens, dtype=torch.long, device='cuda')
86 - b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long, device='cuda')
70 + b_seq_len = torch.tensor(seq_lens, dtype=torch.long)
71 + b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
72 b_start_loc = torch.cumsum(torch.tensor([0] + subquery_lens[:-1],
88 - dtype=torch.long,
89 - device='cuda'),
73 + dtype=torch.long),
74 dim=0)
75 max_input_len = MAX_SEQ_LEN
76 # copy kv to cache
77 b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
94 - dtype=torch.long,
95 - device='cuda'),
78 + dtype=torch.long),
79 dim=0)
80 for i in range(BS):
81 for j in range(subquery_lens[i]):
tests/lora/conftest.py
+2 -2
@@ -126,8 +126,8 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
126 cleanup()
127 get_model_old = get_model
128
129 - def get_model_patched(model_config, lora_config=None):
130 - return get_model_old(model_config,
129 + def get_model_patched(model_config, device_config, lora_config=None):
130 + return get_model_old(model_config, device_config,
131 LoRAConfig(max_loras=4, max_lora_rank=8))
132
133 with patch("vllm.worker.model_runner.get_model", get_model_patched):
tests/lora/test_layers.py
+22 -14
@@ -34,6 +34,9 @@ TOLERANCES = {
34 torch.float32: (5e-3, 5e-3),
35 torch.bfloat16: (3e-2, 2e-2),
36 }
37 +CUDA_DEVICES = [
38 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
39 +]
40
41
42 def get_random_id_to_index(num_loras: int,
@@ -151,14 +154,10 @@ def create_random_inputs(
154 for _ in range(num_inputs):
155 if input_type == torch.int:
156 inputs.append(
154 - torch.randint(low=int(low),
155 - high=int(high),
156 - size=input_size,
157 - device="cuda"))
157 + torch.randint(low=int(low), high=int(high), size=input_size))
158 else:
159 inputs.append(
160 - torch.rand(size=input_size, dtype=input_type, device="cuda") *
161 - high + low)
160 + torch.rand(size=input_size, dtype=input_type) * high + low)
161
162 lora_id = random.choice(active_lora_ids)
163 index_mapping += [lora_id] * input_size[0]
@@ -169,8 +168,10 @@ def create_random_inputs(
168
169 @torch.inference_mode()
170 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
172 -def test_embeddings(dist_init, num_loras) -> None:
171 +@pytest.mark.parametrize("device", CUDA_DEVICES)
172 +def test_embeddings(dist_init, num_loras, device) -> None:
173
174 + torch.set_default_device(device)
175 max_loras = 8
176 lora_config = LoRAConfig(max_loras=max_loras,
177 max_lora_rank=8,
@@ -259,8 +260,10 @@ def test_embeddings(dist_init, num_loras) -> None:
260 @torch.inference_mode()
261 # @pytest.mark.skip(reason="Fails when loras are in any slot other than the first.")
262 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
262 -def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
263 +@pytest.mark.parametrize("device", CUDA_DEVICES)
264 +def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
265
266 + torch.set_default_device(device)
267 max_loras = 8
268 lora_config = LoRAConfig(max_loras=max_loras,
269 max_lora_rank=8,
@@ -305,8 +308,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
308
309 # Add empty embeddings_tensors for unoccupied lora slots.
310 for _ in range(max_loras - len(embeddings_tensors)):
308 - embeddings_tensors.append(
309 - torch.zeros(embeddings_tensors[0].shape, device="cuda"))
311 + embeddings_tensors.append(torch.zeros(embeddings_tensors[0].shape))
312
313 inputs, index_mapping, prompt_mapping = create_random_inputs(
314 active_lora_ids=list(lora_dict.keys()),
@@ -388,8 +390,10 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
390
391 @torch.inference_mode()
392 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
391 -def test_lm_head_sampler(dist_init, num_loras) -> None:
393 +@pytest.mark.parametrize("device", CUDA_DEVICES)
394 +def test_lm_head_sampler(dist_init, num_loras, device) -> None:
395
396 + torch.set_default_device(device)
397 max_loras = 8
398 lora_config = LoRAConfig(max_loras=max_loras,
399 max_lora_rank=8,
@@ -432,7 +436,7 @@ def test_lm_head_sampler(dist_init, num_loras) -> None:
436 )
437 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
438
435 - input_ = torch.rand(20, 1024, device="cuda")
439 + input_ = torch.rand(20, 1024)
440 mapping_info = convert_mapping(
441 lora_mapping,
442 id_to_index,
@@ -500,8 +504,10 @@ def test_lm_head_sampler(dist_init, num_loras) -> None:
504 @torch.inference_mode()
505 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
506 @pytest.mark.parametrize("orientation", ["row", "column"])
503 -def test_linear_parallel(dist_init, num_loras, orientation) -> None:
507 +@pytest.mark.parametrize("device", CUDA_DEVICES)
508 +def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
509
510 + torch.set_default_device(device)
511 max_loras = 8
512 lora_config = LoRAConfig(max_loras=max_loras,
513 max_lora_rank=8,
@@ -597,8 +603,10 @@ def test_linear_parallel(dist_init, num_loras, orientation) -> None:
603 @torch.inference_mode()
604 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
605 @pytest.mark.parametrize("repeats", [2, 3])
600 -def test_column_parallel_packed(dist_init, num_loras, repeats) -> None:
606 +@pytest.mark.parametrize("device", CUDA_DEVICES)
607 +def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
608
609 + torch.set_default_device(device)
610 max_loras = 8
611 lora_config = LoRAConfig(max_loras=max_loras,
612 max_lora_rank=8,
tests/lora/test_worker.py
+3 -1
@@ -5,7 +5,8 @@ from unittest.mock import patch
5
6 from vllm.lora.models import LoRAMapping
7 from vllm.lora.request import LoRARequest
8 -from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig
8 +from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
9 + DeviceConfig, LoRAConfig)
10 from vllm.worker.worker import Worker
11
12
@@ -25,6 +26,7 @@ def test_worker_apply_lora(sql_lora_files):
26 ),
27 parallel_config=ParallelConfig(1, 1, False),
28 scheduler_config=SchedulerConfig(32, 32, 32, 256),
29 + device_config=DeviceConfig("cuda"),
30 local_rank=0,
31 rank=0,
32 lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
tests/samplers/test_rejection_sampler.py
+27 -37
@@ -9,6 +9,10 @@ from vllm.model_executor.utils import set_random_seed
9
10 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
12 +CUDA_DEVICES = [
13 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
14 +]
15 +
16
17 def mock_causal_accepted_tensor(
18 k: int, last_accepted_indices: torch.Tensor) -> torch.Tensor:
@@ -39,11 +43,14 @@ def mock_causal_accepted_tensor(
43 @pytest.mark.parametrize(
44 "which_tokens_accepted",
45 ["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
46 +@pytest.mark.parametrize("device", CUDA_DEVICES)
47 @torch.inference_mode()
43 -def test_correct_output_format(which_tokens_accepted: str, seed: int):
48 +def test_correct_output_format(which_tokens_accepted: str, seed: int,
49 + device: str):
50 """Verify the output has correct format given predetermined accepted matrix.
51 """
52 set_random_seed(seed)
53 + torch.set_default_device(device)
54
55 batch_size = 10
56 k = 5
@@ -66,18 +73,15 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int):
73 recovered_token_ids = torch.randint(low=0,
74 high=vocab_size,
75 size=(batch_size, k),
69 - dtype=torch.int64,
70 - device="cuda")
76 + dtype=torch.int64)
77 draft_token_ids = torch.randint(low=0,
78 high=vocab_size,
79 size=(batch_size, k),
74 - dtype=torch.int64,
75 - device="cuda")
80 + dtype=torch.int64)
81 bonus_token_ids = torch.randint(low=0,
82 high=vocab_size,
83 size=(batch_size, 1),
79 - dtype=torch.int64,
80 - device="cuda")
84 + dtype=torch.int64)
85
86 rejection_sampler = RejectionSampler()
87 rejection_sampler.init_gpu_tensors(rank=0)
@@ -120,31 +124,24 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int):
124 @pytest.mark.parametrize("k", list(range(1, 6)))
125 @pytest.mark.parametrize("vocab_size", [30_000, 50_000])
126 @pytest.mark.parametrize("batch_size", list(range(1, 32)))
127 +@pytest.mark.parametrize("device", CUDA_DEVICES)
128 @torch.inference_mode()
124 -def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int):
129 +def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
130 + device: str):
131 + torch.set_default_device(device)
132 rejection_sampler = RejectionSampler()
133 rejection_sampler.init_gpu_tensors(rank=0)
134
128 - draft_probs = torch.rand(batch_size,
129 - k,
130 - vocab_size,
131 - dtype=torch.float32,
132 - device="cuda")
133 - target_probs = torch.rand(batch_size,
134 - k,
135 - vocab_size,
136 - dtype=torch.float32,
137 - device="cuda")
135 + draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
136 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
137 bonus_token_ids = torch.randint(low=0,
138 high=vocab_size,
139 size=(batch_size, 1),
141 - dtype=torch.int64,
142 - device="cuda")
140 + dtype=torch.int64)
141 draft_token_ids = torch.randint(low=0,
142 high=vocab_size,
143 size=(batch_size, k),
146 - dtype=torch.int64,
147 - device="cuda")
144 + dtype=torch.int64)
145
146 rejection_sampler(target_probs, bonus_token_ids, draft_probs,
147 draft_token_ids)
@@ -153,36 +150,28 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int):
150 @pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
151 @pytest.mark.parametrize("which_token_ids",
152 ["bonus_token_ids", "draft_token_ids"])
153 +@pytest.mark.parametrize("device", CUDA_DEVICES)
154 @torch.inference_mode()
155 def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
158 - which_token_ids: str):
156 + which_token_ids: str, device: str):
157 k = 3
158 batch_size = 5
159 vocab_size = 30_000
160 + torch.set_default_device(device)
161
162 rejection_sampler = RejectionSampler(strict_mode=True)
163 rejection_sampler.init_gpu_tensors(rank=0)
164
166 - draft_probs = torch.rand(batch_size,
167 - k,
168 - vocab_size,
169 - dtype=torch.float32,
170 - device="cuda")
171 - target_probs = torch.rand(batch_size,
172 - k,
173 - vocab_size,
174 - dtype=torch.float32,
175 - device="cuda")
165 + draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
166 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
167 bonus_token_ids = torch.randint(low=0,
168 high=vocab_size,
169 size=(batch_size, 1),
179 - dtype=torch.int64,
180 - device="cuda")
170 + dtype=torch.int64)
171 draft_token_ids = torch.randint(low=0,
172 high=vocab_size,
173 size=(batch_size, k),
184 - dtype=torch.int64,
185 - device="cuda")
174 + dtype=torch.int64)
175
176 oob_token_ids = None
177 if which_token_ids == "bonus_token_ids":
@@ -237,6 +226,7 @@ def test_rejection_sampling_approximates_target_distribution(
226 probabilities are exactly equal. Rejection sampling should
227 still work without any NaNs or exceptions.
228 """
229 + torch.set_default_device("cpu")
230 set_random_seed(seed)
231
232 helper = _CorrectnessTestHelper(
tests/samplers/test_sampler.py
+24 -13
@@ -31,24 +31,26 @@ def _prepare_test(
31 batch_size: int
32 ) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
33 vocab_size = 32000
34 - input_tensor = torch.rand((batch_size, 1024),
35 - device="cuda",
36 - dtype=torch.float16)
34 + input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
35 fake_logits = torch.full((batch_size, vocab_size),
36 1e-2,
39 - device=input_tensor.device,
37 dtype=input_tensor.dtype)
38 sampler = MockLogitsSampler(32000, fake_logits)
42 - model_runner = ModelRunner(None, None, None, None)
39 + model_runner = ModelRunner(None, None, None, None, None)
40 return input_tensor, fake_logits, sampler, model_runner
41
42
43 RANDOM_SEEDS = list(range(128))
44 +CUDA_DEVICES = [
45 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
46 +]
47
48
49 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
50 -def test_sampler_all_greedy(seed: int):
50 +@pytest.mark.parametrize("device", CUDA_DEVICES)
51 +def test_sampler_all_greedy(seed: int, device: str):
52 set_random_seed(seed)
53 + torch.set_default_device(device)
54 batch_size = random.randint(1, 256)
55 input_tensor, fake_logits, sampler, model_runner = _prepare_test(
56 batch_size)
@@ -81,8 +83,10 @@ def test_sampler_all_greedy(seed: int):
83
84
85 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
84 -def test_sampler_all_random(seed: int):
86 +@pytest.mark.parametrize("device", CUDA_DEVICES)
87 +def test_sampler_all_random(seed: int, device: str):
88 set_random_seed(seed)
89 + torch.set_default_device(device)
90 batch_size = random.randint(1, 256)
91 input_tensor, fake_logits, sampler, model_runner = _prepare_test(
92 batch_size)
@@ -120,8 +124,10 @@ def test_sampler_all_random(seed: int):
124
125
126 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
123 -def test_sampler_all_beam(seed: int):
127 +@pytest.mark.parametrize("device", CUDA_DEVICES)
128 +def test_sampler_all_beam(seed: int, device: str):
129 set_random_seed(seed)
130 + torch.set_default_device(device)
131 batch_size = random.randint(1, 256)
132 input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
133
@@ -156,8 +162,10 @@ def test_sampler_all_beam(seed: int):
162
163
164 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
159 -def test_sampler_mixed(seed: int):
165 +@pytest.mark.parametrize("device", CUDA_DEVICES)
166 +def test_sampler_mixed(seed: int, device: str):
167 set_random_seed(seed)
168 + torch.set_default_device(device)
169 batch_size = random.randint(1, 256)
170 input_tensor, fake_logits, sampler, model_runner = _prepare_test(
171 batch_size)
@@ -212,8 +220,10 @@ def test_sampler_mixed(seed: int):
220
221
222 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
215 -def test_sampler_logits_processors(seed: int):
223 +@pytest.mark.parametrize("device", CUDA_DEVICES)
224 +def test_sampler_logits_processors(seed: int, device: str):
225 set_random_seed(seed)
226 + torch.set_default_device(device)
227 batch_size = random.randint(1, 256)
228 input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
229
@@ -252,14 +262,15 @@ def test_sampler_logits_processors(seed: int):
262
263
264 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
255 -def test_sampler_top_k_top_p(seed: int):
265 +@pytest.mark.parametrize("device", CUDA_DEVICES)
266 +def test_sampler_top_k_top_p(seed: int, device: str):
267 set_random_seed(seed)
268 batch_size = random.randint(1, 256)
269 top_k = random.randint(100, 500)
270 top_p = random.random() * 0.1
271 vocab_size = 32000
272 input_tensor = torch.rand((batch_size, 1024),
262 - device="cuda",
273 + device=device,
274 dtype=torch.float16)
275 fake_logits = torch.normal(0,
276 5,
@@ -267,7 +278,7 @@ def test_sampler_top_k_top_p(seed: int):
278 device=input_tensor.device,
279 dtype=input_tensor.dtype)
280 sampler = MockLogitsSampler(32000, fake_logits)
270 - model_runner = ModelRunner(None, None, None, None)
281 + model_runner = ModelRunner(None, None, None, None, None)
282
283 generation_model = GenerationMixin()
284 generation_config = GenerationConfig(top_k=top_k,
tests/worker/spec_decode/utils.py
+2 -1
@@ -84,7 +84,7 @@ def create_worker(cls: type,
84 )
85
86 (model_config, cache_config, parallel_config, scheduler_config,
87 - _) = engine_args.create_engine_configs()
87 + device_config, _) = engine_args.create_engine_configs()
88
89 distributed_init_method = get_distributed_init_method(
90 get_ip(), get_open_port())
@@ -93,6 +93,7 @@ def create_worker(cls: type,
93 model_config=model_config,
94 parallel_config=parallel_config,
95 scheduler_config=scheduler_config,
96 + device_config=device_config,
97 local_rank=0,
98 rank=0,
99 distributed_init_method=distributed_init_method,
tests/worker/test_model_runner.py
+1 -1
@@ -6,7 +6,7 @@ from vllm.worker.model_runner import ModelRunner
6
7
8 def test_prepare_prompt():
9 - model_runner = ModelRunner(None, None, None, None)
9 + model_runner = ModelRunner(None, None, None, None, None)
10 model_runner.set_block_size(16)
11
12 batch_size = random.randint(1, 256)
vllm/config.py
+6
@@ -444,6 +444,12 @@ class SchedulerConfig:
444 f"({self.max_num_seqs}).")
445
446
447 +class DeviceConfig:
448 +
449 + def __init__(self, device: str = "cuda") -> None:
450 + self.device = torch.device(device)
451 +
452 +
453 @dataclass
454 class LoRAConfig:
455 max_lora_rank: int
vllm/engine/arg_utils.py
+18 -7
@@ -3,8 +3,8 @@ import dataclasses
3 from dataclasses import dataclass
4 from typing import Optional, Tuple
5
6 -from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig, LoRAConfig)
6 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
7 + ParallelConfig, SchedulerConfig, LoRAConfig)
8
9
10 @dataclass
@@ -43,6 +43,7 @@ class EngineArgs:
43 lora_extra_vocab_size: int = 256
44 lora_dtype = 'auto'
45 max_cpu_loras: Optional[int] = None
46 + device: str = 'cuda'
47
48 def __post_init__(self):
49 if self.tokenizer is None:
@@ -127,13 +128,13 @@ class EngineArgs:
128 '--kv-cache-dtype',
129 type=str,
130 choices=['auto', 'fp8_e5m2'],
130 - default='auto',
131 + default=EngineArgs.kv_cache_dtype,
132 help='Data type for kv cache storage. If "auto", will use model '
133 'data type. Note FP8 is not supported when cuda version is '
134 'lower than 11.8.')
135 parser.add_argument('--max-model-len',
136 type=int,
136 - default=None,
137 + default=EngineArgs.max_model_len,
138 help='model context length. If unspecified, '
139 'will be automatically derived from the model.')
140 # Parallel arguments
@@ -154,6 +155,7 @@ class EngineArgs:
155 parser.add_argument(
156 '--max-parallel-loading-workers',
157 type=int,
158 + default=EngineArgs.max_parallel_loading_workers,
159 help='load model sequentially in multiple batches, '
160 'to avoid RAM OOM when using tensor '
161 'parallel and large models')
@@ -200,7 +202,7 @@ class EngineArgs:
202 '-q',
203 type=str,
204 choices=['awq', 'gptq', 'squeezellm', None],
203 - default=None,
205 + default=EngineArgs.quantization,
206 help='Method used to quantize the weights. If '
207 'None, we first check the `quantization_config` '
208 'attribute in the model config file. If that is '
@@ -255,6 +257,13 @@ class EngineArgs:
257 help=('Maximum number of LoRAs to store in CPU memory. '
258 'Must be >= than max_num_seqs. '
259 'Defaults to max_num_seqs.'))
260 + parser.add_argument(
261 + "--device",
262 + type=str,
263 + default=EngineArgs.device,
264 + choices=["cuda"],
265 + help=('Device type for vLLM execution. '
266 + 'Currently, only CUDA-compatible devices are supported.'))
267 return parser
268
269 @classmethod
@@ -268,7 +277,8 @@ class EngineArgs:
277 def create_engine_configs(
278 self,
279 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
271 - Optional[LoRAConfig]]:
280 + DeviceConfig, Optional[LoRAConfig]]:
281 + device_config = DeviceConfig(self.device)
282 model_config = ModelConfig(self.model, self.tokenizer,
283 self.tokenizer_mode, self.trust_remote_code,
284 self.download_dir, self.load_format,
@@ -296,7 +306,8 @@ class EngineArgs:
306 lora_dtype=self.lora_dtype,
307 max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
308 and self.max_cpu_loras > 0 else None) if self.enable_lora else None
299 - return model_config, cache_config, parallel_config, scheduler_config, lora_config
309 + return (model_config, cache_config, parallel_config, scheduler_config,
310 + device_config, lora_config)
311
312
313 @dataclass
vllm/engine/llm_engine.py
+10 -2
@@ -6,8 +6,8 @@ from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
6 Union)
7
8 from vllm.lora.request import LoRARequest
9 -from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
10 - SchedulerConfig, LoRAConfig)
9 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10 + ParallelConfig, SchedulerConfig, LoRAConfig)
11 from vllm.core.scheduler import Scheduler, SchedulerOutputs
12 from vllm.engine.arg_utils import EngineArgs
13 from vllm.engine.metrics import StatLogger, Stats
@@ -53,6 +53,7 @@ class LLMEngine:
53 management.
54 parallel_config: The configuration related to distributed execution.
55 scheduler_config: The configuration related to the request scheduler.
56 + device_config: The configuration related to the device.
57 placement_group: Ray placement group for distributed execution.
58 Required for distributed execution.
59 log_stats: Whether to log statistics.
@@ -64,6 +65,7 @@ class LLMEngine:
65 cache_config: CacheConfig,
66 parallel_config: ParallelConfig,
67 scheduler_config: SchedulerConfig,
68 + device_config: DeviceConfig,
69 lora_config: Optional[LoRAConfig],
70 placement_group: Optional["PlacementGroup"],
71 log_stats: bool,
@@ -85,6 +87,7 @@ class LLMEngine:
87 f"quantization={model_config.quantization}, "
88 f"enforce_eager={model_config.enforce_eager}, "
89 f"kv_cache_dtype={cache_config.cache_dtype}, "
90 + f"device_config={device_config.device}, "
91 f"seed={model_config.seed})")
92 # TODO(woosuk): Print more configs in debug mode.
93
@@ -93,6 +96,7 @@ class LLMEngine:
96 self.lora_config = lora_config
97 self.parallel_config = parallel_config
98 self.scheduler_config = scheduler_config
99 + self.device_config = device_config
100 self.log_stats = log_stats
101 self._verify_args()
102
@@ -138,6 +142,7 @@ class LLMEngine:
142 self.model_config,
143 self.parallel_config,
144 self.scheduler_config,
145 + self.device_config,
146 local_rank=0,
147 rank=0,
148 distributed_init_method=distributed_init_method,
@@ -233,6 +238,7 @@ class LLMEngine:
238 model_config = copy.deepcopy(self.model_config)
239 parallel_config = copy.deepcopy(self.parallel_config)
240 scheduler_config = copy.deepcopy(self.scheduler_config)
241 + device_config = copy.deepcopy(self.device_config)
242
243 for rank, (worker, (node_id,
244 _)) in enumerate(zip(self.workers,
@@ -244,6 +250,7 @@ class LLMEngine:
250 model_config,
251 parallel_config,
252 scheduler_config,
253 + device_config,
254 local_rank,
255 rank,
256 distributed_init_method,
@@ -257,6 +264,7 @@ class LLMEngine:
264 model_config,
265 parallel_config,
266 scheduler_config,
267 + device_config,
268 driver_local_rank,
269 driver_rank,
270 distributed_init_method,
vllm/model_executor/layers/activation.py
+1 -3
@@ -89,9 +89,7 @@ class ScaledActivation(nn.Module):
89 if params_dtype is None:
90 params_dtype = torch.get_default_dtype()
91 self.scales = nn.Parameter(
92 - torch.empty(intermediate_size_per_partition,
93 - dtype=params_dtype,
94 - device="cuda"))
92 + torch.empty(intermediate_size_per_partition, dtype=params_dtype))
93 set_weight_attrs(self.scales, {"weight_loader": self.weight_loader})
94
95 def forward(self, x: torch.Tensor) -> torch.Tensor:
vllm/model_executor/layers/attention.py
+1 -1
@@ -200,7 +200,7 @@ def _make_alibi_bias(
200 seq_len: int,
201 dtype: torch.dtype,
202 ) -> LowerTriangularMaskWithTensorBias:
203 - bias = torch.arange(seq_len, dtype=dtype, device="cuda")
203 + bias = torch.arange(seq_len, dtype=dtype)
204 # NOTE(zhuohan): HF uses
205 # `bias = bias[None, :].repeat(prompt_len, 1)`
206 # here. We find that both biases give the same results, but
vllm/model_executor/layers/linear.py
+2 -8
@@ -54,7 +54,6 @@ class UnquantizedLinearMethod(LinearMethodBase):
54 params_dtype: torch.dtype) -> Dict[str, Any]:
55 weight = Parameter(torch.empty(output_size_per_partition,
56 input_size_per_partition,
57 - device=torch.cuda.current_device(),
57 dtype=params_dtype),
58 requires_grad=False)
59 set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
@@ -113,9 +112,7 @@ class ReplicatedLinear(torch.nn.Module):
112 self.register_parameter(name, weight)
113 if bias:
114 self.bias = Parameter(
116 - torch.empty(self.output_size,
117 - device=torch.cuda.current_device(),
118 - dtype=self.params_dtype))
115 + torch.empty(self.output_size, dtype=self.params_dtype))
116 set_weight_attrs(self.bias, {"output_dim": 0})
117 else:
118 self.register_parameter("bias", None)
@@ -183,7 +180,6 @@ class ColumnParallelLinear(torch.nn.Module):
180 if bias:
181 self.bias = Parameter(
182 torch.empty(self.output_size_per_partition,
186 - device=torch.cuda.current_device(),
183 dtype=params_dtype))
184 set_weight_attrs(self.bias, {
185 "output_dim": 0,
@@ -509,9 +505,7 @@ class RowParallelLinear(torch.nn.Module):
505
506 if bias:
507 self.bias = Parameter(
512 - torch.empty(self.output_size,
513 - device=torch.cuda.current_device(),
514 - dtype=params_dtype))
508 + torch.empty(self.output_size, dtype=params_dtype))
509 set_weight_attrs(self.bias, {
510 "output_dim": 0,
511 "weight_loader": self.weight_loader,
vllm/model_executor/layers/quantization/awq.py
-3
@@ -96,7 +96,6 @@ class AWQLinearMethod(LinearMethodBase):
96 torch.empty(
97 input_size_per_partition,
98 output_size_per_partition // self.quant_config.pack_factor,
99 - device="cuda",
99 dtype=torch.int32,
100 ),
101 requires_grad=False,
@@ -112,7 +111,6 @@ class AWQLinearMethod(LinearMethodBase):
111 torch.empty(
112 input_size_per_partition // self.quant_config.group_size,
113 output_size_per_partition // self.quant_config.pack_factor,
115 - device="cuda",
114 dtype=torch.int32,
115 ),
116 requires_grad=False,
@@ -128,7 +126,6 @@ class AWQLinearMethod(LinearMethodBase):
126 torch.empty(
127 input_size_per_partition // self.quant_config.group_size,
128 output_size_per_partition,
131 - device="cuda",
129 dtype=params_dtype,
130 ),
131 requires_grad=False,
vllm/model_executor/layers/quantization/gptq.py
-4
@@ -127,7 +127,6 @@ class GPTQLinearMethod(LinearMethodBase):
127 torch.empty(
128 input_size_per_partition // self.quant_config.pack_factor,
129 output_size_per_partition,
130 - device="cuda",
130 dtype=torch.int32,
131 ),
132 requires_grad=False,
@@ -145,7 +144,6 @@ class GPTQLinearMethod(LinearMethodBase):
144 i // self.quant_config.group_size
145 for i in range(input_size_per_partition)
146 ],
148 - device="cuda",
147 dtype=torch.int32,
148 ),
149 requires_grad=False,
@@ -156,7 +154,6 @@ class GPTQLinearMethod(LinearMethodBase):
154 torch.empty(
155 scale_and_zero_size,
156 output_size_per_partition // self.quant_config.pack_factor,
159 - device="cuda",
157 dtype=torch.int32,
158 ),
159 requires_grad=False,
@@ -172,7 +169,6 @@ class GPTQLinearMethod(LinearMethodBase):
169 torch.empty(
170 scale_and_zero_size,
171 output_size_per_partition,
175 - device="cuda",
172 dtype=params_dtype,
173 ),
174 requires_grad=False,
vllm/model_executor/layers/quantization/squeezellm.py
+2 -4
@@ -80,7 +80,6 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
80 torch.empty(
81 input_size_per_partition // self.quant_config.pack_factor,
82 output_size_per_partition,
83 - device="cuda",
83 dtype=torch.int32,
84 ),
85 requires_grad=False,
@@ -96,7 +95,6 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
95 torch.empty(
96 output_size,
97 self.quant_config.weight_bits**2,
99 - device="cuda",
98 dtype=params_dtype,
99 ),
100 requires_grad=False,
@@ -118,12 +116,12 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
116 out_shape = x.shape[:-1] + (qweight.shape[-1], )
117 reshaped_x = x.reshape(-1, x.shape[-1])
118 if is_hip():
121 - out_f = torch.zeros(out_shape, device="cuda", dtype=torch.float)
119 + out_f = torch.zeros(out_shape, dtype=torch.float)
120 ops.squeezellm_gemm(reshaped_x, qweight, out_f, lookup_table)
121 out = out_f.to(dtype=torch.float16)
122 else:
123 # NOTE: The output tensor should be zero-initialized.
126 - out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
124 + out = torch.zeros(out_shape, dtype=torch.float16)
125 ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
126
127 if bias is not None:
vllm/model_executor/layers/rotary_embedding.py
+9 -13
@@ -77,16 +77,13 @@ class RotaryEmbedding(nn.Module):
77 # create the cache on GPU for faster initialization. This may cause
78 # a slight numerical difference between the HF implementation and ours.
79 inv_freq = 1.0 / (base**(torch.arange(
80 - 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
81 - self.rotary_dim))
80 + 0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim))
81 return inv_freq
82
83 def _compute_cos_sin_cache(self) -> torch.Tensor:
84 """Compute the cos and sin cache."""
85 inv_freq = self._compute_inv_freq(self.base)
87 - t = torch.arange(self.max_position_embeddings,
88 - dtype=torch.float,
89 - device="cuda")
86 + t = torch.arange(self.max_position_embeddings, dtype=torch.float)
87
88 freqs = torch.einsum("i,j -> ij", t, inv_freq)
89 cos = freqs.cos()
@@ -174,7 +171,7 @@ class LinearScalingRotaryEmbedding(RotaryEmbedding):
171 # Thus, the maximum length after applying the rope scaling is
172 # self.max_position_embeddings * self.scaling_factor.
173 max_len = self.max_position_embeddings * self.scaling_factor
177 - t = torch.arange(max_len, dtype=torch.float, device="cuda")
174 + t = torch.arange(max_len, dtype=torch.float)
175 t = t / self.scaling_factor
176
177 freqs = torch.einsum("i,j -> ij", t, inv_freq)
@@ -214,7 +211,7 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
211 (self.scaling_factor - 1))**(self.rotary_dim /
212 (self.rotary_dim - 2))
213 inv_freq = self._compute_inv_freq(base)
217 - t = torch.arange(max_len, dtype=torch.float, device="cuda")
214 + t = torch.arange(max_len, dtype=torch.float)
215
216 freqs = torch.einsum("i,j -> ij", t, inv_freq)
217 cos = freqs.cos()
@@ -297,9 +294,9 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
294 is_neox_style)
295
296 def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
300 - pos_freqs = self.base**(torch.arange(
301 - 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
302 - self.rotary_dim)
297 + pos_freqs = self.base**(
298 + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) /
299 + self.rotary_dim)
300 inv_freq_extrapolation = 1.0 / pos_freqs
301 inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
302
@@ -308,8 +305,8 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
305 self.max_position_embeddings)
306 # Get n-d rotational scaling corrected for extrapolation
307 inv_freq_mask = (1 - _yarn_linear_ramp_mask(
311 - low, high, self.rotary_dim // 2, dtype=torch.float,
312 - device="cuda")) * self.extrapolation_factor
308 + low, high, self.rotary_dim // 2,
309 + dtype=torch.float)) * self.extrapolation_factor
310 inv_freq = inv_freq_interpolation * (
311 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
312 return inv_freq
@@ -317,7 +314,6 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
314 def _compute_cos_sin_cache(self) -> torch.Tensor:
315 inv_freq = self._compute_inv_freq(self.scaling_factor)
316 t = torch.arange(self.max_position_embeddings * self.scaling_factor,
320 - device="cuda",
317 dtype=torch.float32)
318 freqs = torch.einsum("i,j -> ij", t, inv_freq)
319 cos = (freqs.cos() * self.mscale)
vllm/model_executor/layers/vocab_parallel_embedding.py
-2
@@ -77,7 +77,6 @@ class VocabParallelEmbedding(torch.nn.Module):
77 self.weight = Parameter(
78 torch.empty(self.num_embeddings_per_partition,
79 self.embedding_dim,
80 - device=torch.cuda.current_device(),
80 dtype=params_dtype))
81 set_weight_attrs(self.weight, {
82 "parallel_dim": 0,
@@ -139,7 +138,6 @@ class ParallelLMHead(VocabParallelEmbedding):
138 if bias:
139 self.bias = Parameter(
140 torch.empty(self.num_embeddings_per_partition,
142 - device=torch.cuda.current_device(),
141 dtype=params_dtype))
142 set_weight_attrs(self.bias, {
143 "parallel_dim": 0,
vllm/model_executor/model_loader.py
+3 -2
@@ -5,7 +5,7 @@ from typing import Optional, Type
5 import torch
6 import torch.nn as nn
7
8 -from vllm.config import ModelConfig, LoRAConfig
8 +from vllm.config import DeviceConfig, ModelConfig, LoRAConfig
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.model_executor.weight_utils import (get_quant_config,
11 initialize_dummy_weights)
@@ -38,6 +38,7 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
38
39
40 def get_model(model_config: ModelConfig,
41 + device_config: DeviceConfig,
42 lora_config: Optional[LoRAConfig] = None) -> nn.Module:
43 model_class = _get_model_architecture(model_config)
44
@@ -64,7 +65,7 @@ def get_model(model_config: ModelConfig,
65 with _set_default_torch_dtype(model_config.dtype):
66 # Create a model instance.
67 # The weights will be initialized as empty tensors.
67 - with torch.device("cuda"):
68 + with torch.device(device_config.device):
69 if getattr(model_class, "supports_lora", False):
70 model = model_class(model_config.hf_config, linear_method,
71 lora_config)
vllm/utils.py
+2 -1
@@ -228,7 +228,8 @@ def create_kv_caches_with_random(
228 device: Optional[str] = "cuda",
229 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
230 torch.random.manual_seed(seed)
231 - torch.cuda.manual_seed(seed)
231 + if torch.cuda.is_available():
232 + torch.cuda.manual_seed(seed)
233
234 if isinstance(cache_dtype, str):
235 if cache_dtype == "auto":
vllm/worker/cache_engine.py
+2
@@ -104,11 +104,13 @@ class CacheEngine:
104 size=(self.num_cpu_blocks, *key_block_shape),
105 dtype=self.dtype,
106 pin_memory=pin_memory,
107 + device="cpu",
108 )
109 value_blocks = torch.empty(
110 size=(self.num_cpu_blocks, *value_block_shape),
111 dtype=self.dtype,
112 pin_memory=pin_memory,
113 + device="cpu",
114 )
115 cpu_cache.append((key_blocks, value_blocks))
116 return cpu_cache
vllm/worker/model_runner.py
+46 -29
@@ -5,7 +5,7 @@ import numpy as np
5 import torch
6 import torch.nn as nn
7
8 -from vllm.config import ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
8 +from vllm.config import DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 from vllm.model_executor.parallel_utils.communication_op import (
@@ -35,6 +35,7 @@ class ModelRunner:
35 model_config: ModelConfig,
36 parallel_config: ParallelConfig,
37 scheduler_config: SchedulerConfig,
38 + device_config: DeviceConfig,
39 lora_config: Optional[LoRAConfig],
40 kv_cache_dtype: Optional[str] = "auto",
41 is_driver_worker: bool = False,
@@ -49,7 +50,10 @@ class ModelRunner:
50 # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
51 self.sliding_window = (model_config.get_sliding_window()
52 if model_config is not None else None)
52 - self.device = torch.device(torch.cuda.current_device())
53 + self.device_config = (device_config
54 + if device_config is not None else DeviceConfig())
55 + self.device = self.device_config.device
56 +
57 self.model = None
58 self.block_size = None # Set after initial profiling.
59 self.lora_manager = None
@@ -72,7 +76,8 @@ class ModelRunner:
76 self.kv_cache_dtype = kv_cache_dtype
77
78 def load_model(self) -> None:
75 - self.model = get_model(self.model_config, self.lora_config)
79 + self.model = get_model(self.model_config, self.device_config,
80 + self.lora_config)
81
82 vocab_size = self.model.config.vocab_size
83
@@ -182,22 +187,25 @@ class ModelRunner:
187 input_tokens = _make_tensor_with_pad(input_tokens,
188 max_prompt_len,
189 pad=0,
185 - dtype=torch.long)
190 + dtype=torch.long,
191 + device=self.device)
192 input_positions = _make_tensor_with_pad(input_positions,
193 max_prompt_len,
194 pad=0,
189 - dtype=torch.long)
195 + dtype=torch.long,
196 + device=self.device)
197 slot_mapping = _make_tensor_with_pad(slot_mapping,
198 max_prompt_len,
199 pad=_PAD_SLOT_ID,
193 - dtype=torch.long)
200 + dtype=torch.long,
201 + device=self.device)
202 lora_index_mapping = [
203 _pad_to_max(mapping, max_prompt_len, pad=0)
204 for mapping in lora_index_mapping
205 ]
206 context_lens_tensor = torch.tensor(context_lens,
207 dtype=torch.int,
200 - device='cuda')
208 + device=self.device)
209 # Prepare prefix block tables
210 max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
211 block_tables = _make_tensor_with_pad(
@@ -205,15 +213,16 @@ class ModelRunner:
213 max_len=max_prompt_block_table_len,
214 pad=0,
215 dtype=torch.int,
216 + device=self.device,
217 )
218 start_loc_tensor = torch.arange(0,
219 len(prompt_lens) * max_prompt_len,
220 max_prompt_len,
221 dtype=torch.long,
213 - device='cuda')
222 + device=self.device)
223 prompt_lens_tensor = torch.tensor(prompt_lens,
224 dtype=torch.long,
216 - device='cuda')
225 + device=self.device)
226
227 input_metadata = InputMetadata(
228 is_prompt=True,
@@ -305,20 +314,20 @@ class ModelRunner:
314 max_len=1,
315 pad=0,
316 dtype=torch.long,
308 - device="cuda")
317 + device=self.device)
318 input_positions = _make_tensor_with_pad(input_positions,
319 max_len=1,
320 pad=0,
321 dtype=torch.long,
313 - device="cuda")
322 + device=self.device)
323 slot_mapping = _make_tensor_with_pad(slot_mapping,
324 max_len=1,
325 pad=_PAD_SLOT_ID,
326 dtype=torch.long,
318 - device="cuda")
327 + device=self.device)
328 context_lens = torch.tensor(context_lens,
329 dtype=torch.int,
321 - device="cuda")
330 + device=self.device)
331
332 if use_captured_graph:
333 # The shape of graph_block_tables is
@@ -327,7 +336,7 @@ class ModelRunner:
336 for i, block_table in enumerate(block_tables):
337 if block_table:
338 input_block_tables[i, :len(block_table)] = block_table
330 - block_tables = torch.tensor(input_block_tables, device="cuda")
339 + block_tables = torch.tensor(input_block_tables, device=self.device)
340 else:
341 max_block_table_len = max(
342 len(block_table) for block_table in block_tables)
@@ -336,7 +345,7 @@ class ModelRunner:
345 max_len=max_block_table_len,
346 pad=0,
347 dtype=torch.int,
339 - device="cuda",
348 + device=self.device,
349 )
350
351 lora_index_mapping = [
@@ -355,7 +364,8 @@ class ModelRunner:
364 use_cuda_graph=use_captured_graph,
365 kv_cache_dtype=self.kv_cache_dtype,
366 )
358 - return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
367 + return (input_tokens, input_positions, input_metadata,
368 + lora_index_mapping, lora_prompt_mapping, lora_requests)
369
370 def _prepare_sample(
371 self,
@@ -410,9 +420,13 @@ class ModelRunner:
420
421 selected_token_indices = _async_h2d(selected_token_indices,
422 dtype=torch.long,
423 + target_device=self.device,
424 pin_memory=not self.in_wsl)
425 categorized_sample_indices = {
415 - t: _async_h2d(seq_ids, dtype=torch.int, pin_memory=not self.in_wsl)
426 + t: _async_h2d(seq_ids,
427 + dtype=torch.int,
428 + target_device=self.device,
429 + pin_memory=not self.in_wsl)
430 for t, seq_ids in categorized_sample_indices.items()
431 }
432
@@ -511,7 +525,8 @@ class ModelRunner:
525 perform_sampling=False,
526 )
527
514 - return input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping
528 + return (input_tokens, input_positions, input_metadata,
529 + sampling_metadata, lora_requests, lora_mapping)
530
531 @torch.inference_mode()
532 def execute_model(
@@ -519,8 +534,9 @@ class ModelRunner:
534 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
535 kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
536 ) -> Optional[SamplerOutput]:
522 - input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping = (
523 - self.prepare_input_tensors(seq_group_metadata_list))
537 + (input_tokens, input_positions, input_metadata, sampling_metadata,
538 + lora_requests,
539 + lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
540
541 if self.lora_config:
542 self.set_active_loras(lora_requests, lora_mapping)
@@ -789,14 +805,10 @@ def _make_tensor_with_pad(
805 max_len: int,
806 pad: int,
807 dtype: torch.dtype,
792 - device: Union[str, torch.device] = "cuda",
793 - pin_memory: bool = False,
808 + device: Optional[Union[str, torch.device]],
809 ) -> torch.Tensor:
810 padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
796 - return torch.tensor(padded_x,
797 - dtype=dtype,
798 - device=device,
799 - pin_memory=pin_memory and str(device) == "cpu")
811 + return torch.tensor(padded_x, dtype=dtype, device=device)
812
813
814 def _get_graph_batch_size(batch_size: int) -> int:
@@ -808,6 +820,11 @@ def _get_graph_batch_size(batch_size: int) -> int:
820 return (batch_size + 7) // 8 * 8
821
822
811 -def _async_h2d(data: list, dtype, pin_memory):
812 - t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory)
813 - return t.to(device="cuda", non_blocking=True)
823 +def _async_h2d(
824 + data: list,
825 + dtype: torch.dtype,
826 + target_device: Union[str, torch.device],
827 + pin_memory: bool,
828 +) -> torch.Tensor:
829 + t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
830 + return t.to(device=target_device, non_blocking=True)
vllm/worker/worker.py
+23 -17
@@ -6,8 +6,8 @@ from typing import Dict, List, Tuple, Set, Optional
6 import torch
7 import torch.distributed
8
9 -from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
10 - SchedulerConfig, LoRAConfig)
9 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10 + ParallelConfig, SchedulerConfig, LoRAConfig)
11 from vllm.model_executor import set_random_seed
12 from vllm.model_executor.parallel_utils.communication_op import (
13 broadcast_tensor_dict)
@@ -33,6 +33,7 @@ class Worker:
33 model_config: ModelConfig,
34 parallel_config: ParallelConfig,
35 scheduler_config: SchedulerConfig,
36 + device_config: DeviceConfig,
37 local_rank: int,
38 rank: int,
39 distributed_init_method: str,
@@ -43,6 +44,7 @@ class Worker:
44 self.model_config = model_config
45 self.parallel_config = parallel_config
46 self.scheduler_config = scheduler_config
47 + self.device_config = device_config
48 self.local_rank = local_rank
49 self.rank = rank
50 self.distributed_init_method = distributed_init_method
@@ -54,6 +56,7 @@ class Worker:
56 self.model_runner = ModelRunner(model_config,
57 parallel_config,
58 scheduler_config,
59 + device_config,
60 lora_config=self.lora_config,
61 kv_cache_dtype=kv_cache_dtype,
62 is_driver_worker=is_driver_worker)
@@ -65,21 +68,24 @@ class Worker:
68 self.gpu_cache = None
69
70 def init_model(self) -> None:
68 - # torch.distributed.all_reduce does not free the input tensor until
69 - # the synchronization point. This causes the memory usage to grow
70 - # as the number of all_reduce calls increases. This env var disables
71 - # this behavior.
72 - # Related issue:
73 - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
74 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
75 -
76 - # This env var set by Ray causes exceptions with graph building.
77 - os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
78 - self.device = torch.device(f"cuda:{self.local_rank}")
79 - torch.cuda.set_device(self.device)
80 -
81 - _check_if_gpu_supports_dtype(self.model_config.dtype)
82 -
71 + if self.device_config.device.type == "cuda":
72 + # torch.distributed.all_reduce does not free the input tensor until
73 + # the synchronization point. This causes the memory usage to grow
74 + # as the number of all_reduce calls increases. This env var disables
75 + # this behavior.
76 + # Related issue:
77 + # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
78 + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
79 +
80 + # This env var set by Ray causes exceptions with graph building.
81 + os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
82 + self.device = torch.device(f"cuda:{self.local_rank}")
83 + torch.cuda.set_device(self.device)
84 +
85 + _check_if_gpu_supports_dtype(self.model_config.dtype)
86 + else:
87 + raise RuntimeError(
88 + f"Not support device type: {self.device_config.device}")
89 # Initialize the distributed environment.
90 init_distributed_environment(self.parallel_config, self.rank,
91 self.distributed_init_method)