A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Remove hardcoded `device="cuda" ` to support more devices (#2503)
Co-authored-by: Jiang Li <jiang1.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
Feb 2, 2024 at 07:46 UTC
96b6f475dda40a0c7d557f73c36fe09c07be2e9c
32 files changed
+346
-295
benchmarks/benchmark_latency.py
+7
@@ -25,6 +25,7 @@ def main(args: argparse.Namespace):
25
dtype=args.dtype,
26
enforce_eager=args.enforce_eager,
27
kv_cache_dtype=args.kv_cache_dtype,
28
+ device=args.device,
29
)
30
31
sampling_params = SamplingParams(
@@ -135,5 +136,11 @@ if __name__ == '__main__':
136
default=None,
137
help=('path to save the pytorch profiler output. Can be visualized '
138
'with ui.perfetto.dev or Tensorboard.'))
139
+ parser.add_argument(
140
+ "--device",
141
+ type=str,
142
+ default="cuda",
143
+ choices=["cuda"],
144
+ help='device type for vLLM execution, supporting CUDA only currently.')
145
args = parser.parse_args()
146
main(args)
benchmarks/benchmark_throughput.py
+9
-1
@@ -72,6 +72,7 @@ def run_vllm(
72
max_model_len: Optional[int],
73
enforce_eager: bool,
74
kv_cache_dtype: str,
75
+ device: str,
76
) -> float:
77
from vllm import LLM, SamplingParams
78
llm = LLM(
@@ -85,6 +86,7 @@ def run_vllm(
86
max_model_len=max_model_len,
87
enforce_eager=enforce_eager,
88
kv_cache_dtype=kv_cache_dtype,
89
+ device=device,
90
)
91
92
# Add the requests to the engine.
@@ -209,7 +211,7 @@ def main(args: argparse.Namespace):
211
args.seed, args.n, args.use_beam_search,
212
args.trust_remote_code, args.dtype,
213
args.max_model_len, args.enforce_eager,
212
- args.kv_cache_dtype)
214
+ args.kv_cache_dtype, args.device)
215
elif args.backend == "hf":
216
assert args.tensor_parallel_size == 1
217
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -294,6 +296,12 @@ if __name__ == "__main__":
296
default="auto",
297
help=
298
'Data type for kv cache storage. If "auto", will use model data type.')
299
+ parser.add_argument(
300
+ "--device",
301
+ type=str,
302
+ default="cuda",
303
+ choices=["cuda"],
304
+ help='device type for vLLM execution, supporting CUDA only currently.')
305
args = parser.parse_args()
306
if args.tokenizer is None:
307
args.tokenizer = args.model
benchmarks/kernels/benchmark_paged_attention.py
+18
-9
@@ -25,18 +25,20 @@ def main(
25
dtype: torch.dtype,
26
seed: int,
27
do_profile: bool,
28
+ device: str = "cuda",
29
kv_cache_dtype: Optional[str] = None,
30
) -> None:
31
random.seed(seed)
32
torch.random.manual_seed(seed)
32
- torch.cuda.manual_seed(seed)
33
+ if torch.cuda.is_available():
34
+ torch.cuda.manual_seed(seed)
35
36
scale = float(1.0 / (head_size**0.5))
37
query = torch.empty(num_seqs,
38
num_query_heads,
39
head_size,
40
dtype=dtype,
39
- device="cuda")
41
+ device=device)
42
query.uniform_(-scale, scale)
43
44
assert num_query_heads % num_kv_heads == 0
@@ -44,11 +46,11 @@ def main(
46
if use_alibi:
47
alibi_slopes = torch.randn(num_query_heads,
48
dtype=torch.float,
47
- device="cuda")
49
+ device=device)
50
51
context_lens = [context_len for _ in range(num_seqs)]
52
max_context_len = max(context_lens)
51
- context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
53
+ context_lens = torch.tensor(context_lens, dtype=torch.int, device=device)
54
55
# Create the block tables.
56
max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
@@ -59,12 +61,17 @@ def main(
61
for _ in range(max_num_blocks_per_seq)
62
]
63
block_tables.append(block_table)
62
- block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
64
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device=device)
65
66
# Create the KV cache.
65
- key_caches, value_caches = create_kv_caches_with_random(
66
- NUM_BLOCKS, block_size, 1, num_kv_heads, head_size, kv_cache_dtype,
67
- dtype)
67
+ key_caches, value_caches = create_kv_caches_with_random(NUM_BLOCKS,
68
+ block_size,
69
+ 1,
70
+ num_kv_heads,
71
+ head_size,
72
+ kv_cache_dtype,
73
+ dtype,
74
+ device=device)
75
key_cache, value_cache = key_caches[0], value_caches[0]
76
77
# Prepare for the paged attention kernel.
@@ -84,7 +91,7 @@ def main(
91
)
92
max_logits = torch.empty_like(exp_sums)
93
87
- def run_benchmark(num_iters: int, profile: bool = False) -> float:
94
+ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float:
95
torch.cuda.synchronize()
96
if profile:
97
torch.cuda.cudart().cudaProfilerStart()
@@ -135,6 +142,7 @@ def main(
142
143
# Warmup.
144
print("Warming up...")
145
+ run_benchmark = run_cuda_benchmark
146
run_benchmark(num_iters=3, profile=False)
147
148
# Benchmark.
@@ -175,6 +183,7 @@ if __name__ == '__main__':
183
default="auto",
184
help=
185
'Data type for kv cache storage. If "auto", will use model data type.')
186
+ parser.add_argument("--device", type=str, choices=["cuda"], default="cuda")
187
args = parser.parse_args()
188
print(args)
189
tests/kernels/test_activation.py
+21
-16
@@ -7,26 +7,29 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
7
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
8
D = [512, 4096, 5120, 13824] # Arbitrary values for testing
9
SEEDS = [0]
10
-DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
10
+CUDA_DEVICES = [
11
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
12
+]
13
14
15
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
16
@pytest.mark.parametrize("d", D)
17
@pytest.mark.parametrize("dtype", DTYPES)
18
@pytest.mark.parametrize("seed", SEEDS)
17
-@pytest.mark.parametrize("device", DEVICES)
19
+@pytest.mark.parametrize("device", CUDA_DEVICES)
20
@torch.inference_mode()
21
def test_silu_and_mul(
22
num_tokens: int,
23
d: int,
24
dtype: torch.dtype,
25
seed: int,
24
- device: int,
26
+ device: str,
27
) -> None:
28
torch.random.manual_seed(seed)
27
- torch.cuda.manual_seed(seed)
28
- gpu_id = f"cuda:{device}"
29
- x = torch.randn(num_tokens, 2 * d, dtype=dtype, device=gpu_id)
29
+ if torch.cuda.is_available():
30
+ torch.cuda.manual_seed(seed)
31
+ torch.set_default_device(device)
32
+ x = torch.randn(num_tokens, 2 * d, dtype=dtype)
33
layer = SiluAndMul()
34
out = layer(x)
35
ref_out = layer._forward(x)
@@ -37,19 +40,20 @@ def test_silu_and_mul(
40
@pytest.mark.parametrize("d", D)
41
@pytest.mark.parametrize("dtype", DTYPES)
42
@pytest.mark.parametrize("seed", SEEDS)
40
-@pytest.mark.parametrize("device", DEVICES)
43
+@pytest.mark.parametrize("device", CUDA_DEVICES)
44
@torch.inference_mode()
45
def test_gelu_new(
46
num_tokens: int,
47
d: int,
48
dtype: torch.dtype,
49
seed: int,
47
- device: int,
50
+ device: str,
51
) -> None:
52
torch.random.manual_seed(seed)
50
- torch.cuda.manual_seed(seed)
51
- gpu_id = f"cuda:{device}"
52
- x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
53
+ if torch.cuda.is_available():
54
+ torch.cuda.manual_seed(seed)
55
+ torch.set_default_device(device)
56
+ x = torch.randn(num_tokens, d, dtype=dtype)
57
layer = NewGELU()
58
out = layer(x)
59
ref_out = layer._forward(x)
@@ -60,18 +64,19 @@ def test_gelu_new(
64
@pytest.mark.parametrize("d", D)
65
@pytest.mark.parametrize("dtype", DTYPES)
66
@pytest.mark.parametrize("seed", SEEDS)
63
-@pytest.mark.parametrize("device", DEVICES)
67
+@pytest.mark.parametrize("device", CUDA_DEVICES)
68
def test_gelu_fast(
69
num_tokens: int,
70
d: int,
71
dtype: torch.dtype,
72
seed: int,
69
- device: int,
73
+ device: str,
74
) -> None:
75
torch.random.manual_seed(seed)
72
- torch.cuda.manual_seed(seed)
73
- gpu_id = f"cuda:{device}"
74
- x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
76
+ if torch.cuda.is_available():
77
+ torch.cuda.manual_seed(seed)
78
+ torch.set_default_device(device)
79
+ x = torch.randn(num_tokens, d, dtype=dtype)
80
layer = FastGELU()
81
out = layer(x)
82
ref_out = layer._forward(x)
tests/kernels/test_attention.py
+23
-28
@@ -27,7 +27,9 @@ BLOCK_SIZES = [16, 32]
27
USE_ALIBI = [False, True]
28
KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
29
SEEDS = [0]
30
-DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
30
+CUDA_DEVICES = [
31
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
32
+]
33
34
35
def ref_masked_attention(
@@ -91,7 +93,7 @@ def ref_single_query_cached_kv_attention(
93
alibi_bias = None
94
if alibi_slopes is not None:
95
# Create the ALiBi bias used in the paged attention kernel.
94
- position_ids = torch.arange(context_len, device=query.device).int()
96
+ position_ids = torch.arange(context_len).int()
97
alibi_bias = (position_ids - context_len + 1).float()
98
alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
99
1, 1, -1)
@@ -110,7 +112,7 @@ def ref_single_query_cached_kv_attention(
112
@pytest.mark.parametrize("dtype", DTYPES)
113
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
114
@pytest.mark.parametrize("seed", SEEDS)
113
-@pytest.mark.parametrize("device", DEVICES)
115
+@pytest.mark.parametrize("device", CUDA_DEVICES)
116
def test_paged_attention(
117
kv_cache_factory,
118
version: str,
@@ -122,33 +124,28 @@ def test_paged_attention(
124
dtype: torch.dtype,
125
kv_cache_dtype: str,
126
seed: int,
125
- device: int,
127
+ device: str,
128
) -> None:
129
random.seed(seed)
130
torch.random.manual_seed(seed)
129
- torch.cuda.manual_seed(seed)
130
- gpu_id = f"cuda:{device}"
131
+ if torch.cuda.is_available():
132
+ torch.cuda.manual_seed(seed)
133
+ torch.set_default_device(device)
134
scale = float(1.0 / (head_size**0.5))
135
num_query_heads, num_kv_heads = num_heads
133
- query = torch.empty(num_seqs,
134
- num_query_heads,
135
- head_size,
136
- dtype=dtype,
137
- device=gpu_id)
136
+ query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
137
query.uniform_(-scale, scale)
138
139
assert num_query_heads % num_kv_heads == 0
140
num_queries_per_kv = num_query_heads // num_kv_heads
141
alibi_slopes = None
142
if use_alibi:
144
- alibi_slopes = torch.randn(num_query_heads,
145
- dtype=torch.float,
146
- device=gpu_id)
143
+ alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
144
145
context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
146
context_lens[-1] = MAX_SEQ_LEN
147
max_context_len = max(context_lens)
151
- context_lens = torch.tensor(context_lens, dtype=torch.int, device=gpu_id)
148
+ context_lens = torch.tensor(context_lens, dtype=torch.int)
149
150
# Create the block tables.
151
max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
@@ -159,13 +156,13 @@ def test_paged_attention(
156
for _ in range(max_num_blocks_per_seq)
157
]
158
block_tables.append(block_table)
162
- block_tables = torch.tensor(block_tables, dtype=torch.int, device=gpu_id)
159
+ block_tables = torch.tensor(block_tables, dtype=torch.int)
160
161
# Create the KV caches.
162
key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
163
num_kv_heads, head_size,
164
kv_cache_dtype, dtype, seed,
168
- gpu_id)
165
+ device)
166
key_cache, value_cache = key_caches[0], value_caches[0]
167
168
# Call the paged attention kernel.
@@ -193,12 +190,10 @@ def test_paged_attention(
190
tmp_output = torch.empty(
191
size=(num_seqs, num_heads, num_partitions, head_size),
192
dtype=output.dtype,
196
- device=output.device,
193
)
194
exp_sums = torch.empty(
195
size=(num_seqs, num_heads, num_partitions),
196
dtype=torch.float32,
201
- device=output.device,
197
)
198
max_logits = torch.empty_like(exp_sums)
199
ops.paged_attention_v2(
@@ -229,14 +224,14 @@ def test_paged_attention(
224
block_size, x)
225
dequantized_key_cache = torch.empty(size=key_cache_shape,
226
dtype=dtype,
232
- device=gpu_id)
227
+ device=device)
228
cache_ops.convert_fp8_e5m2(key_cache, dequantized_key_cache)
229
key_cache = dequantized_key_cache
230
231
value_cache_shape = value_cache.shape
232
dequantized_value_cache = torch.empty(size=value_cache_shape,
233
dtype=dtype,
239
- device=gpu_id)
234
+ device=device)
235
cache_ops.convert_fp8_e5m2(value_cache, dequantized_value_cache)
236
value_cache = dequantized_value_cache
237
@@ -283,7 +278,7 @@ def ref_multi_query_kv_attention(
278
attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
279
diagonal=1)
280
attn_mask = attn_mask * torch.finfo(dtype).min
286
- attn_mask = attn_mask.to(dtype=dtype, device=query.device)
281
+ attn_mask = attn_mask.to(dtype=dtype)
282
283
ref_output = ref_masked_attention(
284
query[start_idx:end_idx],
@@ -303,7 +298,7 @@ def ref_multi_query_kv_attention(
298
@pytest.mark.parametrize("head_size", HEAD_SIZES)
299
@pytest.mark.parametrize("dtype", DTYPES)
300
@pytest.mark.parametrize("seed", SEEDS)
306
-@pytest.mark.parametrize("device", DEVICES)
301
+@pytest.mark.parametrize("device", CUDA_DEVICES)
302
@torch.inference_mode()
303
def test_multi_query_kv_attention(
304
num_seqs: int,
@@ -311,12 +306,13 @@ def test_multi_query_kv_attention(
306
head_size: int,
307
dtype: torch.dtype,
308
seed: int,
314
- device: int,
309
+ device: str,
310
) -> None:
311
random.seed(seed)
312
torch.random.manual_seed(seed)
318
- torch.cuda.manual_seed(seed)
319
- gpu_id = f"cuda:{device}"
313
+ if torch.cuda.is_available():
314
+ torch.cuda.manual_seed(seed)
315
+ torch.set_default_device(device)
316
# MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
317
# As the xformers library is already tested with its own tests, we can use
318
# a smaller MAX_SEQ_LEN here.
@@ -329,8 +325,7 @@ def test_multi_query_kv_attention(
325
qkv = torch.empty(num_tokens,
326
num_query_heads + 2 * num_kv_heads,
327
head_size,
332
- dtype=dtype,
333
- device=gpu_id)
328
+ dtype=dtype)
329
qkv.uniform_(-scale, scale)
330
query, key, value = qkv.split(
331
[num_query_heads, num_kv_heads, num_kv_heads], dim=1)
tests/kernels/test_cache.py
+21
-21
@@ -17,7 +17,9 @@ BLOCK_SIZES = [8, 16, 32]
17
NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
18
NUM_MAPPINGS = [256] # Arbitrary values for testing
19
SEEDS = [0]
20
-DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
20
+CUDA_DEVICES = [
21
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
22
+]
23
KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
24
25
@@ -29,7 +31,7 @@ KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
31
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
32
@pytest.mark.parametrize("dtype", DTYPES)
33
@pytest.mark.parametrize("seed", SEEDS)
32
-@pytest.mark.parametrize("device", DEVICES)
34
+@pytest.mark.parametrize("device", CUDA_DEVICES)
35
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
36
@torch.inference_mode()
37
def test_copy_blocks(
@@ -42,13 +44,14 @@ def test_copy_blocks(
44
num_blocks: int,
45
dtype: torch.dtype,
46
seed: int,
45
- device: int,
47
kv_cache_dtype: str,
48
+ device: str,
49
) -> None:
50
random.seed(seed)
51
torch.random.manual_seed(seed)
50
- torch.cuda.manual_seed(seed)
51
- gpu_id = f"cuda:{device}"
52
+ if torch.cuda.is_available():
53
+ torch.cuda.manual_seed(seed)
54
+ torch.set_default_device(device)
55
# Generate random block mappings where each source block is mapped to two
56
# destination blocks.
57
assert 2 * num_mappings <= num_blocks
@@ -66,7 +69,7 @@ def test_copy_blocks(
69
key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
70
num_layers, num_heads,
71
head_size, kv_cache_dtype,
69
- dtype, seed, gpu_id)
72
+ dtype, seed, device)
73
74
# Clone the KV caches.
75
cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
@@ -98,7 +101,7 @@ def test_copy_blocks(
101
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
102
@pytest.mark.parametrize("dtype", DTYPES)
103
@pytest.mark.parametrize("seed", SEEDS)
101
-@pytest.mark.parametrize("device", DEVICES)
104
+@pytest.mark.parametrize("device", CUDA_DEVICES)
105
@torch.inference_mode()
106
def test_reshape_and_cache(
107
kv_cache_factory,
@@ -109,29 +112,25 @@ def test_reshape_and_cache(
112
num_blocks: int,
113
dtype: torch.dtype,
114
seed: int,
112
- device: int,
115
+ device: str,
116
) -> None:
117
random.seed(seed)
118
torch.random.manual_seed(seed)
116
- torch.cuda.manual_seed(seed)
117
- gpu_id = f"cuda:{device}"
119
+ if torch.cuda.is_available():
120
+ torch.cuda.manual_seed(seed)
121
+ torch.set_default_device(device)
122
# Create a random slot mapping.
123
num_slots = block_size * num_blocks
124
slot_mapping = random.sample(range(num_slots), num_tokens)
121
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=gpu_id)
122
-
123
- qkv = torch.randn(num_tokens,
124
- 3,
125
- num_heads,
126
- head_size,
127
- dtype=dtype,
128
- device=gpu_id)
125
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.long)
126
+
127
+ qkv = torch.randn(num_tokens, 3, num_heads, head_size, dtype=dtype)
128
_, key, value = qkv.unbind(dim=1)
129
130
# Create the KV caches.
131
key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
132
num_heads, head_size, dtype,
134
- None, seed, gpu_id)
133
+ None, seed, device)
134
key_cache, value_cache = key_caches[0], value_caches[0]
135
136
# Clone the KV caches.
@@ -166,7 +165,7 @@ def test_reshape_and_cache(
165
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
166
@pytest.mark.parametrize("dtype", DTYPES)
167
@pytest.mark.parametrize("seed", SEEDS)
169
-@pytest.mark.parametrize("device", DEVICES)
168
+@pytest.mark.parametrize("device", CUDA_DEVICES)
169
@torch.inference_mode()
170
def test_swap_blocks(
171
kv_cache_factory,
@@ -182,7 +181,8 @@ def test_swap_blocks(
181
) -> None:
182
random.seed(seed)
183
torch.random.manual_seed(seed)
185
- torch.cuda.manual_seed(seed)
184
+ if torch.cuda.is_available():
185
+ torch.cuda.manual_seed(seed)
186
src_device = f"{direction[0]}:{device}" if direction[
187
0] == "cuda" else direction[0]
188
dst_device = f"{direction[1]}:{device}" if direction[
tests/kernels/test_layernorm.py
+10
-7
@@ -8,7 +8,9 @@ NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8
HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
9
ADD_RESIDUAL = [False, True]
10
SEEDS = [0]
11
-DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
11
+CUDA_DEVICES = [
12
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
13
+]
14
15
16
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -16,7 +18,7 @@ DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
18
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
19
@pytest.mark.parametrize("dtype", DTYPES)
20
@pytest.mark.parametrize("seed", SEEDS)
19
-@pytest.mark.parametrize("device", DEVICES)
21
+@pytest.mark.parametrize("device", CUDA_DEVICES)
22
@torch.inference_mode()
23
def test_rms_norm(
24
num_tokens: int,
@@ -24,15 +26,16 @@ def test_rms_norm(
26
add_residual: bool,
27
dtype: torch.dtype,
28
seed: int,
27
- device: int,
29
+ device: str,
30
) -> None:
31
torch.random.manual_seed(seed)
30
- torch.cuda.manual_seed(seed)
31
- gpu_id = f"cuda:{device}"
32
- layer = RMSNorm(hidden_size).to(dtype=dtype, device=gpu_id)
32
+ if torch.cuda.is_available():
33
+ torch.cuda.manual_seed(seed)
34
+ torch.set_default_device(device)
35
+ layer = RMSNorm(hidden_size).to(dtype=dtype)
36
layer.weight.data.normal_(mean=1.0, std=0.1)
37
scale = 1 / (2 * hidden_size)
35
- x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=gpu_id)
38
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype)
39
x *= scale
40
residual = torch.randn_like(x) * scale if add_residual else None
41
tests/kernels/test_pos_encoding.py
+11
-11
@@ -13,7 +13,9 @@ NUM_HEADS = [7, 17] # Arbitrary values for testing
13
BATCH_SIZES = [1, 5] # Arbitrary values for testing
14
SEQ_LENS = [11, 8192] # Arbitrary values for testing
15
SEEDS = [0]
16
-DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
16
+CUDA_DEVICES = [
17
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
18
+]
19
20
21
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -24,7 +26,7 @@ DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
26
@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
27
@pytest.mark.parametrize("dtype", DTYPES)
28
@pytest.mark.parametrize("seed", SEEDS)
27
-@pytest.mark.parametrize("device", DEVICES)
29
+@pytest.mark.parametrize("device", CUDA_DEVICES)
30
@torch.inference_mode()
31
def test_rotary_embedding(
32
is_neox_style: bool,
@@ -35,28 +37,26 @@ def test_rotary_embedding(
37
rotary_dim: Optional[int],
38
dtype: torch.dtype,
39
seed: int,
38
- device: int,
40
+ device: str,
41
max_position: int = 8192,
42
base: int = 10000,
43
) -> None:
44
if rotary_dim is None:
45
rotary_dim = head_size
46
torch.random.manual_seed(seed)
45
- torch.cuda.manual_seed(seed)
46
- gpu_id = f"cuda:{device}"
47
+ if torch.cuda.is_available():
48
+ torch.cuda.manual_seed(seed)
49
+ torch.set_default_device(device)
50
if rotary_dim is None:
51
rotary_dim = head_size
52
rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
50
- rope = rope.to(dtype=dtype, device=gpu_id)
53
+ rope = rope.to(dtype=dtype)
54
52
- positions = torch.randint(0,
53
- max_position, (batch_size, seq_len),
54
- device=gpu_id)
55
+ positions = torch.randint(0, max_position, (batch_size, seq_len))
56
query = torch.randn(batch_size,
57
seq_len,
58
num_heads * head_size,
58
- dtype=dtype,
59
- device=gpu_id)
59
+ dtype=dtype)
60
key = torch.randn_like(query)
61
62
# NOTE(woosuk): The reference implementation should be executed first
tests/kernels/test_prefix_prefill.py
+20
-37
@@ -11,19 +11,27 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
11
NUM_HEADS = [12]
12
HEAD_SIZES = [128]
13
DTYPES = [torch.float16]
14
+CUDA_DEVICES = [
15
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
16
+]
17
18
19
@pytest.mark.parametrize("num_heads", NUM_HEADS)
20
@pytest.mark.parametrize("head_size", HEAD_SIZES)
21
@pytest.mark.parametrize("dtype", DTYPES)
22
+@pytest.mark.parametrize("device", CUDA_DEVICES)
23
@torch.inference_mode()
24
def test_contexted_kv_attention(
25
num_heads: int,
26
head_size: int,
27
dtype: torch.dtype,
28
+ device: str,
29
) -> None:
30
random.seed(0)
31
torch.manual_seed(0)
32
+ if torch.cuda.is_available():
33
+ torch.cuda.manual_seed(0)
34
+ torch.set_default_device(device)
35
MAX_SEQ_LEN = 1024
36
MAX_CTX_LEN = 1024
37
BS = 10
@@ -35,24 +43,11 @@ def test_contexted_kv_attention(
43
seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
44
45
num_tokens = sum(subquery_lens)
38
- query = torch.empty(num_tokens,
39
- num_heads,
40
- head_size,
41
- dtype=dtype,
42
- device='cuda')
46
+ query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
47
query.uniform_(-1e-3, 1e-3)
44
- output = torch.empty(num_tokens,
45
- num_heads,
46
- head_size,
47
- dtype=dtype,
48
- device='cuda')
48
+ output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
49
50
- kv = torch.empty(sum(seq_lens),
51
- 2,
52
- num_heads,
53
- head_size,
54
- dtype=dtype,
55
- device='cuda')
50
+ kv = torch.empty(sum(seq_lens), 2, num_heads, head_size, dtype=dtype)
51
kv.uniform_(-1e-3, 1e-3)
52
key, value = kv.unbind(dim=1)
53
@@ -60,39 +55,27 @@ def test_contexted_kv_attention(
55
block_size,
56
num_heads,
57
head_size,
63
- dtype=dtype,
64
- device='cuda')
58
+ dtype=dtype)
59
v_cache = torch.zeros(cache_size,
60
block_size,
61
num_heads,
62
head_size,
69
- dtype=dtype,
70
- device='cuda')
71
- k = torch.zeros(sum(subquery_lens),
72
- num_heads,
73
- head_size,
74
- dtype=dtype,
75
- device='cuda')
76
- v = torch.zeros(sum(subquery_lens),
77
- num_heads,
78
- head_size,
79
- dtype=dtype,
80
- device='cuda')
81
- values = torch.arange(0, cache_size, dtype=torch.long, device='cuda')
63
+ dtype=dtype)
64
+ k = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
65
+ v = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
66
+ values = torch.arange(0, cache_size, dtype=torch.long)
67
values = values[torch.randperm(cache_size)]
68
block_table = values[:BS * max_block_per_request].view(
69
BS, max_block_per_request)
85
- b_seq_len = torch.tensor(seq_lens, dtype=torch.long, device='cuda')
86
- b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long, device='cuda')
70
+ b_seq_len = torch.tensor(seq_lens, dtype=torch.long)
71
+ b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
72
b_start_loc = torch.cumsum(torch.tensor([0] + subquery_lens[:-1],
88
- dtype=torch.long,
89
- device='cuda'),
73
+ dtype=torch.long),
74
dim=0)
75
max_input_len = MAX_SEQ_LEN
76
# copy kv to cache
77
b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
94
- dtype=torch.long,
95
- device='cuda'),
78
+ dtype=torch.long),
79
dim=0)
80
for i in range(BS):
81
for j in range(subquery_lens[i]):
tests/lora/conftest.py
+2
-2
@@ -126,8 +126,8 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
126
cleanup()
127
get_model_old = get_model
128
129
- def get_model_patched(model_config, lora_config=None):
130
- return get_model_old(model_config,
129
+ def get_model_patched(model_config, device_config, lora_config=None):
130
+ return get_model_old(model_config, device_config,
131
LoRAConfig(max_loras=4, max_lora_rank=8))
132
133
with patch("vllm.worker.model_runner.get_model", get_model_patched):
tests/lora/test_layers.py
+22
-14
@@ -34,6 +34,9 @@ TOLERANCES = {
34
torch.float32: (5e-3, 5e-3),
35
torch.bfloat16: (3e-2, 2e-2),
36
}
37
+CUDA_DEVICES = [
38
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
39
+]
40
41
42
def get_random_id_to_index(num_loras: int,
@@ -151,14 +154,10 @@ def create_random_inputs(
154
for _ in range(num_inputs):
155
if input_type == torch.int:
156
inputs.append(
154
- torch.randint(low=int(low),
155
- high=int(high),
156
- size=input_size,
157
- device="cuda"))
157
+ torch.randint(low=int(low), high=int(high), size=input_size))
158
else:
159
inputs.append(
160
- torch.rand(size=input_size, dtype=input_type, device="cuda") *
161
- high + low)
160
+ torch.rand(size=input_size, dtype=input_type) * high + low)
161
162
lora_id = random.choice(active_lora_ids)
163
index_mapping += [lora_id] * input_size[0]
@@ -169,8 +168,10 @@ def create_random_inputs(
168
169
@torch.inference_mode()
170
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
172
-def test_embeddings(dist_init, num_loras) -> None:
171
+@pytest.mark.parametrize("device", CUDA_DEVICES)
172
+def test_embeddings(dist_init, num_loras, device) -> None:
173
174
+ torch.set_default_device(device)
175
max_loras = 8
176
lora_config = LoRAConfig(max_loras=max_loras,
177
max_lora_rank=8,
@@ -259,8 +260,10 @@ def test_embeddings(dist_init, num_loras) -> None:
260
@torch.inference_mode()
261
# @pytest.mark.skip(reason="Fails when loras are in any slot other than the first.")
262
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
262
-def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
263
+@pytest.mark.parametrize("device", CUDA_DEVICES)
264
+def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
265
266
+ torch.set_default_device(device)
267
max_loras = 8
268
lora_config = LoRAConfig(max_loras=max_loras,
269
max_lora_rank=8,
@@ -305,8 +308,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
308
309
# Add empty embeddings_tensors for unoccupied lora slots.
310
for _ in range(max_loras - len(embeddings_tensors)):
308
- embeddings_tensors.append(
309
- torch.zeros(embeddings_tensors[0].shape, device="cuda"))
311
+ embeddings_tensors.append(torch.zeros(embeddings_tensors[0].shape))
312
313
inputs, index_mapping, prompt_mapping = create_random_inputs(
314
active_lora_ids=list(lora_dict.keys()),
@@ -388,8 +390,10 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
390
391
@torch.inference_mode()
392
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
391
-def test_lm_head_sampler(dist_init, num_loras) -> None:
393
+@pytest.mark.parametrize("device", CUDA_DEVICES)
394
+def test_lm_head_sampler(dist_init, num_loras, device) -> None:
395
396
+ torch.set_default_device(device)
397
max_loras = 8
398
lora_config = LoRAConfig(max_loras=max_loras,
399
max_lora_rank=8,
@@ -432,7 +436,7 @@ def test_lm_head_sampler(dist_init, num_loras) -> None:
436
)
437
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
438
435
- input_ = torch.rand(20, 1024, device="cuda")
439
+ input_ = torch.rand(20, 1024)
440
mapping_info = convert_mapping(
441
lora_mapping,
442
id_to_index,
@@ -500,8 +504,10 @@ def test_lm_head_sampler(dist_init, num_loras) -> None:
504
@torch.inference_mode()
505
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
506
@pytest.mark.parametrize("orientation", ["row", "column"])
503
-def test_linear_parallel(dist_init, num_loras, orientation) -> None:
507
+@pytest.mark.parametrize("device", CUDA_DEVICES)
508
+def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
509
510
+ torch.set_default_device(device)
511
max_loras = 8
512
lora_config = LoRAConfig(max_loras=max_loras,
513
max_lora_rank=8,
@@ -597,8 +603,10 @@ def test_linear_parallel(dist_init, num_loras, orientation) -> None:
603
@torch.inference_mode()
604
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
605
@pytest.mark.parametrize("repeats", [2, 3])
600
-def test_column_parallel_packed(dist_init, num_loras, repeats) -> None:
606
+@pytest.mark.parametrize("device", CUDA_DEVICES)
607
+def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
608
609
+ torch.set_default_device(device)
610
max_loras = 8
611
lora_config = LoRAConfig(max_loras=max_loras,
612
max_lora_rank=8,
tests/lora/test_worker.py
+3
-1
@@ -5,7 +5,8 @@ from unittest.mock import patch
5
6
from vllm.lora.models import LoRAMapping
7
from vllm.lora.request import LoRARequest
8
-from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig
8
+from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
9
+ DeviceConfig, LoRAConfig)
10
from vllm.worker.worker import Worker
11
12
@@ -25,6 +26,7 @@ def test_worker_apply_lora(sql_lora_files):
26
),
27
parallel_config=ParallelConfig(1, 1, False),
28
scheduler_config=SchedulerConfig(32, 32, 32, 256),
29
+ device_config=DeviceConfig("cuda"),
30
local_rank=0,
31
rank=0,
32
lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
tests/samplers/test_rejection_sampler.py
+27
-37
@@ -9,6 +9,10 @@ from vllm.model_executor.utils import set_random_seed
9
10
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
12
+CUDA_DEVICES = [
13
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
14
+]
15
+
16
17
def mock_causal_accepted_tensor(
18
k: int, last_accepted_indices: torch.Tensor) -> torch.Tensor:
@@ -39,11 +43,14 @@ def mock_causal_accepted_tensor(
43
@pytest.mark.parametrize(
44
"which_tokens_accepted",
45
["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
46
+@pytest.mark.parametrize("device", CUDA_DEVICES)
47
@torch.inference_mode()
43
-def test_correct_output_format(which_tokens_accepted: str, seed: int):
48
+def test_correct_output_format(which_tokens_accepted: str, seed: int,
49
+ device: str):
50
"""Verify the output has correct format given predetermined accepted matrix.
51
"""
52
set_random_seed(seed)
53
+ torch.set_default_device(device)
54
55
batch_size = 10
56
k = 5
@@ -66,18 +73,15 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int):
73
recovered_token_ids = torch.randint(low=0,
74
high=vocab_size,
75
size=(batch_size, k),
69
- dtype=torch.int64,
70
- device="cuda")
76
+ dtype=torch.int64)
77
draft_token_ids = torch.randint(low=0,
78
high=vocab_size,
79
size=(batch_size, k),
74
- dtype=torch.int64,
75
- device="cuda")
80
+ dtype=torch.int64)
81
bonus_token_ids = torch.randint(low=0,
82
high=vocab_size,
83
size=(batch_size, 1),
79
- dtype=torch.int64,
80
- device="cuda")
84
+ dtype=torch.int64)
85
86
rejection_sampler = RejectionSampler()
87
rejection_sampler.init_gpu_tensors(rank=0)
@@ -120,31 +124,24 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int):
124
@pytest.mark.parametrize("k", list(range(1, 6)))
125
@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
126
@pytest.mark.parametrize("batch_size", list(range(1, 32)))
127
+@pytest.mark.parametrize("device", CUDA_DEVICES)
128
@torch.inference_mode()
124
-def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int):
129
+def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
130
+ device: str):
131
+ torch.set_default_device(device)
132
rejection_sampler = RejectionSampler()
133
rejection_sampler.init_gpu_tensors(rank=0)
134
128
- draft_probs = torch.rand(batch_size,
129
- k,
130
- vocab_size,
131
- dtype=torch.float32,
132
- device="cuda")
133
- target_probs = torch.rand(batch_size,
134
- k,
135
- vocab_size,
136
- dtype=torch.float32,
137
- device="cuda")
135
+ draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
136
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
137
bonus_token_ids = torch.randint(low=0,
138
high=vocab_size,
139
size=(batch_size, 1),
141
- dtype=torch.int64,
142
- device="cuda")
140
+ dtype=torch.int64)
141
draft_token_ids = torch.randint(low=0,
142
high=vocab_size,
143
size=(batch_size, k),
146
- dtype=torch.int64,
147
- device="cuda")
144
+ dtype=torch.int64)
145
146
rejection_sampler(target_probs, bonus_token_ids, draft_probs,
147
draft_token_ids)
@@ -153,36 +150,28 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int):
150
@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
151
@pytest.mark.parametrize("which_token_ids",
152
["bonus_token_ids", "draft_token_ids"])
153
+@pytest.mark.parametrize("device", CUDA_DEVICES)
154
@torch.inference_mode()
155
def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
158
- which_token_ids: str):
156
+ which_token_ids: str, device: str):
157
k = 3
158
batch_size = 5
159
vocab_size = 30_000
160
+ torch.set_default_device(device)
161
162
rejection_sampler = RejectionSampler(strict_mode=True)
163
rejection_sampler.init_gpu_tensors(rank=0)
164
166
- draft_probs = torch.rand(batch_size,
167
- k,
168
- vocab_size,
169
- dtype=torch.float32,
170
- device="cuda")
171
- target_probs = torch.rand(batch_size,
172
- k,
173
- vocab_size,
174
- dtype=torch.float32,
175
- device="cuda")
165
+ draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
166
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
167
bonus_token_ids = torch.randint(low=0,
168
high=vocab_size,
169
size=(batch_size, 1),
179
- dtype=torch.int64,
180
- device="cuda")
170
+ dtype=torch.int64)
171
draft_token_ids = torch.randint(low=0,
172
high=vocab_size,
173
size=(batch_size, k),
184
- dtype=torch.int64,
185
- device="cuda")
174
+ dtype=torch.int64)
175
176
oob_token_ids = None
177
if which_token_ids == "bonus_token_ids":
@@ -237,6 +226,7 @@ def test_rejection_sampling_approximates_target_distribution(
226
probabilities are exactly equal. Rejection sampling should
227
still work without any NaNs or exceptions.
228
"""
229
+ torch.set_default_device("cpu")
230
set_random_seed(seed)
231
232
helper = _CorrectnessTestHelper(
tests/samplers/test_sampler.py
+24
-13
@@ -31,24 +31,26 @@ def _prepare_test(
31
batch_size: int
32
) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
33
vocab_size = 32000
34
- input_tensor = torch.rand((batch_size, 1024),
35
- device="cuda",
36
- dtype=torch.float16)
34
+ input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
35
fake_logits = torch.full((batch_size, vocab_size),
36
1e-2,
39
- device=input_tensor.device,
37
dtype=input_tensor.dtype)
38
sampler = MockLogitsSampler(32000, fake_logits)
42
- model_runner = ModelRunner(None, None, None, None)
39
+ model_runner = ModelRunner(None, None, None, None, None)
40
return input_tensor, fake_logits, sampler, model_runner
41
42
43
RANDOM_SEEDS = list(range(128))
44
+CUDA_DEVICES = [
45
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
46
+]
47
48
49
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
50
-def test_sampler_all_greedy(seed: int):
50
+@pytest.mark.parametrize("device", CUDA_DEVICES)
51
+def test_sampler_all_greedy(seed: int, device: str):
52
set_random_seed(seed)
53
+ torch.set_default_device(device)
54
batch_size = random.randint(1, 256)
55
input_tensor, fake_logits, sampler, model_runner = _prepare_test(
56
batch_size)
@@ -81,8 +83,10 @@ def test_sampler_all_greedy(seed: int):
83
84
85
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
84
-def test_sampler_all_random(seed: int):
86
+@pytest.mark.parametrize("device", CUDA_DEVICES)
87
+def test_sampler_all_random(seed: int, device: str):
88
set_random_seed(seed)
89
+ torch.set_default_device(device)
90
batch_size = random.randint(1, 256)
91
input_tensor, fake_logits, sampler, model_runner = _prepare_test(
92
batch_size)
@@ -120,8 +124,10 @@ def test_sampler_all_random(seed: int):
124
125
126
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
123
-def test_sampler_all_beam(seed: int):
127
+@pytest.mark.parametrize("device", CUDA_DEVICES)
128
+def test_sampler_all_beam(seed: int, device: str):
129
set_random_seed(seed)
130
+ torch.set_default_device(device)
131
batch_size = random.randint(1, 256)
132
input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
133
@@ -156,8 +162,10 @@ def test_sampler_all_beam(seed: int):
162
163
164
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
159
-def test_sampler_mixed(seed: int):
165
+@pytest.mark.parametrize("device", CUDA_DEVICES)
166
+def test_sampler_mixed(seed: int, device: str):
167
set_random_seed(seed)
168
+ torch.set_default_device(device)
169
batch_size = random.randint(1, 256)
170
input_tensor, fake_logits, sampler, model_runner = _prepare_test(
171
batch_size)
@@ -212,8 +220,10 @@ def test_sampler_mixed(seed: int):
220
221
222
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
215
-def test_sampler_logits_processors(seed: int):
223
+@pytest.mark.parametrize("device", CUDA_DEVICES)
224
+def test_sampler_logits_processors(seed: int, device: str):
225
set_random_seed(seed)
226
+ torch.set_default_device(device)
227
batch_size = random.randint(1, 256)
228
input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
229
@@ -252,14 +262,15 @@ def test_sampler_logits_processors(seed: int):
262
263
264
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
255
-def test_sampler_top_k_top_p(seed: int):
265
+@pytest.mark.parametrize("device", CUDA_DEVICES)
266
+def test_sampler_top_k_top_p(seed: int, device: str):
267
set_random_seed(seed)
268
batch_size = random.randint(1, 256)
269
top_k = random.randint(100, 500)
270
top_p = random.random() * 0.1
271
vocab_size = 32000
272
input_tensor = torch.rand((batch_size, 1024),
262
- device="cuda",
273
+ device=device,
274
dtype=torch.float16)
275
fake_logits = torch.normal(0,
276
5,
@@ -267,7 +278,7 @@ def test_sampler_top_k_top_p(seed: int):
278
device=input_tensor.device,
279
dtype=input_tensor.dtype)
280
sampler = MockLogitsSampler(32000, fake_logits)
270
- model_runner = ModelRunner(None, None, None, None)
281
+ model_runner = ModelRunner(None, None, None, None, None)
282
283
generation_model = GenerationMixin()
284
generation_config = GenerationConfig(top_k=top_k,
tests/worker/spec_decode/utils.py
+2
-1
@@ -84,7 +84,7 @@ def create_worker(cls: type,
84
)
85
86
(model_config, cache_config, parallel_config, scheduler_config,
87
- _) = engine_args.create_engine_configs()
87
+ device_config, _) = engine_args.create_engine_configs()
88
89
distributed_init_method = get_distributed_init_method(
90
get_ip(), get_open_port())
@@ -93,6 +93,7 @@ def create_worker(cls: type,
93
model_config=model_config,
94
parallel_config=parallel_config,
95
scheduler_config=scheduler_config,
96
+ device_config=device_config,
97
local_rank=0,
98
rank=0,
99
distributed_init_method=distributed_init_method,
tests/worker/test_model_runner.py
+1
-1
@@ -6,7 +6,7 @@ from vllm.worker.model_runner import ModelRunner
6
7
8
def test_prepare_prompt():
9
- model_runner = ModelRunner(None, None, None, None)
9
+ model_runner = ModelRunner(None, None, None, None, None)
10
model_runner.set_block_size(16)
11
12
batch_size = random.randint(1, 256)
vllm/config.py
+6
@@ -444,6 +444,12 @@ class SchedulerConfig:
444
f"({self.max_num_seqs}).")
445
446
447
+class DeviceConfig:
448
+
449
+ def __init__(self, device: str = "cuda") -> None:
450
+ self.device = torch.device(device)
451
+
452
+
453
@dataclass
454
class LoRAConfig:
455
max_lora_rank: int
vllm/engine/arg_utils.py
+18
-7
@@ -3,8 +3,8 @@ import dataclasses
3
from dataclasses import dataclass
4
from typing import Optional, Tuple
5
6
-from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
- SchedulerConfig, LoRAConfig)
6
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
7
+ ParallelConfig, SchedulerConfig, LoRAConfig)
8
9
10
@dataclass
@@ -43,6 +43,7 @@ class EngineArgs:
43
lora_extra_vocab_size: int = 256
44
lora_dtype = 'auto'
45
max_cpu_loras: Optional[int] = None
46
+ device: str = 'cuda'
47
48
def __post_init__(self):
49
if self.tokenizer is None:
@@ -127,13 +128,13 @@ class EngineArgs:
128
'--kv-cache-dtype',
129
type=str,
130
choices=['auto', 'fp8_e5m2'],
130
- default='auto',
131
+ default=EngineArgs.kv_cache_dtype,
132
help='Data type for kv cache storage. If "auto", will use model '
133
'data type. Note FP8 is not supported when cuda version is '
134
'lower than 11.8.')
135
parser.add_argument('--max-model-len',
136
type=int,
136
- default=None,
137
+ default=EngineArgs.max_model_len,
138
help='model context length. If unspecified, '
139
'will be automatically derived from the model.')
140
# Parallel arguments
@@ -154,6 +155,7 @@ class EngineArgs:
155
parser.add_argument(
156
'--max-parallel-loading-workers',
157
type=int,
158
+ default=EngineArgs.max_parallel_loading_workers,
159
help='load model sequentially in multiple batches, '
160
'to avoid RAM OOM when using tensor '
161
'parallel and large models')
@@ -200,7 +202,7 @@ class EngineArgs:
202
'-q',
203
type=str,
204
choices=['awq', 'gptq', 'squeezellm', None],
203
- default=None,
205
+ default=EngineArgs.quantization,
206
help='Method used to quantize the weights. If '
207
'None, we first check the `quantization_config` '
208
'attribute in the model config file. If that is '
@@ -255,6 +257,13 @@ class EngineArgs:
257
help=('Maximum number of LoRAs to store in CPU memory. '
258
'Must be >= than max_num_seqs. '
259
'Defaults to max_num_seqs.'))
260
+ parser.add_argument(
261
+ "--device",
262
+ type=str,
263
+ default=EngineArgs.device,
264
+ choices=["cuda"],
265
+ help=('Device type for vLLM execution. '
266
+ 'Currently, only CUDA-compatible devices are supported.'))
267
return parser
268
269
@classmethod
@@ -268,7 +277,8 @@ class EngineArgs:
277
def create_engine_configs(
278
self,
279
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
271
- Optional[LoRAConfig]]:
280
+ DeviceConfig, Optional[LoRAConfig]]:
281
+ device_config = DeviceConfig(self.device)
282
model_config = ModelConfig(self.model, self.tokenizer,
283
self.tokenizer_mode, self.trust_remote_code,
284
self.download_dir, self.load_format,
@@ -296,7 +306,8 @@ class EngineArgs:
306
lora_dtype=self.lora_dtype,
307
max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
308
and self.max_cpu_loras > 0 else None) if self.enable_lora else None
299
- return model_config, cache_config, parallel_config, scheduler_config, lora_config
309
+ return (model_config, cache_config, parallel_config, scheduler_config,
310
+ device_config, lora_config)
311
312
313
@dataclass
vllm/engine/llm_engine.py
+10
-2
@@ -6,8 +6,8 @@ from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
6
Union)
7
8
from vllm.lora.request import LoRARequest
9
-from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
10
- SchedulerConfig, LoRAConfig)
9
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10
+ ParallelConfig, SchedulerConfig, LoRAConfig)
11
from vllm.core.scheduler import Scheduler, SchedulerOutputs
12
from vllm.engine.arg_utils import EngineArgs
13
from vllm.engine.metrics import StatLogger, Stats
@@ -53,6 +53,7 @@ class LLMEngine:
53
management.
54
parallel_config: The configuration related to distributed execution.
55
scheduler_config: The configuration related to the request scheduler.
56
+ device_config: The configuration related to the device.
57
placement_group: Ray placement group for distributed execution.
58
Required for distributed execution.
59
log_stats: Whether to log statistics.
@@ -64,6 +65,7 @@ class LLMEngine:
65
cache_config: CacheConfig,
66
parallel_config: ParallelConfig,
67
scheduler_config: SchedulerConfig,
68
+ device_config: DeviceConfig,
69
lora_config: Optional[LoRAConfig],
70
placement_group: Optional["PlacementGroup"],
71
log_stats: bool,
@@ -85,6 +87,7 @@ class LLMEngine:
87
f"quantization={model_config.quantization}, "
88
f"enforce_eager={model_config.enforce_eager}, "
89
f"kv_cache_dtype={cache_config.cache_dtype}, "
90
+ f"device_config={device_config.device}, "
91
f"seed={model_config.seed})")
92
# TODO(woosuk): Print more configs in debug mode.
93
@@ -93,6 +96,7 @@ class LLMEngine:
96
self.lora_config = lora_config
97
self.parallel_config = parallel_config
98
self.scheduler_config = scheduler_config
99
+ self.device_config = device_config
100
self.log_stats = log_stats
101
self._verify_args()
102
@@ -138,6 +142,7 @@ class LLMEngine:
142
self.model_config,
143
self.parallel_config,
144
self.scheduler_config,
145
+ self.device_config,
146
local_rank=0,
147
rank=0,
148
distributed_init_method=distributed_init_method,
@@ -233,6 +238,7 @@ class LLMEngine:
238
model_config = copy.deepcopy(self.model_config)
239
parallel_config = copy.deepcopy(self.parallel_config)
240
scheduler_config = copy.deepcopy(self.scheduler_config)
241
+ device_config = copy.deepcopy(self.device_config)
242
243
for rank, (worker, (node_id,
244
_)) in enumerate(zip(self.workers,
@@ -244,6 +250,7 @@ class LLMEngine:
250
model_config,
251
parallel_config,
252
scheduler_config,
253
+ device_config,
254
local_rank,
255
rank,
256
distributed_init_method,
@@ -257,6 +264,7 @@ class LLMEngine:
264
model_config,
265
parallel_config,
266
scheduler_config,
267
+ device_config,
268
driver_local_rank,
269
driver_rank,
270
distributed_init_method,
vllm/model_executor/layers/activation.py
+1
-3
@@ -89,9 +89,7 @@ class ScaledActivation(nn.Module):
89
if params_dtype is None:
90
params_dtype = torch.get_default_dtype()
91
self.scales = nn.Parameter(
92
- torch.empty(intermediate_size_per_partition,
93
- dtype=params_dtype,
94
- device="cuda"))
92
+ torch.empty(intermediate_size_per_partition, dtype=params_dtype))
93
set_weight_attrs(self.scales, {"weight_loader": self.weight_loader})
94
95
def forward(self, x: torch.Tensor) -> torch.Tensor:
vllm/model_executor/layers/attention.py
+1
-1
@@ -200,7 +200,7 @@ def _make_alibi_bias(
200
seq_len: int,
201
dtype: torch.dtype,
202
) -> LowerTriangularMaskWithTensorBias:
203
- bias = torch.arange(seq_len, dtype=dtype, device="cuda")
203
+ bias = torch.arange(seq_len, dtype=dtype)
204
# NOTE(zhuohan): HF uses
205
# `bias = bias[None, :].repeat(prompt_len, 1)`
206
# here. We find that both biases give the same results, but
vllm/model_executor/layers/linear.py
+2
-8
@@ -54,7 +54,6 @@ class UnquantizedLinearMethod(LinearMethodBase):
54
params_dtype: torch.dtype) -> Dict[str, Any]:
55
weight = Parameter(torch.empty(output_size_per_partition,
56
input_size_per_partition,
57
- device=torch.cuda.current_device(),
57
dtype=params_dtype),
58
requires_grad=False)
59
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
@@ -113,9 +112,7 @@ class ReplicatedLinear(torch.nn.Module):
112
self.register_parameter(name, weight)
113
if bias:
114
self.bias = Parameter(
116
- torch.empty(self.output_size,
117
- device=torch.cuda.current_device(),
118
- dtype=self.params_dtype))
115
+ torch.empty(self.output_size, dtype=self.params_dtype))
116
set_weight_attrs(self.bias, {"output_dim": 0})
117
else:
118
self.register_parameter("bias", None)
@@ -183,7 +180,6 @@ class ColumnParallelLinear(torch.nn.Module):
180
if bias:
181
self.bias = Parameter(
182
torch.empty(self.output_size_per_partition,
186
- device=torch.cuda.current_device(),
183
dtype=params_dtype))
184
set_weight_attrs(self.bias, {
185
"output_dim": 0,
@@ -509,9 +505,7 @@ class RowParallelLinear(torch.nn.Module):
505
506
if bias:
507
self.bias = Parameter(
512
- torch.empty(self.output_size,
513
- device=torch.cuda.current_device(),
514
- dtype=params_dtype))
508
+ torch.empty(self.output_size, dtype=params_dtype))
509
set_weight_attrs(self.bias, {
510
"output_dim": 0,
511
"weight_loader": self.weight_loader,
vllm/model_executor/layers/quantization/awq.py
-3
@@ -96,7 +96,6 @@ class AWQLinearMethod(LinearMethodBase):
96
torch.empty(
97
input_size_per_partition,
98
output_size_per_partition // self.quant_config.pack_factor,
99
- device="cuda",
99
dtype=torch.int32,
100
),
101
requires_grad=False,
@@ -112,7 +111,6 @@ class AWQLinearMethod(LinearMethodBase):
111
torch.empty(
112
input_size_per_partition // self.quant_config.group_size,
113
output_size_per_partition // self.quant_config.pack_factor,
115
- device="cuda",
114
dtype=torch.int32,
115
),
116
requires_grad=False,
@@ -128,7 +126,6 @@ class AWQLinearMethod(LinearMethodBase):
126
torch.empty(
127
input_size_per_partition // self.quant_config.group_size,
128
output_size_per_partition,
131
- device="cuda",
129
dtype=params_dtype,
130
),
131
requires_grad=False,
vllm/model_executor/layers/quantization/gptq.py
-4
@@ -127,7 +127,6 @@ class GPTQLinearMethod(LinearMethodBase):
127
torch.empty(
128
input_size_per_partition // self.quant_config.pack_factor,
129
output_size_per_partition,
130
- device="cuda",
130
dtype=torch.int32,
131
),
132
requires_grad=False,
@@ -145,7 +144,6 @@ class GPTQLinearMethod(LinearMethodBase):
144
i // self.quant_config.group_size
145
for i in range(input_size_per_partition)
146
],
148
- device="cuda",
147
dtype=torch.int32,
148
),
149
requires_grad=False,
@@ -156,7 +154,6 @@ class GPTQLinearMethod(LinearMethodBase):
154
torch.empty(
155
scale_and_zero_size,
156
output_size_per_partition // self.quant_config.pack_factor,
159
- device="cuda",
157
dtype=torch.int32,
158
),
159
requires_grad=False,
@@ -172,7 +169,6 @@ class GPTQLinearMethod(LinearMethodBase):
169
torch.empty(
170
scale_and_zero_size,
171
output_size_per_partition,
175
- device="cuda",
172
dtype=params_dtype,
173
),
174
requires_grad=False,
vllm/model_executor/layers/quantization/squeezellm.py
+2
-4
@@ -80,7 +80,6 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
80
torch.empty(
81
input_size_per_partition // self.quant_config.pack_factor,
82
output_size_per_partition,
83
- device="cuda",
83
dtype=torch.int32,
84
),
85
requires_grad=False,
@@ -96,7 +95,6 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
95
torch.empty(
96
output_size,
97
self.quant_config.weight_bits**2,
99
- device="cuda",
98
dtype=params_dtype,
99
),
100
requires_grad=False,
@@ -118,12 +116,12 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
116
out_shape = x.shape[:-1] + (qweight.shape[-1], )
117
reshaped_x = x.reshape(-1, x.shape[-1])
118
if is_hip():
121
- out_f = torch.zeros(out_shape, device="cuda", dtype=torch.float)
119
+ out_f = torch.zeros(out_shape, dtype=torch.float)
120
ops.squeezellm_gemm(reshaped_x, qweight, out_f, lookup_table)
121
out = out_f.to(dtype=torch.float16)
122
else:
123
# NOTE: The output tensor should be zero-initialized.
126
- out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
124
+ out = torch.zeros(out_shape, dtype=torch.float16)
125
ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
126
127
if bias is not None:
vllm/model_executor/layers/rotary_embedding.py
+9
-13
@@ -77,16 +77,13 @@ class RotaryEmbedding(nn.Module):
77
# create the cache on GPU for faster initialization. This may cause
78
# a slight numerical difference between the HF implementation and ours.
79
inv_freq = 1.0 / (base**(torch.arange(
80
- 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
81
- self.rotary_dim))
80
+ 0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim))
81
return inv_freq
82
83
def _compute_cos_sin_cache(self) -> torch.Tensor:
84
"""Compute the cos and sin cache."""
85
inv_freq = self._compute_inv_freq(self.base)
87
- t = torch.arange(self.max_position_embeddings,
88
- dtype=torch.float,
89
- device="cuda")
86
+ t = torch.arange(self.max_position_embeddings, dtype=torch.float)
87
88
freqs = torch.einsum("i,j -> ij", t, inv_freq)
89
cos = freqs.cos()
@@ -174,7 +171,7 @@ class LinearScalingRotaryEmbedding(RotaryEmbedding):
171
# Thus, the maximum length after applying the rope scaling is
172
# self.max_position_embeddings * self.scaling_factor.
173
max_len = self.max_position_embeddings * self.scaling_factor
177
- t = torch.arange(max_len, dtype=torch.float, device="cuda")
174
+ t = torch.arange(max_len, dtype=torch.float)
175
t = t / self.scaling_factor
176
177
freqs = torch.einsum("i,j -> ij", t, inv_freq)
@@ -214,7 +211,7 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
211
(self.scaling_factor - 1))**(self.rotary_dim /
212
(self.rotary_dim - 2))
213
inv_freq = self._compute_inv_freq(base)
217
- t = torch.arange(max_len, dtype=torch.float, device="cuda")
214
+ t = torch.arange(max_len, dtype=torch.float)
215
216
freqs = torch.einsum("i,j -> ij", t, inv_freq)
217
cos = freqs.cos()
@@ -297,9 +294,9 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
294
is_neox_style)
295
296
def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
300
- pos_freqs = self.base**(torch.arange(
301
- 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
302
- self.rotary_dim)
297
+ pos_freqs = self.base**(
298
+ torch.arange(0, self.rotary_dim, 2, dtype=torch.float) /
299
+ self.rotary_dim)
300
inv_freq_extrapolation = 1.0 / pos_freqs
301
inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
302
@@ -308,8 +305,8 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
305
self.max_position_embeddings)
306
# Get n-d rotational scaling corrected for extrapolation
307
inv_freq_mask = (1 - _yarn_linear_ramp_mask(
311
- low, high, self.rotary_dim // 2, dtype=torch.float,
312
- device="cuda")) * self.extrapolation_factor
308
+ low, high, self.rotary_dim // 2,
309
+ dtype=torch.float)) * self.extrapolation_factor
310
inv_freq = inv_freq_interpolation * (
311
1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
312
return inv_freq
@@ -317,7 +314,6 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
314
def _compute_cos_sin_cache(self) -> torch.Tensor:
315
inv_freq = self._compute_inv_freq(self.scaling_factor)
316
t = torch.arange(self.max_position_embeddings * self.scaling_factor,
320
- device="cuda",
317
dtype=torch.float32)
318
freqs = torch.einsum("i,j -> ij", t, inv_freq)
319
cos = (freqs.cos() * self.mscale)
vllm/model_executor/layers/vocab_parallel_embedding.py
-2
@@ -77,7 +77,6 @@ class VocabParallelEmbedding(torch.nn.Module):
77
self.weight = Parameter(
78
torch.empty(self.num_embeddings_per_partition,
79
self.embedding_dim,
80
- device=torch.cuda.current_device(),
80
dtype=params_dtype))
81
set_weight_attrs(self.weight, {
82
"parallel_dim": 0,
@@ -139,7 +138,6 @@ class ParallelLMHead(VocabParallelEmbedding):
138
if bias:
139
self.bias = Parameter(
140
torch.empty(self.num_embeddings_per_partition,
142
- device=torch.cuda.current_device(),
141
dtype=params_dtype))
142
set_weight_attrs(self.bias, {
143
"parallel_dim": 0,
vllm/model_executor/model_loader.py
+3
-2
@@ -5,7 +5,7 @@ from typing import Optional, Type
5
import torch
6
import torch.nn as nn
7
8
-from vllm.config import ModelConfig, LoRAConfig
8
+from vllm.config import DeviceConfig, ModelConfig, LoRAConfig
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.model_executor.weight_utils import (get_quant_config,
11
initialize_dummy_weights)
@@ -38,6 +38,7 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
38
39
40
def get_model(model_config: ModelConfig,
41
+ device_config: DeviceConfig,
42
lora_config: Optional[LoRAConfig] = None) -> nn.Module:
43
model_class = _get_model_architecture(model_config)
44
@@ -64,7 +65,7 @@ def get_model(model_config: ModelConfig,
65
with _set_default_torch_dtype(model_config.dtype):
66
# Create a model instance.
67
# The weights will be initialized as empty tensors.
67
- with torch.device("cuda"):
68
+ with torch.device(device_config.device):
69
if getattr(model_class, "supports_lora", False):
70
model = model_class(model_config.hf_config, linear_method,
71
lora_config)
vllm/utils.py
+2
-1
@@ -228,7 +228,8 @@ def create_kv_caches_with_random(
228
device: Optional[str] = "cuda",
229
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
230
torch.random.manual_seed(seed)
231
- torch.cuda.manual_seed(seed)
231
+ if torch.cuda.is_available():
232
+ torch.cuda.manual_seed(seed)
233
234
if isinstance(cache_dtype, str):
235
if cache_dtype == "auto":
vllm/worker/cache_engine.py
+2
@@ -104,11 +104,13 @@ class CacheEngine:
104
size=(self.num_cpu_blocks, *key_block_shape),
105
dtype=self.dtype,
106
pin_memory=pin_memory,
107
+ device="cpu",
108
)
109
value_blocks = torch.empty(
110
size=(self.num_cpu_blocks, *value_block_shape),
111
dtype=self.dtype,
112
pin_memory=pin_memory,
113
+ device="cpu",
114
)
115
cpu_cache.append((key_blocks, value_blocks))
116
return cpu_cache
vllm/worker/model_runner.py
+46
-29
@@ -5,7 +5,7 @@ import numpy as np
5
import torch
6
import torch.nn as nn
7
8
-from vllm.config import ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
8
+from vllm.config import DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
from vllm.model_executor.parallel_utils.communication_op import (
@@ -35,6 +35,7 @@ class ModelRunner:
35
model_config: ModelConfig,
36
parallel_config: ParallelConfig,
37
scheduler_config: SchedulerConfig,
38
+ device_config: DeviceConfig,
39
lora_config: Optional[LoRAConfig],
40
kv_cache_dtype: Optional[str] = "auto",
41
is_driver_worker: bool = False,
@@ -49,7 +50,10 @@ class ModelRunner:
50
# FIXME(woosuk): This is a hack to make the tests work. Refactor this.
51
self.sliding_window = (model_config.get_sliding_window()
52
if model_config is not None else None)
52
- self.device = torch.device(torch.cuda.current_device())
53
+ self.device_config = (device_config
54
+ if device_config is not None else DeviceConfig())
55
+ self.device = self.device_config.device
56
+
57
self.model = None
58
self.block_size = None # Set after initial profiling.
59
self.lora_manager = None
@@ -72,7 +76,8 @@ class ModelRunner:
76
self.kv_cache_dtype = kv_cache_dtype
77
78
def load_model(self) -> None:
75
- self.model = get_model(self.model_config, self.lora_config)
79
+ self.model = get_model(self.model_config, self.device_config,
80
+ self.lora_config)
81
82
vocab_size = self.model.config.vocab_size
83
@@ -182,22 +187,25 @@ class ModelRunner:
187
input_tokens = _make_tensor_with_pad(input_tokens,
188
max_prompt_len,
189
pad=0,
185
- dtype=torch.long)
190
+ dtype=torch.long,
191
+ device=self.device)
192
input_positions = _make_tensor_with_pad(input_positions,
193
max_prompt_len,
194
pad=0,
189
- dtype=torch.long)
195
+ dtype=torch.long,
196
+ device=self.device)
197
slot_mapping = _make_tensor_with_pad(slot_mapping,
198
max_prompt_len,
199
pad=_PAD_SLOT_ID,
193
- dtype=torch.long)
200
+ dtype=torch.long,
201
+ device=self.device)
202
lora_index_mapping = [
203
_pad_to_max(mapping, max_prompt_len, pad=0)
204
for mapping in lora_index_mapping
205
]
206
context_lens_tensor = torch.tensor(context_lens,
207
dtype=torch.int,
200
- device='cuda')
208
+ device=self.device)
209
# Prepare prefix block tables
210
max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
211
block_tables = _make_tensor_with_pad(
@@ -205,15 +213,16 @@ class ModelRunner:
213
max_len=max_prompt_block_table_len,
214
pad=0,
215
dtype=torch.int,
216
+ device=self.device,
217
)
218
start_loc_tensor = torch.arange(0,
219
len(prompt_lens) * max_prompt_len,
220
max_prompt_len,
221
dtype=torch.long,
213
- device='cuda')
222
+ device=self.device)
223
prompt_lens_tensor = torch.tensor(prompt_lens,
224
dtype=torch.long,
216
- device='cuda')
225
+ device=self.device)
226
227
input_metadata = InputMetadata(
228
is_prompt=True,
@@ -305,20 +314,20 @@ class ModelRunner:
314
max_len=1,
315
pad=0,
316
dtype=torch.long,
308
- device="cuda")
317
+ device=self.device)
318
input_positions = _make_tensor_with_pad(input_positions,
319
max_len=1,
320
pad=0,
321
dtype=torch.long,
313
- device="cuda")
322
+ device=self.device)
323
slot_mapping = _make_tensor_with_pad(slot_mapping,
324
max_len=1,
325
pad=_PAD_SLOT_ID,
326
dtype=torch.long,
318
- device="cuda")
327
+ device=self.device)
328
context_lens = torch.tensor(context_lens,
329
dtype=torch.int,
321
- device="cuda")
330
+ device=self.device)
331
332
if use_captured_graph:
333
# The shape of graph_block_tables is
@@ -327,7 +336,7 @@ class ModelRunner:
336
for i, block_table in enumerate(block_tables):
337
if block_table:
338
input_block_tables[i, :len(block_table)] = block_table
330
- block_tables = torch.tensor(input_block_tables, device="cuda")
339
+ block_tables = torch.tensor(input_block_tables, device=self.device)
340
else:
341
max_block_table_len = max(
342
len(block_table) for block_table in block_tables)
@@ -336,7 +345,7 @@ class ModelRunner:
345
max_len=max_block_table_len,
346
pad=0,
347
dtype=torch.int,
339
- device="cuda",
348
+ device=self.device,
349
)
350
351
lora_index_mapping = [
@@ -355,7 +364,8 @@ class ModelRunner:
364
use_cuda_graph=use_captured_graph,
365
kv_cache_dtype=self.kv_cache_dtype,
366
)
358
- return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
367
+ return (input_tokens, input_positions, input_metadata,
368
+ lora_index_mapping, lora_prompt_mapping, lora_requests)
369
370
def _prepare_sample(
371
self,
@@ -410,9 +420,13 @@ class ModelRunner:
420
421
selected_token_indices = _async_h2d(selected_token_indices,
422
dtype=torch.long,
423
+ target_device=self.device,
424
pin_memory=not self.in_wsl)
425
categorized_sample_indices = {
415
- t: _async_h2d(seq_ids, dtype=torch.int, pin_memory=not self.in_wsl)
426
+ t: _async_h2d(seq_ids,
427
+ dtype=torch.int,
428
+ target_device=self.device,
429
+ pin_memory=not self.in_wsl)
430
for t, seq_ids in categorized_sample_indices.items()
431
}
432
@@ -511,7 +525,8 @@ class ModelRunner:
525
perform_sampling=False,
526
)
527
514
- return input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping
528
+ return (input_tokens, input_positions, input_metadata,
529
+ sampling_metadata, lora_requests, lora_mapping)
530
531
@torch.inference_mode()
532
def execute_model(
@@ -519,8 +534,9 @@ class ModelRunner:
534
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
535
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
536
) -> Optional[SamplerOutput]:
522
- input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping = (
523
- self.prepare_input_tensors(seq_group_metadata_list))
537
+ (input_tokens, input_positions, input_metadata, sampling_metadata,
538
+ lora_requests,
539
+ lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
540
541
if self.lora_config:
542
self.set_active_loras(lora_requests, lora_mapping)
@@ -789,14 +805,10 @@ def _make_tensor_with_pad(
805
max_len: int,
806
pad: int,
807
dtype: torch.dtype,
792
- device: Union[str, torch.device] = "cuda",
793
- pin_memory: bool = False,
808
+ device: Optional[Union[str, torch.device]],
809
) -> torch.Tensor:
810
padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
796
- return torch.tensor(padded_x,
797
- dtype=dtype,
798
- device=device,
799
- pin_memory=pin_memory and str(device) == "cpu")
811
+ return torch.tensor(padded_x, dtype=dtype, device=device)
812
813
814
def _get_graph_batch_size(batch_size: int) -> int:
@@ -808,6 +820,11 @@ def _get_graph_batch_size(batch_size: int) -> int:
820
return (batch_size + 7) // 8 * 8
821
822
811
-def _async_h2d(data: list, dtype, pin_memory):
812
- t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory)
813
- return t.to(device="cuda", non_blocking=True)
823
+def _async_h2d(
824
+ data: list,
825
+ dtype: torch.dtype,
826
+ target_device: Union[str, torch.device],
827
+ pin_memory: bool,
828
+) -> torch.Tensor:
829
+ t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
830
+ return t.to(device=target_device, non_blocking=True)
vllm/worker/worker.py
+23
-17
@@ -6,8 +6,8 @@ from typing import Dict, List, Tuple, Set, Optional
6
import torch
7
import torch.distributed
8
9
-from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
10
- SchedulerConfig, LoRAConfig)
9
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10
+ ParallelConfig, SchedulerConfig, LoRAConfig)
11
from vllm.model_executor import set_random_seed
12
from vllm.model_executor.parallel_utils.communication_op import (
13
broadcast_tensor_dict)
@@ -33,6 +33,7 @@ class Worker:
33
model_config: ModelConfig,
34
parallel_config: ParallelConfig,
35
scheduler_config: SchedulerConfig,
36
+ device_config: DeviceConfig,
37
local_rank: int,
38
rank: int,
39
distributed_init_method: str,
@@ -43,6 +44,7 @@ class Worker:
44
self.model_config = model_config
45
self.parallel_config = parallel_config
46
self.scheduler_config = scheduler_config
47
+ self.device_config = device_config
48
self.local_rank = local_rank
49
self.rank = rank
50
self.distributed_init_method = distributed_init_method
@@ -54,6 +56,7 @@ class Worker:
56
self.model_runner = ModelRunner(model_config,
57
parallel_config,
58
scheduler_config,
59
+ device_config,
60
lora_config=self.lora_config,
61
kv_cache_dtype=kv_cache_dtype,
62
is_driver_worker=is_driver_worker)
@@ -65,21 +68,24 @@ class Worker:
68
self.gpu_cache = None
69
70
def init_model(self) -> None:
68
- # torch.distributed.all_reduce does not free the input tensor until
69
- # the synchronization point. This causes the memory usage to grow
70
- # as the number of all_reduce calls increases. This env var disables
71
- # this behavior.
72
- # Related issue:
73
- # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
74
- os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
75
-
76
- # This env var set by Ray causes exceptions with graph building.
77
- os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
78
- self.device = torch.device(f"cuda:{self.local_rank}")
79
- torch.cuda.set_device(self.device)
80
-
81
- _check_if_gpu_supports_dtype(self.model_config.dtype)
82
-
71
+ if self.device_config.device.type == "cuda":
72
+ # torch.distributed.all_reduce does not free the input tensor until
73
+ # the synchronization point. This causes the memory usage to grow
74
+ # as the number of all_reduce calls increases. This env var disables
75
+ # this behavior.
76
+ # Related issue:
77
+ # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
78
+ os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
79
+
80
+ # This env var set by Ray causes exceptions with graph building.
81
+ os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
82
+ self.device = torch.device(f"cuda:{self.local_rank}")
83
+ torch.cuda.set_device(self.device)
84
+
85
+ _check_if_gpu_supports_dtype(self.model_config.dtype)
86
+ else:
87
+ raise RuntimeError(
88
+ f"Not support device type: {self.device_config.device}")
89
# Initialize the distributed environment.
90
init_distributed_environment(self.parallel_config, self.rank,
91
self.distributed_init_method)