A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Support FP32 (#141)
Woosuk Kwon committed
Jun 7, 2023 at 00:40 UTC
e38074b1e6ad0975acbfa15d858c4bd7cd005e99
8 files changed
+65
-54
cacheflow/config.py
+3
-4
@@ -164,7 +164,7 @@ def _get_and_verify_dtype(
164
config_dtype = torch.float32
165
166
dtype = dtype.lower()
167
- if dtype == "default":
167
+ if dtype == "auto":
168
if config_dtype == torch.float32:
169
# Following the common practice, we use float16 for float32 models.
170
torch_dtype = torch.float16
@@ -184,9 +184,8 @@ def _get_and_verify_dtype(
184
# Downcasting from float32 to float16 or bfloat16 is allowed.
185
pass
186
else:
187
- # Casting between float16 and bfloat16 is not allowed.
188
- raise ValueError(
189
- f"Cannot use {torch_dtype} for {config_dtype} model.")
187
+ # Casting between float16 and bfloat16 is allowed with a warning.
188
+ logger.warn(f"Casting {config_dtype} to {torch_dtype}.")
189
190
# Check if the GPU supports the dtype.
191
if torch_dtype == torch.bfloat16:
cacheflow/entrypoints/llm.py
+5
-4
@@ -28,9 +28,10 @@ class LLM:
28
tensor_parallel_size: The number of GPUs to use for distributed
29
execution with tensor parallelism.
30
dtype: The data type for the model weights and activations. Currently,
31
- we support `float16` and `bfloat16`. If `default`, we use the
32
- `torch_dtype` attribute of the model config. If the `torch_dtype`
33
- is `float32`, we use `float16` instead.
31
+ we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
32
+ the `torch_dtype` attribute specified in the model config file.
33
+ However, if the `torch_dtype` in the config is `float32`, we will
34
+ use `float16` instead.
35
seed: The seed to initialize the random number generator for sampling.
36
"""
37
@@ -38,7 +39,7 @@ class LLM:
39
self,
40
model: str,
41
tensor_parallel_size: int = 1,
41
- dtype: str = "default",
42
+ dtype: str = "auto",
43
seed: int = 0,
44
**kwargs,
45
) -> None:
cacheflow/model_executor/layers/attention.py
+3
-5
@@ -10,7 +10,7 @@ from cacheflow import cache_ops
10
from cacheflow import pos_encoding_ops
11
from cacheflow.model_executor.input_metadata import InputMetadata
12
13
-_SUPPORTED_HEAD_SIZES = [32, 64, 80, 96, 128, 160, 192, 256]
13
+_SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
14
15
16
class GPTCacheFlowAttention(nn.Module):
@@ -49,10 +49,8 @@ class GPTCacheFlowAttention(nn.Module):
49
self.attn_op = xops.fmha.cutlass.FwOp()
50
51
if self.head_size not in _SUPPORTED_HEAD_SIZES:
52
- raise ValueError(f'head_size ({self.head_size}) is not supported by '
53
- 'the single_query_cached_kv_attention kernel. '
54
- 'Use one of the following head sizes: '
55
- f'{_SUPPORTED_HEAD_SIZES}.')
52
+ raise ValueError(f"head_size ({self.head_size}) is not supported. "
53
+ f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
54
55
def multi_query_kv_attention(
56
self,
cacheflow/server/arg_utils.py
+4
-4
@@ -13,7 +13,7 @@ class ServerArgs:
13
download_dir: Optional[str] = None
14
use_np_weights: bool = False
15
use_dummy_weights: bool = False
16
- dtype: str = "default"
16
+ dtype: str = "auto"
17
seed: int = 0
18
worker_use_ray: bool = False
19
pipeline_parallel_size: int = 1
@@ -49,9 +49,9 @@ class ServerArgs:
49
help='use dummy values for model weights')
50
# TODO(woosuk): Support FP32.
51
parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
52
- choices=['default', 'half', 'bfloat16'],
52
+ choices=['auto', 'half', 'bfloat16', 'float'],
53
help='data type for model weights and activations. '
54
- 'The "default" option will use FP16 precision '
54
+ 'The "auto" option will use FP16 precision '
55
'for FP32 and FP16 models, and BF16 precision '
56
'for BF16 models.')
57
# Parallel arguments
@@ -67,7 +67,7 @@ class ServerArgs:
67
# KV cache arguments
68
parser.add_argument('--block-size', type=int,
69
default=ServerArgs.block_size,
70
- choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
70
+ choices=[8, 16, 32],
71
help='token block size')
72
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
73
parser.add_argument('--seed', type=int, default=ServerArgs.seed,
csrc/attention/attention_kernels.cu
+37
-32
@@ -370,9 +370,11 @@ void single_query_cached_kv_attention_launcher(
370
dim3 block(NUM_THREADS);
371
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
372
switch (head_size) {
373
- case 32:
374
- LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
375
- break;
373
+ // NOTE(woosuk): To reduce the compilation time, we omitted head sizes
374
+ // 32, 160, 192, 256.
375
+ // case 32:
376
+ // LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
377
+ // break;
378
case 64:
379
LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
380
break;
@@ -385,15 +387,15 @@ void single_query_cached_kv_attention_launcher(
387
case 128:
388
LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
389
break;
388
- case 160:
389
- LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
390
- break;
391
- case 192:
392
- LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
393
- break;
394
- case 256:
395
- LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
396
- break;
390
+ // case 160:
391
+ // LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
392
+ // break;
393
+ // case 192:
394
+ // LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
395
+ // break;
396
+ // case 256:
397
+ // LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
398
+ // break;
399
default:
400
TORCH_CHECK(false, "Unsupported head size: ", head_size);
401
break;
@@ -411,17 +413,19 @@ void single_query_cached_kv_attention_launcher(
413
context_lens, \
414
max_context_len);
415
416
+// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
417
+// 1, 2, 4, 64, 128, 256.
418
#define CALL_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
419
switch (block_size) { \
416
- case 1: \
417
- CALL_KERNEL_LAUNCHER(T, 1); \
418
- break; \
419
- case 2: \
420
- CALL_KERNEL_LAUNCHER(T, 2); \
421
- break; \
422
- case 4: \
423
- CALL_KERNEL_LAUNCHER(T, 4); \
424
- break; \
420
+ /* case 1: */ \
421
+ /* CALL_KERNEL_LAUNCHER(T, 1); */ \
422
+ /* break; */ \
423
+ /* case 2: */ \
424
+ /* CALL_KERNEL_LAUNCHER(T, 2); */ \
425
+ /* break; */ \
426
+ /* case 4: */ \
427
+ /* CALL_KERNEL_LAUNCHER(T, 4); */ \
428
+ /* break; */ \
429
case 8: \
430
CALL_KERNEL_LAUNCHER(T, 8); \
431
break; \
@@ -431,15 +435,15 @@ void single_query_cached_kv_attention_launcher(
435
case 32: \
436
CALL_KERNEL_LAUNCHER(T, 32); \
437
break; \
434
- case 64: \
435
- CALL_KERNEL_LAUNCHER(T, 64); \
436
- break; \
437
- case 128: \
438
- CALL_KERNEL_LAUNCHER(T, 128); \
439
- break; \
440
- case 256: \
441
- CALL_KERNEL_LAUNCHER(T, 256); \
442
- break; \
438
+ /* case 64: */ \
439
+ /* CALL_KERNEL_LAUNCHER(T, 64); */ \
440
+ /* break; */ \
441
+ /* case 128: */ \
442
+ /* CALL_KERNEL_LAUNCHER(T, 128); */ \
443
+ /* break; */ \
444
+ /* case 256: */ \
445
+ /* CALL_KERNEL_LAUNCHER(T, 256); */ \
446
+ /* break; */ \
447
default: \
448
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
449
break; \
@@ -455,8 +459,9 @@ void single_query_cached_kv_attention(
459
torch::Tensor& context_lens, // [num_seqs]
460
int block_size,
461
int max_context_len) {
458
- // TODO(woosuk): Support FP32.
459
- if (query.dtype() == at::ScalarType::Half) {
462
+ if (query.dtype() == at::ScalarType::Float) {
463
+ CALL_KERNEL_LAUNCHER_BLOCK_SIZE(float);
464
+ } else if (query.dtype() == at::ScalarType::Half) {
465
CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
466
} else if (query.dtype() == at::ScalarType::BFloat16) {
467
CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
docs/source/getting_started/installation.rst
+3
@@ -18,8 +18,11 @@ CacheFlow can run on systems that meet the following requirements:
18
19
.. code-block:: console
20
21
+ $ # Pull the Docker image with CUDA 11.8.
22
$ docker run --gpus all -it --rm --shm-size=8g nvcr.io/nvidia/pytorch:22.12-py3
23
24
+ Inside the Docker container, please execute :code:`pip uninstall torch` before installing CacheFlow.
25
+
26
Install with pip
27
----------------
28
setup.py
+5
@@ -66,6 +66,11 @@ if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
66
raise RuntimeError(
67
"CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
68
69
+# Use NVCC threads to parallelize the build.
70
+if nvcc_cuda_version >= Version("11.2"):
71
+ num_threads = min(os.cpu_count(), 8)
72
+ NVCC_FLAGS += ["--threads", str(num_threads)]
73
+
74
ext_modules = []
75
76
# Cache operations.
tests/kernels/test_attention.py
+5
-5
@@ -270,9 +270,9 @@ def run_multi_query_kv_attention(
270
def test_single_query_cached_kv_attention() -> None:
271
torch.random.manual_seed(TEST_SEED)
272
torch.cuda.manual_seed(TEST_SEED)
273
- for dtype in [torch.half, torch.bfloat16]:
274
- for block_size in [8, 16, 32, 64]:
275
- for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
273
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
274
+ for block_size in [8, 16, 32]:
275
+ for head_size in [64, 80, 96, 128]:
276
print(f'Testing single_query_cached_kv_attention with '
277
f'dtype={dtype}, block_size={block_size}, '
278
f'head_size={head_size}')
@@ -289,8 +289,8 @@ def test_single_query_cached_kv_attention() -> None:
289
def test_multi_query_kv_attention() -> None:
290
torch.random.manual_seed(TEST_SEED)
291
torch.cuda.manual_seed(TEST_SEED)
292
- for dtype in [torch.half, torch.bfloat16]:
293
- for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
292
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
293
+ for head_size in [64, 80, 96, 128]:
294
print(f'Testing multi_query_kv_attention with dtype={dtype}, '
295
f'head_size={head_size}')
296
run_multi_query_kv_attention(