@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e38074b1e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support FP32 (#141)

Woosuk Kwon committed Jun 7, 2023 at 00:40 UTC e38074b1e6ad0975acbfa15d858c4bd7cd005e99
8 files changed +65 -54
cacheflow/config.py
+3 -4
@@ -164,7 +164,7 @@ def _get_and_verify_dtype(
164 config_dtype = torch.float32
165
166 dtype = dtype.lower()
167 - if dtype == "default":
167 + if dtype == "auto":
168 if config_dtype == torch.float32:
169 # Following the common practice, we use float16 for float32 models.
170 torch_dtype = torch.float16
@@ -184,9 +184,8 @@ def _get_and_verify_dtype(
184 # Downcasting from float32 to float16 or bfloat16 is allowed.
185 pass
186 else:
187 - # Casting between float16 and bfloat16 is not allowed.
188 - raise ValueError(
189 - f"Cannot use {torch_dtype} for {config_dtype} model.")
187 + # Casting between float16 and bfloat16 is allowed with a warning.
188 + logger.warn(f"Casting {config_dtype} to {torch_dtype}.")
189
190 # Check if the GPU supports the dtype.
191 if torch_dtype == torch.bfloat16:
cacheflow/entrypoints/llm.py
+5 -4
@@ -28,9 +28,10 @@ class LLM:
28 tensor_parallel_size: The number of GPUs to use for distributed
29 execution with tensor parallelism.
30 dtype: The data type for the model weights and activations. Currently,
31 - we support `float16` and `bfloat16`. If `default`, we use the
32 - `torch_dtype` attribute of the model config. If the `torch_dtype`
33 - is `float32`, we use `float16` instead.
31 + we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
32 + the `torch_dtype` attribute specified in the model config file.
33 + However, if the `torch_dtype` in the config is `float32`, we will
34 + use `float16` instead.
35 seed: The seed to initialize the random number generator for sampling.
36 """
37
@@ -38,7 +39,7 @@ class LLM:
39 self,
40 model: str,
41 tensor_parallel_size: int = 1,
41 - dtype: str = "default",
42 + dtype: str = "auto",
43 seed: int = 0,
44 **kwargs,
45 ) -> None:
cacheflow/model_executor/layers/attention.py
+3 -5
@@ -10,7 +10,7 @@ from cacheflow import cache_ops
10 from cacheflow import pos_encoding_ops
11 from cacheflow.model_executor.input_metadata import InputMetadata
12
13 -_SUPPORTED_HEAD_SIZES = [32, 64, 80, 96, 128, 160, 192, 256]
13 +_SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
14
15
16 class GPTCacheFlowAttention(nn.Module):
@@ -49,10 +49,8 @@ class GPTCacheFlowAttention(nn.Module):
49 self.attn_op = xops.fmha.cutlass.FwOp()
50
51 if self.head_size not in _SUPPORTED_HEAD_SIZES:
52 - raise ValueError(f'head_size ({self.head_size}) is not supported by '
53 - 'the single_query_cached_kv_attention kernel. '
54 - 'Use one of the following head sizes: '
55 - f'{_SUPPORTED_HEAD_SIZES}.')
52 + raise ValueError(f"head_size ({self.head_size}) is not supported. "
53 + f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
54
55 def multi_query_kv_attention(
56 self,
cacheflow/server/arg_utils.py
+4 -4
@@ -13,7 +13,7 @@ class ServerArgs:
13 download_dir: Optional[str] = None
14 use_np_weights: bool = False
15 use_dummy_weights: bool = False
16 - dtype: str = "default"
16 + dtype: str = "auto"
17 seed: int = 0
18 worker_use_ray: bool = False
19 pipeline_parallel_size: int = 1
@@ -49,9 +49,9 @@ class ServerArgs:
49 help='use dummy values for model weights')
50 # TODO(woosuk): Support FP32.
51 parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
52 - choices=['default', 'half', 'bfloat16'],
52 + choices=['auto', 'half', 'bfloat16', 'float'],
53 help='data type for model weights and activations. '
54 - 'The "default" option will use FP16 precision '
54 + 'The "auto" option will use FP16 precision '
55 'for FP32 and FP16 models, and BF16 precision '
56 'for BF16 models.')
57 # Parallel arguments
@@ -67,7 +67,7 @@ class ServerArgs:
67 # KV cache arguments
68 parser.add_argument('--block-size', type=int,
69 default=ServerArgs.block_size,
70 - choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
70 + choices=[8, 16, 32],
71 help='token block size')
72 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
73 parser.add_argument('--seed', type=int, default=ServerArgs.seed,
csrc/attention/attention_kernels.cu
+37 -32
@@ -370,9 +370,11 @@ void single_query_cached_kv_attention_launcher(
370 dim3 block(NUM_THREADS);
371 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
372 switch (head_size) {
373 - case 32:
374 - LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
375 - break;
373 + // NOTE(woosuk): To reduce the compilation time, we omitted head sizes
374 + // 32, 160, 192, 256.
375 + // case 32:
376 + // LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
377 + // break;
378 case 64:
379 LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
380 break;
@@ -385,15 +387,15 @@ void single_query_cached_kv_attention_launcher(
387 case 128:
388 LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
389 break;
388 - case 160:
389 - LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
390 - break;
391 - case 192:
392 - LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
393 - break;
394 - case 256:
395 - LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
396 - break;
390 + // case 160:
391 + // LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
392 + // break;
393 + // case 192:
394 + // LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
395 + // break;
396 + // case 256:
397 + // LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
398 + // break;
399 default:
400 TORCH_CHECK(false, "Unsupported head size: ", head_size);
401 break;
@@ -411,17 +413,19 @@ void single_query_cached_kv_attention_launcher(
413 context_lens, \
414 max_context_len);
415
416 +// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
417 +// 1, 2, 4, 64, 128, 256.
418 #define CALL_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
419 switch (block_size) { \
416 - case 1: \
417 - CALL_KERNEL_LAUNCHER(T, 1); \
418 - break; \
419 - case 2: \
420 - CALL_KERNEL_LAUNCHER(T, 2); \
421 - break; \
422 - case 4: \
423 - CALL_KERNEL_LAUNCHER(T, 4); \
424 - break; \
420 + /* case 1: */ \
421 + /* CALL_KERNEL_LAUNCHER(T, 1); */ \
422 + /* break; */ \
423 + /* case 2: */ \
424 + /* CALL_KERNEL_LAUNCHER(T, 2); */ \
425 + /* break; */ \
426 + /* case 4: */ \
427 + /* CALL_KERNEL_LAUNCHER(T, 4); */ \
428 + /* break; */ \
429 case 8: \
430 CALL_KERNEL_LAUNCHER(T, 8); \
431 break; \
@@ -431,15 +435,15 @@ void single_query_cached_kv_attention_launcher(
435 case 32: \
436 CALL_KERNEL_LAUNCHER(T, 32); \
437 break; \
434 - case 64: \
435 - CALL_KERNEL_LAUNCHER(T, 64); \
436 - break; \
437 - case 128: \
438 - CALL_KERNEL_LAUNCHER(T, 128); \
439 - break; \
440 - case 256: \
441 - CALL_KERNEL_LAUNCHER(T, 256); \
442 - break; \
438 + /* case 64: */ \
439 + /* CALL_KERNEL_LAUNCHER(T, 64); */ \
440 + /* break; */ \
441 + /* case 128: */ \
442 + /* CALL_KERNEL_LAUNCHER(T, 128); */ \
443 + /* break; */ \
444 + /* case 256: */ \
445 + /* CALL_KERNEL_LAUNCHER(T, 256); */ \
446 + /* break; */ \
447 default: \
448 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
449 break; \
@@ -455,8 +459,9 @@ void single_query_cached_kv_attention(
459 torch::Tensor& context_lens, // [num_seqs]
460 int block_size,
461 int max_context_len) {
458 - // TODO(woosuk): Support FP32.
459 - if (query.dtype() == at::ScalarType::Half) {
462 + if (query.dtype() == at::ScalarType::Float) {
463 + CALL_KERNEL_LAUNCHER_BLOCK_SIZE(float);
464 + } else if (query.dtype() == at::ScalarType::Half) {
465 CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
466 } else if (query.dtype() == at::ScalarType::BFloat16) {
467 CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
docs/source/getting_started/installation.rst
+3
@@ -18,8 +18,11 @@ CacheFlow can run on systems that meet the following requirements:
18
19 .. code-block:: console
20
21 + $ # Pull the Docker image with CUDA 11.8.
22 $ docker run --gpus all -it --rm --shm-size=8g nvcr.io/nvidia/pytorch:22.12-py3
23
24 + Inside the Docker container, please execute :code:`pip uninstall torch` before installing CacheFlow.
25 +
26 Install with pip
27 ----------------
28
setup.py
+5
@@ -66,6 +66,11 @@ if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
66 raise RuntimeError(
67 "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
68
69 +# Use NVCC threads to parallelize the build.
70 +if nvcc_cuda_version >= Version("11.2"):
71 + num_threads = min(os.cpu_count(), 8)
72 + NVCC_FLAGS += ["--threads", str(num_threads)]
73 +
74 ext_modules = []
75
76 # Cache operations.
tests/kernels/test_attention.py
+5 -5
@@ -270,9 +270,9 @@ def run_multi_query_kv_attention(
270 def test_single_query_cached_kv_attention() -> None:
271 torch.random.manual_seed(TEST_SEED)
272 torch.cuda.manual_seed(TEST_SEED)
273 - for dtype in [torch.half, torch.bfloat16]:
274 - for block_size in [8, 16, 32, 64]:
275 - for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
273 + for dtype in [torch.half, torch.bfloat16, torch.float]:
274 + for block_size in [8, 16, 32]:
275 + for head_size in [64, 80, 96, 128]:
276 print(f'Testing single_query_cached_kv_attention with '
277 f'dtype={dtype}, block_size={block_size}, '
278 f'head_size={head_size}')
@@ -289,8 +289,8 @@ def test_single_query_cached_kv_attention() -> None:
289 def test_multi_query_kv_attention() -> None:
290 torch.random.manual_seed(TEST_SEED)
291 torch.cuda.manual_seed(TEST_SEED)
292 - for dtype in [torch.half, torch.bfloat16]:
293 - for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
292 + for dtype in [torch.half, torch.bfloat16, torch.float]:
293 + for head_size in [64, 80, 96, 128]:
294 print(f'Testing multi_query_kv_attention with dtype={dtype}, '
295 f'head_size={head_size}')
296 run_multi_query_kv_attention(