A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Disable cuda version check in vllm-openai image (#4530)
zhaoyang-star committed
May 6, 2024 at 07:58 UTC
0650e5935b0f6af35fb2acf71769982c47b804d7
2 files changed
+2
-33
vllm/config.py
+1
-10
@@ -4,15 +4,13 @@ from dataclasses import dataclass, field, fields
4
from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
5
6
import torch
7
-from packaging.version import Version
7
from transformers import PretrainedConfig
8
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
11
get_quantization_config)
12
from vllm.transformers_utils.config import get_config, get_hf_text_config
14
-from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
15
- is_neuron)
13
+from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
14
15
GPTQMarlinConfig = get_quantization_config("gptq_marlin")
16
@@ -369,13 +367,6 @@ class CacheConfig:
367
if self.cache_dtype == "auto":
368
pass
369
elif self.cache_dtype == "fp8":
372
- if not is_hip():
373
- nvcc_cuda_version = get_nvcc_cuda_version()
374
- if nvcc_cuda_version is not None \
375
- and nvcc_cuda_version < Version("11.8"):
376
- raise ValueError(
377
- "FP8 is not supported when cuda version is"
378
- "lower than 11.8.")
370
logger.info(
371
"Using fp8 data type to store kv cache. It reduces the GPU "
372
"memory footprint and boosts the performance. "
vllm/utils.py
+1
-23
@@ -19,7 +19,6 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
19
20
import psutil
21
import torch
22
-from packaging.version import Version, parse
22
23
import vllm.envs as envs
24
from vllm.logger import enable_trace_function_call, init_logger
@@ -314,27 +313,6 @@ def cdiv(a: int, b: int) -> int:
313
return -(a // -b)
314
315
317
-@lru_cache(maxsize=None)
318
-def get_nvcc_cuda_version() -> Optional[Version]:
319
- cuda_home = envs.CUDA_HOME
320
- if not cuda_home:
321
- cuda_home = '/usr/local/cuda'
322
- if os.path.isfile(cuda_home + '/bin/nvcc'):
323
- logger.info(
324
- 'CUDA_HOME is not found in the environment. '
325
- 'Using %s as CUDA_HOME.', cuda_home)
326
- else:
327
- logger.warning('Not found nvcc in %s. Skip cuda version check!',
328
- cuda_home)
329
- return None
330
- nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
331
- universal_newlines=True)
332
- output = nvcc_output.split()
333
- release_idx = output.index("release") + 1
334
- nvcc_cuda_version = parse(output[release_idx].split(",")[0])
335
- return nvcc_cuda_version
336
-
337
-
316
def _generate_random_fp8(
317
tensor: torch.tensor,
318
low: float,
@@ -560,7 +538,7 @@ def maybe_expand_dim(tensor: torch.Tensor,
538
def merge_dicts(dict1: Dict[Any, List[Any]],
539
dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
540
"""Merge 2 dicts that have key -> List of items.
563
-
541
+
542
When a key conflicts, the values in dict1 is prioritized.
543
"""
544
merged_dict = defaultdict(list)