@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0650e5935b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Disable cuda version check in vllm-openai image (#4530)

zhaoyang-star committed May 6, 2024 at 07:58 UTC 0650e5935b0f6af35fb2acf71769982c47b804d7
2 files changed +2 -33
vllm/config.py
+1 -10
@@ -4,15 +4,13 @@ from dataclasses import dataclass, field, fields
4 from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
5
6 import torch
7 -from packaging.version import Version
7 from transformers import PretrainedConfig
8
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
11 get_quantization_config)
12 from vllm.transformers_utils.config import get_config, get_hf_text_config
14 -from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
15 - is_neuron)
13 +from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
14
15 GPTQMarlinConfig = get_quantization_config("gptq_marlin")
16
@@ -369,13 +367,6 @@ class CacheConfig:
367 if self.cache_dtype == "auto":
368 pass
369 elif self.cache_dtype == "fp8":
372 - if not is_hip():
373 - nvcc_cuda_version = get_nvcc_cuda_version()
374 - if nvcc_cuda_version is not None \
375 - and nvcc_cuda_version < Version("11.8"):
376 - raise ValueError(
377 - "FP8 is not supported when cuda version is"
378 - "lower than 11.8.")
370 logger.info(
371 "Using fp8 data type to store kv cache. It reduces the GPU "
372 "memory footprint and boosts the performance. "
vllm/utils.py
+1 -23
@@ -19,7 +19,6 @@ from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
19
20 import psutil
21 import torch
22 -from packaging.version import Version, parse
22
23 import vllm.envs as envs
24 from vllm.logger import enable_trace_function_call, init_logger
@@ -314,27 +313,6 @@ def cdiv(a: int, b: int) -> int:
313 return -(a // -b)
314
315
317 -@lru_cache(maxsize=None)
318 -def get_nvcc_cuda_version() -> Optional[Version]:
319 - cuda_home = envs.CUDA_HOME
320 - if not cuda_home:
321 - cuda_home = '/usr/local/cuda'
322 - if os.path.isfile(cuda_home + '/bin/nvcc'):
323 - logger.info(
324 - 'CUDA_HOME is not found in the environment. '
325 - 'Using %s as CUDA_HOME.', cuda_home)
326 - else:
327 - logger.warning('Not found nvcc in %s. Skip cuda version check!',
328 - cuda_home)
329 - return None
330 - nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
331 - universal_newlines=True)
332 - output = nvcc_output.split()
333 - release_idx = output.index("release") + 1
334 - nvcc_cuda_version = parse(output[release_idx].split(",")[0])
335 - return nvcc_cuda_version
336 -
337 -
316 def _generate_random_fp8(
317 tensor: torch.tensor,
318 low: float,
@@ -560,7 +538,7 @@ def maybe_expand_dim(tensor: torch.Tensor,
538 def merge_dicts(dict1: Dict[Any, List[Any]],
539 dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
540 """Merge 2 dicts that have key -> List of items.
563 -
541 +
542 When a key conflicts, the values in dict1 is prioritized.
543 """
544 merged_dict = defaultdict(list)