@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6b78837b29
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix setup.py neuron-ls issue (#2671)

Simon Mo committed Mar 16, 2024 at 16:00 UTC 6b78837b29b5045a71e6ecfa68442b1f4fd2d0a6
1 file changed +24 -19
setup.py
+24 -19
@@ -2,6 +2,7 @@ import contextlib
2 import io
3 import os
4 import re
5 +import shutil
6 import subprocess
7 import warnings
8 from pathlib import Path
@@ -38,6 +39,10 @@ ROCM_SUPPORTED_ARCHS = {"gfx908", "gfx90a", "gfx942", "gfx1100"}
39 # SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
40
41
42 +def _is_cuda() -> bool:
43 + return torch.version.cuda is not None
44 +
45 +
46 def _is_hip() -> bool:
47 return torch.version.hip is not None
48
@@ -46,15 +51,11 @@ def _is_neuron() -> bool:
51 torch_neuronx_installed = True
52 try:
53 subprocess.run(["neuron-ls"], capture_output=True, check=True)
49 - except (FileNotFoundError, PermissionError):
54 + except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
55 torch_neuronx_installed = False
56 return torch_neuronx_installed
57
58
54 -def _is_cuda() -> bool:
55 - return (torch.version.cuda is not None) and not _is_neuron()
56 -
57 -
59 # Compiler flags.
60 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
61 # TODO(woosuk): Should we use -O3?
@@ -400,7 +401,12 @@ def find_version(filepath: str) -> str:
401 def get_vllm_version() -> str:
402 version = find_version(get_path("vllm", "__init__.py"))
403
403 - if _is_hip():
404 + if _is_cuda():
405 + cuda_version = str(nvcc_cuda_version)
406 + if cuda_version != MAIN_CUDA_VERSION:
407 + cuda_version_str = cuda_version.replace(".", "")[:3]
408 + version += f"+cu{cuda_version_str}"
409 + elif _is_hip():
410 # Get the HIP version
411 hipcc_version = get_hipcc_rocm_version()
412 if hipcc_version != MAIN_CUDA_VERSION:
@@ -412,13 +418,8 @@ def get_vllm_version() -> str:
418 if neuron_version != MAIN_CUDA_VERSION:
419 neuron_version_str = neuron_version.replace(".", "")[:3]
420 version += f"+neuron{neuron_version_str}"
415 - elif _is_cuda():
416 - cuda_version = str(nvcc_cuda_version)
417 - if cuda_version != MAIN_CUDA_VERSION:
418 - cuda_version_str = cuda_version.replace(".", "")[:3]
419 - version += f"+cu{cuda_version_str}"
421 else:
421 - raise RuntimeError("Unknown runtime environment.")
422 + raise RuntimeError("Unknown runtime environment")
423
424 return version
425
@@ -434,13 +435,7 @@ def read_readme() -> str:
435
436 def get_requirements() -> List[str]:
437 """Get Python package dependencies from requirements.txt."""
437 - if _is_hip():
438 - with open(get_path("requirements-rocm.txt")) as f:
439 - requirements = f.read().strip().split("\n")
440 - elif _is_neuron():
441 - with open(get_path("requirements-neuron.txt")) as f:
442 - requirements = f.read().strip().split("\n")
443 - else:
438 + if _is_cuda():
439 with open(get_path("requirements.txt")) as f:
440 requirements = f.read().strip().split("\n")
441 if nvcc_cuda_version <= Version("11.8"):
@@ -449,6 +444,16 @@ def get_requirements() -> List[str]:
444 if requirements[i].startswith("cupy-cuda12x"):
445 requirements[i] = "cupy-cuda11x"
446 break
447 + elif _is_hip():
448 + with open(get_path("requirements-rocm.txt")) as f:
449 + requirements = f.read().strip().split("\n")
450 + elif _is_neuron():
451 + with open(get_path("requirements-neuron.txt")) as f:
452 + requirements = f.read().strip().split("\n")
453 + else:
454 + raise ValueError(
455 + "Unsupported platform, please use CUDA, ROCM or Neuron.")
456 +
457 return requirements
458
459