A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix setup.py neuron-ls issue (#2671)
Simon Mo committed
Mar 16, 2024 at 16:00 UTC
6b78837b29b5045a71e6ecfa68442b1f4fd2d0a6
1 file changed
+24
-19
setup.py
+24
-19
@@ -2,6 +2,7 @@ import contextlib
2
import io
3
import os
4
import re
5
+import shutil
6
import subprocess
7
import warnings
8
from pathlib import Path
@@ -38,6 +39,10 @@ ROCM_SUPPORTED_ARCHS = {"gfx908", "gfx90a", "gfx942", "gfx1100"}
39
# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
40
41
42
+def _is_cuda() -> bool:
43
+ return torch.version.cuda is not None
44
+
45
+
46
def _is_hip() -> bool:
47
return torch.version.hip is not None
48
@@ -46,15 +51,11 @@ def _is_neuron() -> bool:
51
torch_neuronx_installed = True
52
try:
53
subprocess.run(["neuron-ls"], capture_output=True, check=True)
49
- except (FileNotFoundError, PermissionError):
54
+ except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
55
torch_neuronx_installed = False
56
return torch_neuronx_installed
57
58
54
-def _is_cuda() -> bool:
55
- return (torch.version.cuda is not None) and not _is_neuron()
56
-
57
-
59
# Compiler flags.
60
CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
61
# TODO(woosuk): Should we use -O3?
@@ -400,7 +401,12 @@ def find_version(filepath: str) -> str:
401
def get_vllm_version() -> str:
402
version = find_version(get_path("vllm", "__init__.py"))
403
403
- if _is_hip():
404
+ if _is_cuda():
405
+ cuda_version = str(nvcc_cuda_version)
406
+ if cuda_version != MAIN_CUDA_VERSION:
407
+ cuda_version_str = cuda_version.replace(".", "")[:3]
408
+ version += f"+cu{cuda_version_str}"
409
+ elif _is_hip():
410
# Get the HIP version
411
hipcc_version = get_hipcc_rocm_version()
412
if hipcc_version != MAIN_CUDA_VERSION:
@@ -412,13 +418,8 @@ def get_vllm_version() -> str:
418
if neuron_version != MAIN_CUDA_VERSION:
419
neuron_version_str = neuron_version.replace(".", "")[:3]
420
version += f"+neuron{neuron_version_str}"
415
- elif _is_cuda():
416
- cuda_version = str(nvcc_cuda_version)
417
- if cuda_version != MAIN_CUDA_VERSION:
418
- cuda_version_str = cuda_version.replace(".", "")[:3]
419
- version += f"+cu{cuda_version_str}"
421
else:
421
- raise RuntimeError("Unknown runtime environment.")
422
+ raise RuntimeError("Unknown runtime environment")
423
424
return version
425
@@ -434,13 +435,7 @@ def read_readme() -> str:
435
436
def get_requirements() -> List[str]:
437
"""Get Python package dependencies from requirements.txt."""
437
- if _is_hip():
438
- with open(get_path("requirements-rocm.txt")) as f:
439
- requirements = f.read().strip().split("\n")
440
- elif _is_neuron():
441
- with open(get_path("requirements-neuron.txt")) as f:
442
- requirements = f.read().strip().split("\n")
443
- else:
438
+ if _is_cuda():
439
with open(get_path("requirements.txt")) as f:
440
requirements = f.read().strip().split("\n")
441
if nvcc_cuda_version <= Version("11.8"):
@@ -449,6 +444,16 @@ def get_requirements() -> List[str]:
444
if requirements[i].startswith("cupy-cuda12x"):
445
requirements[i] = "cupy-cuda11x"
446
break
447
+ elif _is_hip():
448
+ with open(get_path("requirements-rocm.txt")) as f:
449
+ requirements = f.read().strip().split("\n")
450
+ elif _is_neuron():
451
+ with open(get_path("requirements-neuron.txt")) as f:
452
+ requirements = f.read().strip().split("\n")
453
+ else:
454
+ raise ValueError(
455
+ "Unsupported platform, please use CUDA, ROCM or Neuron.")
456
+
457
return requirements
458
459