A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Only require XGrammar on x86 (#10865)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 3, 2024 at 13:32 UTC
7090c27bb2cb0d9c4e0acd644e484291df3aff2a
4 files changed
+36
-3
requirements-common.txt
+1
-1
@@ -19,7 +19,7 @@ prometheus-fastapi-instrumentator >= 7.0.0
19
tiktoken >= 0.6.0 # Required for DBRX tokenizer
20
lm-format-enforcer >= 0.10.9, < 0.11
21
outlines >= 0.0.43, < 0.1
22
-xgrammar
22
+xgrammar >= 0.1.5; platform_machine == "x86_64"
23
typing_extensions >= 4.10
24
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
25
partial-json-parser # used for parsing partial JSON outputs
vllm/model_executor/guided_decoding/__init__.py
+7
@@ -3,6 +3,7 @@ from __future__ import annotations
3
from typing import TYPE_CHECKING
4
5
from vllm.logger import init_logger
6
+from vllm.platforms import CpuArchEnum, current_platform
7
8
if TYPE_CHECKING:
9
from transformers import PreTrainedTokenizer
@@ -25,6 +26,12 @@ def maybe_backend_fallback(
26
guided_params.backend = "xgrammar"
27
28
if guided_params.backend == "xgrammar":
29
+ # xgrammar only has x86 wheels for linux, fallback to outlines
30
+ if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
31
+ logger.warning("xgrammar is only supported on x86 CPUs. "
32
+ "Falling back to use outlines instead.")
33
+ guided_params.backend = "outlines"
34
+
35
# xgrammar doesn't support regex or choice, fallback to outlines
36
if guided_params.regex is not None or guided_params.choice is not None:
37
logger.warning(
vllm/platforms/__init__.py
+2
-2
@@ -1,5 +1,5 @@
1
from .interface import _Backend # noqa: F401
2
-from .interface import Platform, PlatformEnum, UnspecifiedPlatform
2
+from .interface import CpuArchEnum, Platform, PlatformEnum, UnspecifiedPlatform
3
4
current_platform: Platform
5
@@ -120,4 +120,4 @@ elif is_openvino:
120
else:
121
current_platform = UnspecifiedPlatform()
122
123
-__all__ = ['Platform', 'PlatformEnum', 'current_platform']
123
+__all__ = ['Platform', 'PlatformEnum', 'current_platform', 'CpuArchEnum']
vllm/platforms/interface.py
+26
@@ -1,4 +1,5 @@
1
import enum
2
+import platform
3
import random
4
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
5
@@ -37,6 +38,14 @@ class PlatformEnum(enum.Enum):
38
UNSPECIFIED = enum.auto()
39
40
41
+class CpuArchEnum(enum.Enum):
42
+ X86 = enum.auto()
43
+ ARM = enum.auto()
44
+ POWERPC = enum.auto()
45
+ OTHER = enum.auto()
46
+ UNKNOWN = enum.auto()
47
+
48
+
49
class DeviceCapability(NamedTuple):
50
major: int
51
minor: int
@@ -184,6 +193,23 @@ class Platform:
193
f"{quant} quantization is currently not supported in "
194
f"{cls.device_name}.")
195
196
+ @classmethod
197
+ def get_cpu_architecture(cls) -> CpuArchEnum:
198
+ """
199
+ Determine the CPU architecture of the current system.
200
+ Returns CpuArchEnum indicating the architecture type.
201
+ """
202
+ machine = platform.machine().lower()
203
+
204
+ if machine in ("x86_64", "amd64", "i386", "i686"):
205
+ return CpuArchEnum.X86
206
+ elif machine.startswith("arm") or machine.startswith("aarch"):
207
+ return CpuArchEnum.ARM
208
+ elif machine.startswith("ppc"):
209
+ return CpuArchEnum.POWERPC
210
+
211
+ return CpuArchEnum.OTHER if machine else CpuArchEnum.UNKNOWN
212
+
213
214
class UnspecifiedPlatform(Platform):
215
_enum = PlatformEnum.UNSPECIFIED