@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7090c27bb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Only require XGrammar on x86 (#10865)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 3, 2024 at 13:32 UTC 7090c27bb2cb0d9c4e0acd644e484291df3aff2a
4 files changed +36 -3
requirements-common.txt
+1 -1
@@ -19,7 +19,7 @@ prometheus-fastapi-instrumentator >= 7.0.0
19 tiktoken >= 0.6.0 # Required for DBRX tokenizer
20 lm-format-enforcer >= 0.10.9, < 0.11
21 outlines >= 0.0.43, < 0.1
22 -xgrammar
22 +xgrammar >= 0.1.5; platform_machine == "x86_64"
23 typing_extensions >= 4.10
24 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
25 partial-json-parser # used for parsing partial JSON outputs
vllm/model_executor/guided_decoding/__init__.py
+7
@@ -3,6 +3,7 @@ from __future__ import annotations
3 from typing import TYPE_CHECKING
4
5 from vllm.logger import init_logger
6 +from vllm.platforms import CpuArchEnum, current_platform
7
8 if TYPE_CHECKING:
9 from transformers import PreTrainedTokenizer
@@ -25,6 +26,12 @@ def maybe_backend_fallback(
26 guided_params.backend = "xgrammar"
27
28 if guided_params.backend == "xgrammar":
29 + # xgrammar only has x86 wheels for linux, fallback to outlines
30 + if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
31 + logger.warning("xgrammar is only supported on x86 CPUs. "
32 + "Falling back to use outlines instead.")
33 + guided_params.backend = "outlines"
34 +
35 # xgrammar doesn't support regex or choice, fallback to outlines
36 if guided_params.regex is not None or guided_params.choice is not None:
37 logger.warning(
vllm/platforms/__init__.py
+2 -2
@@ -1,5 +1,5 @@
1 from .interface import _Backend # noqa: F401
2 -from .interface import Platform, PlatformEnum, UnspecifiedPlatform
2 +from .interface import CpuArchEnum, Platform, PlatformEnum, UnspecifiedPlatform
3
4 current_platform: Platform
5
@@ -120,4 +120,4 @@ elif is_openvino:
120 else:
121 current_platform = UnspecifiedPlatform()
122
123 -__all__ = ['Platform', 'PlatformEnum', 'current_platform']
123 +__all__ = ['Platform', 'PlatformEnum', 'current_platform', 'CpuArchEnum']
vllm/platforms/interface.py
+26
@@ -1,4 +1,5 @@
1 import enum
2 +import platform
3 import random
4 from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
5
@@ -37,6 +38,14 @@ class PlatformEnum(enum.Enum):
38 UNSPECIFIED = enum.auto()
39
40
41 +class CpuArchEnum(enum.Enum):
42 + X86 = enum.auto()
43 + ARM = enum.auto()
44 + POWERPC = enum.auto()
45 + OTHER = enum.auto()
46 + UNKNOWN = enum.auto()
47 +
48 +
49 class DeviceCapability(NamedTuple):
50 major: int
51 minor: int
@@ -184,6 +193,23 @@ class Platform:
193 f"{quant} quantization is currently not supported in "
194 f"{cls.device_name}.")
195
196 + @classmethod
197 + def get_cpu_architecture(cls) -> CpuArchEnum:
198 + """
199 + Determine the CPU architecture of the current system.
200 + Returns CpuArchEnum indicating the architecture type.
201 + """
202 + machine = platform.machine().lower()
203 +
204 + if machine in ("x86_64", "amd64", "i386", "i686"):
205 + return CpuArchEnum.X86
206 + elif machine.startswith("arm") or machine.startswith("aarch"):
207 + return CpuArchEnum.ARM
208 + elif machine.startswith("ppc"):
209 + return CpuArchEnum.POWERPC
210 +
211 + return CpuArchEnum.OTHER if machine else CpuArchEnum.UNKNOWN
212 +
213
214 class UnspecifiedPlatform(Platform):
215 _enum = PlatformEnum.UNSPECIFIED