A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix] Migrate to REGEX Library to prevent catastrophic backtracking (#18454)
Signed-off-by: Crucifixion-Fxl <xmufxl@gmail.com> Co-authored-by: Crucifixion-Fxl <xmufxl@gmail.com>
Feng XiaoLong committed
May 24, 2025 at 07:16 UTC
4fc1bf813ad80172c1db31264beaef7d93fe0601
52 files changed
+62
-58
.github/scripts/cleanup_pr_body.sh
+1
-1
@@ -26,7 +26,7 @@ sed -i '/\*\*BEFORE SUBMITTING, PLEASE READ.*\*\*/,$d' "${NEW}"
26
27
# Remove HTML <details> section that includes <summary> text of "PR Checklist (Click to Expand)"
28
python3 - <<EOF
29
-import re
29
+import regex as re
30
31
with open("${NEW}", "r") as file:
32
content = file.read()
benchmarks/benchmark_serving_structured_output.py
+2
-2
@@ -672,7 +672,7 @@ async def benchmark(
672
def evaluate(ret, args):
673
def _eval_correctness_json(expected, actual):
674
# extract json string from string using regex
675
- import re
675
+ import regex as re
676
677
actual = actual.replace("\n", "").replace(" ", "").strip()
678
try:
@@ -687,7 +687,7 @@ def evaluate(ret, args):
687
return actual in args.choice
688
689
def _eval_correctness_regex(expected, actual):
690
- import re
690
+ import regex as re
691
692
return re.match(args.regex, actual) is not None
693
benchmarks/kernels/graph_machete_bench.py
+1
-1
@@ -2,11 +2,11 @@
2
3
import math
4
import pickle
5
-import re
5
from collections import defaultdict
6
7
import matplotlib.pyplot as plt
8
import pandas as pd
9
+import regex as re
10
import seaborn as sns
11
from torch.utils.benchmark import Measurement as TMeasurement
12
examples/offline_inference/prithvi_geospatial_mae.py
+1
-1
@@ -20,12 +20,12 @@ python prithvi_geospatial_mae.py
20
import argparse
21
import datetime
22
import os
23
-import re
23
from typing import Union
24
25
import albumentations
26
import numpy as np
27
import rasterio
28
+import regex as re
29
import torch
30
from einops import rearrange
31
from terratorch.datamodules import Sen1Floods11NonGeoDataModule
pyproject.toml
+1
@@ -8,6 +8,7 @@ requires = [
8
"setuptools-scm>=8.0",
9
"torch == 2.7.0",
10
"wheel",
11
+ "regex",
12
"jinja2",
13
]
14
build-backend = "setuptools.build_meta"
requirements/build.txt
+1
@@ -7,3 +7,4 @@ setuptools-scm>=8
7
torch==2.7.0
8
wheel
9
jinja2>=3.1.6
10
+regex
requirements/common.txt
+1
@@ -1,3 +1,4 @@
1
+regex # Replace re for higher-performance regex matching
2
cachetools
3
psutil
4
sentencepiece # Required for LLaMA tokenizer.
requirements/nightly_torch_test.txt
+1
-1
@@ -38,4 +38,4 @@ matplotlib # required for qwen-vl test
38
# required for Multi-Modal Models Test (Standard)
39
num2words # required for smolvlm test
40
pqdm
41
-timm # required for internvl test
41
+timm # required for internvl test
\ No newline at end of file
setup.py
+1
-2
@@ -5,12 +5,12 @@ import importlib.util
5
import json
6
import logging
7
import os
8
-import re
8
import subprocess
9
import sys
10
from pathlib import Path
11
from shutil import which
12
13
+import regex as re
14
import torch
15
from packaging.version import Version, parse
16
from setuptools import Extension, setup
@@ -389,7 +389,6 @@ class repackage_wheel(build_ext):
389
# vllm_flash_attn python code:
390
# Regex from
391
# `glob.translate('vllm/vllm_flash_attn/**/*.py', recursive=True)`
392
- import re
392
compiled_regex = re.compile(
393
r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py")
394
file_members += list(
tests/entrypoints/llm/test_guided_generate.py
+1
-1
@@ -1,12 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
import weakref
5
from enum import Enum
6
7
import jsonschema
8
import pytest
9
+import regex as re
10
from pydantic import BaseModel
11
12
from vllm.distributed import cleanup_dist_env_and_memory
tests/entrypoints/openai/test_chat.py
+1
-1
@@ -2,13 +2,13 @@
2
3
# imports for guided decoding tests
4
import json
5
-import re
5
from typing import Optional
6
7
import jsonschema
8
import openai # use the official client for correctness check
9
import pytest
10
import pytest_asyncio
11
+import regex as re
12
import requests
13
import torch
14
from openai import BadRequestError, OpenAI
tests/entrypoints/openai/test_completion.py
+1
-2
@@ -1,8 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-
2
# imports for guided decoding tests
3
import json
5
-import re
4
import shutil
5
from tempfile import TemporaryDirectory
6
from typing import Optional
@@ -11,6 +9,7 @@ import jsonschema
9
import openai # use the official client for correctness check
10
import pytest
11
import pytest_asyncio
12
+import regex as re
13
# downloading lora to test lora requests
14
from huggingface_hub import snapshot_download
15
from openai import BadRequestError
tests/entrypoints/openai/test_prompt_validation.py
+6
-6
@@ -1,10 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
# imports for guided decoding tests
4
-import re
5
-
4
import openai
5
import pytest
6
+import regex as re
7
8
from ...utils import RemoteOpenAIServer
9
@@ -32,7 +31,7 @@ async def test_out_of_vocab_token_ids():
31
client = remote_server.get_async_client()
32
33
with pytest.raises(openai.BadRequestError,
35
- match=re.compile('.*out of vocabulary.*')):
34
+ match=re.compile('.*out of vocabulary.*').pattern):
35
await client.completions.create(model=model_name,
36
prompt=[999999],
37
max_tokens=5,
@@ -46,9 +45,10 @@ async def test_reject_multistep_with_guided_decoding():
45
with RemoteOpenAIServer(model_name, server_args) as remote_server:
46
client = remote_server.get_async_client()
47
49
- with pytest.raises(openai.BadRequestError,
50
- match=re.compile(
51
- '.*Guided decoding .* multi-step decoding.*')):
48
+ with pytest.raises(
49
+ openai.BadRequestError,
50
+ match=re.compile(
51
+ '.*Guided decoding .* multi-step decoding.*').pattern):
52
await client.completions.create(
53
model=model_name,
54
prompt="Hello",
tests/models/multimodal/generation/test_phi4mm.py
+1
-1
@@ -1,12 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import os
4
-import re
4
from collections.abc import Sequence
5
from typing import Optional
6
7
import librosa
8
import pytest
9
+import regex as re
10
from huggingface_hub import snapshot_download
11
from transformers import AutoTokenizer
12
tests/models/multimodal/generation/vlm_utils/model_utils.py
+1
-1
@@ -3,11 +3,11 @@
3
for manipulating the input / output of HF & vLLM test runners, which are
4
typically specific to a small subset of models.
5
"""
6
-import re
6
import types
7
from pathlib import PosixPath
8
from typing import Optional, Union
9
10
+import regex as re
11
import torch
12
from PIL.Image import Image
13
from transformers import (AutoConfig, AutoTokenizer, BatchFeature,
tests/tool_use/test_tool_choice_required.py
+2
-2
@@ -1,10 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
import json
3
-import re
3
from copy import deepcopy
4
from unittest.mock import MagicMock
5
6
import pytest
7
+import regex as re
8
from pydantic import TypeAdapter
9
10
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -333,4 +333,4 @@ def test_streaming_output_valid(output, empty_params, delta_len):
333
combined_messages += message.tool_calls[0].function.arguments
334
combined_messages += "}]"
335
assert json.loads(combined_messages) == output
336
- assert json.dumps(json.loads(combined_messages)) == output_json
336
+ assert json.dumps(json.loads(combined_messages)) == output_json
\ No newline at end of file
tests/v1/entrypoints/llm/test_struct_output_generate.py
+1
-1
@@ -4,12 +4,12 @@
4
from __future__ import annotations
5
6
import json
7
-import re
7
from enum import Enum
8
from typing import TYPE_CHECKING, Any
9
10
import jsonschema
11
import pytest
12
+import regex as re
13
from pydantic import BaseModel
14
15
from tests.reasoning.utils import run_reasoning_extraction
tests/v1/entrypoints/openai/test_completion.py
+1
-1
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
from typing import Optional
4
5
import openai # use the official client for correctness check
6
import pytest
7
import pytest_asyncio
8
+import regex as re
9
from openai import BadRequestError
10
11
from tests.utils import RemoteOpenAIServer
tests/v1/sample/utils.py
+2
-1
@@ -1,9 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
from enum import Enum
4
from typing import Optional
5
6
+import regex as re
7
+
8
from vllm import CompletionOutput
9
10
vllm/collect_env.py
+1
-1
@@ -815,4 +815,4 @@ def main():
815
816
817
if __name__ == '__main__':
818
- main()
818
+ main()
\ No newline at end of file
vllm/config.py
+1
-1
@@ -6,7 +6,6 @@ import enum
6
import hashlib
7
import inspect
8
import json
9
-import re
9
import textwrap
10
import uuid
11
import warnings
@@ -20,6 +19,7 @@ from pathlib import Path
19
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Literal, Optional,
20
Protocol, TypeVar, Union, cast, get_args, get_origin)
21
22
+import regex as re
23
import torch
24
from torch.distributed import ProcessGroup, ReduceOp
25
from transformers import PretrainedConfig
vllm/engine/arg_utils.py
+1
-1
@@ -4,7 +4,6 @@
4
import argparse
5
import dataclasses
6
import json
7
-import re
7
import sys
8
import threading
9
import warnings
@@ -13,6 +12,7 @@ from itertools import permutations
12
from typing import (Annotated, Any, Callable, Dict, List, Literal, Optional,
13
Type, TypeVar, Union, cast, get_args, get_origin)
14
15
+import regex as re
16
import torch
17
from typing_extensions import TypeIs, deprecated
18
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -7,7 +7,6 @@ import importlib
7
import inspect
8
import multiprocessing
9
import os
10
-import re
10
import signal
11
import socket
12
import tempfile
@@ -21,6 +20,7 @@ from json import JSONDecodeError
20
from typing import Annotated, Optional, Union
21
22
import prometheus_client
23
+import regex as re
24
import uvloop
25
from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
26
from fastapi.exceptions import RequestValidationError
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -3,11 +3,11 @@
3
# Adapted from
4
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
5
import json
6
-import re
6
import time
7
from http import HTTPStatus
8
from typing import Annotated, Any, ClassVar, Literal, Optional, Union
9
10
+import regex as re
11
import torch
12
from fastapi import HTTPException, UploadFile
13
from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -2,7 +2,6 @@
2
3
import asyncio
4
import json
5
-import re
5
import time
6
from collections.abc import AsyncGenerator, AsyncIterator
7
from collections.abc import Sequence as GenericSequence
@@ -10,6 +9,7 @@ from typing import Callable, Final, Optional, Union
9
10
import jinja2
11
import partial_json_parser
12
+import regex as re
13
from fastapi import Request
14
from pydantic import TypeAdapter
15
vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py
+2
-1
@@ -1,9 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
from collections.abc import Sequence
4
from typing import Union
5
6
+import regex as re
7
+
8
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9
DeltaFunctionCall, DeltaMessage,
10
DeltaToolCall,
vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py
+1
-1
@@ -1,12 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
from collections.abc import Sequence
5
from json import JSONDecoder
6
from typing import Union
7
8
import partial_json_parser
9
+import regex as re
10
from partial_json_parser.core.options import Allow
11
12
from vllm.entrypoints.chat_utils import random_tool_call_id
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+1
-1
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
from collections.abc import Sequence
5
from typing import Union
6
7
import partial_json_parser
8
+import regex as re
9
from partial_json_parser.core.options import Allow
10
11
from vllm.entrypoints.chat_utils import random_tool_call_id
vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py
+1
-1
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
from collections.abc import Sequence
5
from typing import Union
6
7
import partial_json_parser
8
+import regex as re
9
from partial_json_parser.core.options import Allow
10
11
from vllm.entrypoints.chat_utils import random_tool_call_id
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+1
-1
@@ -1,12 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
from collections.abc import Sequence
5
from json import JSONDecoder
6
from typing import Union
7
8
import partial_json_parser
9
+import regex as re
10
from partial_json_parser.core.options import Allow
11
from transformers import PreTrainedTokenizerBase
12
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1
-1
@@ -1,13 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
from collections.abc import Sequence
5
from random import choices
6
from string import ascii_letters, digits
7
from typing import Union
8
9
import partial_json_parser
10
+import regex as re
11
from partial_json_parser.core.options import Allow
12
from pydantic import Field
13
vllm/entrypoints/openai/tool_parsers/phi4mini_tool_parser.py
+1
-1
@@ -1,10 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-import re
4
from collections.abc import Sequence
5
from typing import Any, Optional
6
7
+import regex as re
8
from transformers import PreTrainedTokenizerBase
9
10
from vllm.entrypoints.chat_utils import random_tool_call_id
vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py
+1
-1
@@ -2,10 +2,10 @@
2
3
import ast
4
import json
5
-import re
5
from collections.abc import Sequence
6
from typing import Any, Union
7
8
+import regex as re
9
from transformers import PreTrainedTokenizerBase
10
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
vllm/lora/models.py
+1
-1
@@ -3,11 +3,11 @@
3
import copy
4
import math
5
import os
6
-import re
6
from collections.abc import Sequence
7
from dataclasses import dataclass, field
8
from typing import Any, Callable, Optional, Union
9
10
+import regex as re
11
import safetensors.torch
12
import torch
13
from torch import nn
vllm/lora/utils.py
+1
-1
@@ -1,10 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import os
4
-import re
4
from typing import Optional, Union
5
6
import huggingface_hub
7
+import regex as re
8
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
9
HFValidationError, RepositoryNotFoundError)
10
from torch import nn
vllm/model_executor/guided_decoding/utils.py
+1
-1
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
+import regex as re
4
5
6
def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+1
-1
@@ -4,10 +4,10 @@
4
from __future__ import annotations
5
6
import json
7
-import re
7
from dataclasses import dataclass, field
8
from typing import TYPE_CHECKING, Any
9
10
+import regex as re
11
import torch
12
13
import vllm.envs
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+1
-1
@@ -1,10 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
from collections.abc import Iterable, Mapping
4
from types import MappingProxyType
5
from typing import Optional
6
7
+import regex as re
8
from compressed_tensors import CompressionFormat
9
from torch.nn import Module
10
vllm/model_executor/layers/quantization/modelopt.py
+1
-1
@@ -228,7 +228,7 @@ class ModelOptNvFp4Config(QuantizationConfig):
228
exclude_modules, group_size)
229
230
def is_layer_excluded(self, prefix: str, exclude_modules: list):
231
- import re
231
+ import regex as re
232
for pattern in exclude_modules:
233
regex_str = pattern.replace('.', r'\.').replace('*', r'.*')
234
if re.fullmatch(regex_str, prefix):
vllm/model_executor/layers/quantization/quark/utils.py
+2
-1
@@ -1,10 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
from collections.abc import Iterable, Mapping
4
from types import MappingProxyType
5
from typing import Any, Optional
6
7
+import regex as re
8
+
9
10
def deep_compare(dict1: Any, dict2: Any) -> bool:
11
if type(dict1) is not type(dict2):
vllm/model_executor/layers/quantization/utils/gptq_utils.py
+1
-1
@@ -1,8 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-import re
2
from copy import deepcopy
3
from typing import Optional, Union
4
5
+import regex as re
6
import torch
7
8
from vllm.config import QuantizationConfig
vllm/model_executor/model_loader/tensorizer.py
+1
-1
@@ -7,7 +7,6 @@ import dataclasses
7
import io
8
import json
9
import os
10
-import re
10
import threading
11
import time
12
from collections.abc import Generator
@@ -15,6 +14,7 @@ from dataclasses import dataclass
14
from functools import partial
15
from typing import Any, BinaryIO, Optional, Union
16
17
+import regex as re
18
import torch
19
from torch import nn
20
from torch.utils._python_dispatch import TorchDispatchMode
vllm/model_executor/models/mimo_mtp.py
+1
-1
@@ -250,7 +250,7 @@ class MiMoMTP(nn.Module):
250
return loaded_params
251
252
def map_model_name_to_mtp_param_name(self, name: str) -> str:
253
- import re
253
+ import regex as re
254
name_without_prefix = [
255
"token_layernorm", "hidden_layernorm", "input_proj",
256
"final_layernorm"
vllm/model_executor/models/minimax_text_01.py
+1
-1
@@ -2,10 +2,10 @@
2
"""Inference-only MiniMaxText01 model."""
3
import copy
4
import math
5
-import re
5
from collections.abc import Iterable
6
from typing import Optional, Union
7
8
+import regex as re
9
import torch
10
import torch.distributed
11
import torch.nn.functional as F
vllm/model_executor/models/phi3v.py
+1
-1
@@ -14,10 +14,10 @@
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
-import re
17
from collections.abc import Iterable, Mapping, Sequence
18
from typing import Any, Literal, Optional, TypedDict, Union
19
20
+import regex as re
21
import torch
22
import torch.nn as nn
23
from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,
vllm/model_executor/models/qwen_vl.py
+1
-1
@@ -7,12 +7,12 @@
7
8
import copy
9
import math
10
-import re
10
import unicodedata
11
from collections.abc import Collection, Mapping, Sequence, Set
12
from functools import lru_cache, partial
13
from typing import Callable, Literal, Optional, TypedDict, Union
14
15
+import regex as re
16
import torch
17
from torch import nn
18
from torchvision import transforms
vllm/model_executor/models/transformers.py
+1
-1
@@ -14,11 +14,11 @@
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
16
"""Wrapper around `transformers` models"""
17
-import re
17
from collections.abc import Iterable
18
from contextlib import nullcontext
19
from typing import Literal, Optional, Union
20
21
+import regex as re
22
import torch
23
from torch import nn
24
from transformers import AutoModel, PretrainedConfig, PreTrainedModel
vllm/multimodal/processing.py
+1
-1
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
import json
3
-import re
3
import sys
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
@@ -12,6 +11,7 @@ from functools import lru_cache
11
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
12
TypeVar, Union, cast)
13
14
+import regex as re
15
import torch
16
from typing_extensions import assert_never
17
vllm/reasoning/granite_reasoning_parser.py
+1
-1
@@ -1,9 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import re
3
from collections.abc import Sequence
4
from typing import Optional, Union
5
6
+import regex as re
7
from transformers import PreTrainedTokenizerBase
8
9
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
vllm/transformers_utils/tokenizers/mistral.py
+1
-1
@@ -1,12 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import os
4
-import re
4
from dataclasses import dataclass
5
from pathlib import Path
6
from typing import TYPE_CHECKING, Any, Optional, Union, cast
7
8
import huggingface_hub
9
+import regex as re
10
from huggingface_hub import HfApi, hf_hub_download
11
12
from vllm.logger import init_logger
vllm/utils.py
+1
-1
@@ -19,7 +19,6 @@ import json
19
import multiprocessing
20
import os
21
import pickle
22
-import re
22
import signal
23
import socket
24
import subprocess
@@ -54,6 +53,7 @@ import cloudpickle
53
import numpy as np
54
import numpy.typing as npt
55
import psutil
56
+import regex as re
57
import torch
58
import torch.types
59
import yaml
vllm/v1/structured_output/utils.py
+1
-1
@@ -2,7 +2,7 @@
2
3
from __future__ import annotations
4
5
-import re
5
+import regex as re
6
7
8
def grammar_is_likely_lark(grammar_str: str) -> bool: