A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
Disable outlines cache by default (#14837)
Russell Bryant committed
Mar 14, 2025 at 23:57 UTC
776dcec8fe51860d7580001de86216406629df0f
2 files changed
+16
-1
vllm/envs.py
+7
@@ -95,6 +95,7 @@ if TYPE_CHECKING:
95
VLLM_DP_MASTER_IP: str = ""
96
VLLM_DP_MASTER_PORT: int = 0
97
VLLM_MARLIN_USE_ATOMIC_ADD: bool = False
98
+ VLLM_V0_USE_OUTLINES_CACHE: bool = False
99
100
101
def get_default_cache_root():
@@ -623,6 +624,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
624
# Whether to use atomicAdd reduce in gptq/awq marlin kernel.
625
"VLLM_MARLIN_USE_ATOMIC_ADD":
626
lambda: os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1",
627
+
628
+ # Whether to turn on the outlines cache for V0
629
+ # This cache is unbounded and on disk, so it's not safe to use in
630
+ # an environment with potentially malicious users.
631
+ "VLLM_V0_USE_OUTLINES_CACHE":
632
+ lambda: os.environ.get("VLLM_V0_USE_OUTLINES_CACHE", "0") == "1",
633
}
634
635
# end-env-vars-definition
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+9
-1
@@ -24,7 +24,7 @@ from typing import Callable, DefaultDict, Dict, List, Optional, Union
24
import numpy as np
25
import torch
26
from outlines import grammars
27
-from outlines.caching import cache
27
+from outlines.caching import cache, disable_cache
28
from outlines.fsm.guide import (CFGGuide, CFGState, Generate, Guide,
29
RegexGuide, Write)
30
from outlines.fsm.parsing import PartialLark
@@ -32,12 +32,20 @@ from outlines_core.fsm.json_schema import build_regex_from_schema
32
from pydantic import BaseModel
33
from transformers import PreTrainedTokenizerBase
34
35
+import vllm.envs as envs
36
from vllm.logger import init_logger
37
from vllm.model_executor.guided_decoding.reasoner import Reasoner
38
from vllm.platforms import current_platform
39
40
logger = init_logger(__name__)
41
42
+if envs.VLLM_V0_USE_OUTLINES_CACHE:
43
+ logger.warning("Enabling outlines cache. This is an unbounded on-disk "
44
+ "cache. It may consume a lot of disk space and should "
45
+ "not be used with untrusted clients.")
46
+else:
47
+ disable_cache()
48
+
49
50
class BaseLogitsProcessor:
51