@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 185ad31f37
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] use diskcache in outlines _get_guide #5436 (#6203)

Eric committed Jul 9, 2024 at 02:23 UTC 185ad31f37541ac205b55f446bfd71542f83075a
1 file changed +3 -2
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+3 -2
@@ -21,6 +21,7 @@ from functools import lru_cache
21 from typing import Callable, DefaultDict, Dict, List, Union
22
23 import torch
24 +from outlines.caching import cache
25 from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
26 from outlines.fsm.json_schema import build_regex_from_schema
27 from pydantic import BaseModel
@@ -67,7 +68,7 @@ class BaseLogitsProcessor:
68 class RegexLogitsProcessor(BaseLogitsProcessor):
69
70 @classmethod
70 - @lru_cache(maxsize=32)
71 + @cache()
72 def _get_guide(cls, regex_string: str,
73 tokenizer: PreTrainedTokenizerBase) -> Guide:
74 tokenizer = _adapt_tokenizer(tokenizer)
@@ -126,7 +127,7 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
127 class CFGLogitsProcessor(BaseLogitsProcessor):
128
129 @classmethod
129 - @lru_cache(maxsize=32)
130 + @cache()
131 def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide:
132 tokenizer = _adapt_tokenizer(tokenizer)
133 return CFGGuide(cfg, tokenizer)