A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Refactor system architecture (#82)
Woosuk Kwon committed
May 9, 2023 at 15:30 UTC
7c041ab5784760416f85d68eb8925a1d1f981932
40 files changed
+195
-447
cacheflow/core/block_manager.py
renamed
cacheflow/core/policy.py
renamed
cacheflow/core/scheduler.py
renamed
+2
-2
@@ -4,8 +4,8 @@ import pickle
4
import time
5
from typing import Any, Dict, List, Optional, Tuple
6
7
-from cacheflow.master.block_manager import BlockSpaceManager
8
-from cacheflow.master.policy import PolicyFactory
7
+from cacheflow.core.block_manager import BlockSpaceManager
8
+from cacheflow.core.policy import PolicyFactory
9
from cacheflow.sampling_params import SamplingParams
10
from cacheflow.sequence import Sequence
11
from cacheflow.sequence import SequenceGroup
cacheflow/core/server.py
renamed
+5
-4
@@ -8,20 +8,21 @@ try:
8
except ImportError:
9
ray = None
10
11
+from cacheflow.core.scheduler import Scheduler
12
+from cacheflow.frontend.simple_frontend import SimpleFrontend
13
from cacheflow.logger import init_logger
12
-from cacheflow.master.scheduler import Scheduler
13
-from cacheflow.master.simple_frontend import SimpleFrontend
14
-from cacheflow.models import get_memory_analyzer
15
-from cacheflow.worker.controller import Controller, DeviceID
14
+from cacheflow.model_executor import get_memory_analyzer
15
from cacheflow.sequence import SequenceGroup
16
from cacheflow.sampling_params import SamplingParams
17
from cacheflow.utils import get_gpu_memory, get_cpu_memory
18
+from cacheflow.worker.controller import Controller, DeviceID
19
20
21
logger = init_logger(__name__)
22
23
24
class Server:
25
+
26
def __init__(
27
self,
28
model: str,
cacheflow/frontend/fastapi_frontend.py
renamed
+7
-7
@@ -1,22 +1,22 @@
1
import argparse
2
import asyncio
3
+import json
4
import time
5
from typing import List, Dict, Optional
5
-import json
6
7
-import ray
8
-from transformers import AutoTokenizer
7
from fastapi import FastAPI, Request
8
from fastapi.responses import StreamingResponse
9
+import ray
10
+from transformers import AutoTokenizer
11
import uvicorn
12
13
+from cacheflow.core.server import (Server, add_server_arguments,
14
+ process_server_arguments,
15
+ initialize_cluster)
16
from cacheflow.sampling_params import SamplingParams
17
from cacheflow.sequence import Sequence, SequenceGroup
15
-from cacheflow.master.server import (Server, add_server_arguments,
16
- process_server_arguments,
17
- initialize_cluster)
18
-from cacheflow.worker.controller import DeviceID
18
from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
19
+from cacheflow.worker.controller import DeviceID
20
21
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
22
app = FastAPI()
cacheflow/frontend/simple_frontend.py
renamed
cacheflow/model_executor/__init__.py
new
+11
@@ -0,0 +1,11 @@
1
+from cacheflow.model_executor.input_metadata import InputMetadata
2
+from cacheflow.model_executor.model_loader import get_model, get_memory_analyzer
3
+from cacheflow.model_executor.utils import set_random_seed
4
+
5
+
6
+__all__ = [
7
+ "InputMetadata",
8
+ "get_model",
9
+ "get_memory_analyzer",
10
+ "set_random_seed",
11
+]
cacheflow/model_executor/input_metadata.py
renamed
cacheflow/model_executor/layers/activation.py
renamed
cacheflow/model_executor/layers/attention.py
renamed
+1
-1
@@ -7,7 +7,7 @@ from xformers import ops as xops
7
from cacheflow import attention_ops
8
from cacheflow import cache_ops
9
from cacheflow import pos_encoding_ops
10
-from cacheflow.models import InputMetadata
10
+from cacheflow.model_executor.input_metadata import InputMetadata
11
12
13
class GPTCacheFlowAttention(nn.Module):
cacheflow/model_executor/layers/layernorm.py
renamed
cacheflow/model_executor/layers/sampler.py
renamed
+4
-3
@@ -3,10 +3,11 @@ from typing import Dict, List, Tuple
3
import torch
4
import torch.nn as nn
5
6
-from cacheflow.models import InputMetadata
6
+from cacheflow.model_executor.input_metadata import InputMetadata
7
+from cacheflow.model_executor.parallel_utils.tensor_parallel import (
8
+ gather_from_tensor_model_parallel_region)
9
from cacheflow.sampling_params import SamplingParams
10
from cacheflow.sequence import SequenceOutputs
9
-from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_parallel_region
11
12
13
class Sampler(nn.Module):
@@ -27,7 +28,7 @@ class Sampler(nn.Module):
28
# Get the logits for the next tokens.
29
logits = torch.matmul(hidden_states, embedding.t())
30
logits = gather_from_tensor_model_parallel_region(logits)
30
- # Remove paddings in vocab.
31
+ # Remove paddings in vocab (if any).
32
logits = logits[:, :self.vocab_size]
33
34
# Apply temperature scaling.
cacheflow/model_executor/memory_analyzer.py
renamed
+1
-1
@@ -2,7 +2,7 @@ import torch
2
from transformers import AutoConfig
3
4
from cacheflow.logger import init_logger
5
-from cacheflow.models.utils import get_dtype_size
5
+from cacheflow.model_executor.utils import get_dtype_size
6
7
8
logger = init_logger(__name__)
cacheflow/model_executor/model_loader.py
renamed
+8
-11
@@ -5,16 +5,13 @@ import torch.nn as nn
5
from transformers import AutoConfig
6
from transformers import PretrainedConfig
7
8
-from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9
-from cacheflow.models.memory_analyzer import GPT2MemoryAnalyzer
10
-from cacheflow.models.memory_analyzer import GPTNeoXMemoryAnalyzer
11
-from cacheflow.models.memory_analyzer import LlamaMemoryAnalyzer
12
-from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
13
-from cacheflow.models.gpt2 import GPT2LMHeadModel
14
-from cacheflow.models.gpt_neox import GPTNeoXForCausalLM
15
-from cacheflow.models.llama import LlamaForCausalLM
16
-from cacheflow.models.opt import OPTForCausalLM
17
-from cacheflow.models.utils import get_torch_dtype
8
+from cacheflow.model_executor.memory_analyzer import (
9
+ CacheFlowMemoryAnalyzer, GPT2MemoryAnalyzer, GPTNeoXMemoryAnalyzer,
10
+ LlamaMemoryAnalyzer, OPTMemoryAnalyzer)
11
+from cacheflow.model_executor.models import (
12
+ GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
13
+from cacheflow.model_executor.utils import get_torch_dtype
14
+from cacheflow.model_executor.weight_utils import initialize_dummy_weights
15
16
17
_MODELS = {
@@ -77,7 +74,7 @@ def get_model(
74
model = model.cuda()
75
# NOTE(woosuk): For precise performance evaluation, we assign
76
# random values to the weights.
80
- model.initialize_dummy_weights()
77
+ initialize_dummy_weights(model)
78
else:
79
# Create a model instance.
80
model = model_class(config)
cacheflow/model_executor/models/__init__.py
new
+12
@@ -0,0 +1,12 @@
1
+from cacheflow.model_executor.models.gpt_neox import GPTNeoXForCausalLM
2
+from cacheflow.model_executor.models.gpt2 import GPT2LMHeadModel
3
+from cacheflow.model_executor.models.llama import LlamaForCausalLM
4
+from cacheflow.model_executor.models.opt import OPTForCausalLM
5
+
6
+
7
+__all__ = [
8
+ "GPT2LMHeadModel",
9
+ "GPTNeoXForCausalLM",
10
+ "LlamaForCausalLM",
11
+ "OPTForCausalLM",
12
+]
cacheflow/model_executor/models/gpt2.py
renamed
+10
-14
@@ -5,16 +5,15 @@ import torch
5
from torch import nn
6
from transformers import GPT2Config
7
8
-from cacheflow.models import InputMetadata
9
-from cacheflow.models.attention import GPTCacheFlowAttention
10
-from cacheflow.models.sample import Sampler
11
-from cacheflow.models.utils import (hf_model_weights_iterator,
12
- load_tensor_parallel_weights)
13
-from cacheflow.parallel_utils.parallel_state import (
8
+from cacheflow.model_executor.input_metadata import InputMetadata
9
+from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
10
+from cacheflow.model_executor.layers.sampler import Sampler
11
+from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
12
+ load_tensor_parallel_weights)
13
+from cacheflow.model_executor.parallel_utils.parallel_state import (
14
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15
-from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
16
- ColumnParallelLinear,
17
- RowParallelLinear)
15
+from cacheflow.model_executor.parallel_utils.tensor_parallel import (
16
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
17
from cacheflow.sequence import SequenceOutputs
18
19
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -258,8 +257,5 @@ class GPT2LMHeadModel(nn.Module):
257
raise ValueError(f"Unexpected parameter name {name}")
258
load_tensor_parallel_weights(param, loaded_weight, name,
259
self._column_parallel_weights,
261
- self._row_parallel_weights)
262
-
263
- def initialize_dummy_weights(self) -> None:
264
- for param in self.state_dict().values():
265
- param.data.uniform_(-1e-3, 1e-3)
260
+ self._row_parallel_weights,
261
+ tensor_model_parallel_rank)
cacheflow/model_executor/models/gpt_neox.py
renamed
+16
-19
@@ -3,17 +3,17 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
from torch import nn
6
-
7
-from cacheflow.models import InputMetadata
8
-from cacheflow.models.attention import GPTNeoXCacheFlowAttention
9
-from cacheflow.models.sample import Sampler
10
-from cacheflow.models.utils import (hf_model_weights_iterator,
11
- load_tensor_parallel_weights)
12
-from cacheflow.parallel_utils.parallel_state import (
6
+from transformers import GPTNeoXConfig
7
+
8
+from cacheflow.model_executor.input_metadata import InputMetadata
9
+from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
10
+from cacheflow.model_executor.layers.sampler import Sampler
11
+from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
12
+ load_tensor_parallel_weights)
13
+from cacheflow.model_executor.parallel_utils.parallel_state import (
14
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
14
-from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
15
- ColumnParallelLinear,
16
- RowParallelLinear)
15
+from cacheflow.model_executor.parallel_utils.tensor_parallel import (
16
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
17
from cacheflow.sequence import SequenceOutputs
18
19
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -21,7 +21,7 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
21
22
class GPTNeoXAttention(nn.Module):
23
24
- def __init__(self, config):
24
+ def __init__(self, config: GPTNeoXConfig):
25
super().__init__()
26
self.total_num_heads = config.num_attention_heads
27
self.hidden_size = config.hidden_size
@@ -63,7 +63,7 @@ class GPTNeoXAttention(nn.Module):
63
64
65
class GPTNeoXMLP(nn.Module):
66
- def __init__(self, config):
66
+ def __init__(self, config: GPTNeoXConfig):
67
super().__init__()
68
self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
69
config.intermediate_size,
@@ -86,7 +86,7 @@ class GPTNeoXMLP(nn.Module):
86
87
class GPTNeoXLayer(nn.Module):
88
89
- def __init__(self, config):
89
+ def __init__(self, config: GPTNeoXConfig):
90
super().__init__()
91
self.use_parallel_residual = config.use_parallel_residual
92
self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
@@ -129,7 +129,7 @@ class GPTNeoXLayer(nn.Module):
129
130
131
class GPTNeoXModel(nn.Module):
132
- def __init__(self, config):
132
+ def __init__(self, config: GPTNeoXConfig):
133
super().__init__()
134
self.config = config
135
@@ -227,8 +227,5 @@ class GPTNeoXForCausalLM(nn.Module):
227
raise ValueError(f"Unexpected weight name: {name}")
228
load_tensor_parallel_weights(param, loaded_weight, name,
229
self._column_parallel_weights,
230
- self._row_parallel_weights)
231
-
232
- def initialize_dummy_weights(self) -> None:
233
- for param in self.state_dict().values():
234
- param.data.uniform_(-1e-3, 1e-3)
230
+ self._row_parallel_weights,
231
+ tensor_model_parallel_rank)
cacheflow/model_executor/models/llama.py
renamed
+13
-16
@@ -5,18 +5,18 @@ import torch
5
from torch import nn
6
from transformers import LlamaConfig
7
8
-from cacheflow.models import InputMetadata
9
-from cacheflow.models.activation import SiluAndMul
10
-from cacheflow.models.attention import GPTNeoXCacheFlowAttention
11
-from cacheflow.models.layernorm import RMSNorm
12
-from cacheflow.models.sample import Sampler
13
-from cacheflow.models.utils import (hf_model_weights_iterator,
14
- load_tensor_parallel_weights)
15
-from cacheflow.parallel_utils.parallel_state import (
8
+from cacheflow.sequence import SequenceOutputs
9
+from cacheflow.model_executor.input_metadata import InputMetadata
10
+from cacheflow.model_executor.layers.activation import SiluAndMul
11
+from cacheflow.model_executor.layers.layernorm import RMSNorm
12
+from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
13
+from cacheflow.model_executor.layers.sampler import Sampler
14
+from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
15
+ load_tensor_parallel_weights)
16
+from cacheflow.model_executor.parallel_utils.parallel_state import (
17
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17
-from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
18
- ColumnParallelLinear,
19
- RowParallelLinear)
18
+from cacheflow.model_executor.parallel_utils.tensor_parallel import (
19
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
20
from cacheflow.sequence import SequenceOutputs
21
22
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -263,8 +263,5 @@ class LlamaForCausalLM(nn.Module):
263
param = state_dict[name]
264
load_tensor_parallel_weights(param, loaded_weight, name,
265
self._column_parallel_weights,
266
- self._row_parallel_weights)
267
-
268
- def initialize_dummy_weights(self) -> None:
269
- for param in self.state_dict().values():
270
- param.data.uniform_(-1e-3, 1e-3)
266
+ self._row_parallel_weights,
267
+ tensor_model_parallel_rank)
cacheflow/model_executor/models/opt.py
renamed
+10
-14
@@ -5,16 +5,15 @@ import torch
5
from torch import nn
6
from transformers import OPTConfig
7
8
-from cacheflow.models import InputMetadata
9
-from cacheflow.models.attention import GPTCacheFlowAttention
10
-from cacheflow.models.sample import Sampler
11
-from cacheflow.models.utils import (hf_model_weights_iterator,
12
- load_tensor_parallel_weights)
13
-from cacheflow.parallel_utils.parallel_state import (
8
+from cacheflow.model_executor.input_metadata import InputMetadata
9
+from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
10
+from cacheflow.model_executor.layers.sampler import Sampler
11
+from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
12
+ load_tensor_parallel_weights)
13
+from cacheflow.model_executor.parallel_utils.parallel_state import (
14
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15
-from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
16
- ColumnParallelLinear,
17
- RowParallelLinear)
15
+from cacheflow.model_executor.parallel_utils.tensor_parallel import (
16
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
17
from cacheflow.sequence import SequenceOutputs
18
19
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -288,8 +287,5 @@ class OPTForCausalLM(nn.Module):
287
param = state_dict[name]
288
load_tensor_parallel_weights(param, loaded_weight, name,
289
self._column_parallel_weights,
291
- self._row_parallel_weights)
292
-
293
- def initialize_dummy_weights(self) -> None:
294
- for param in self.state_dict().values():
295
- param.data.uniform_(-1e-3, 1e-3)
290
+ self._row_parallel_weights,
291
+ tensor_model_parallel_rank)
cacheflow/model_executor/parallel_utils/README.md
renamed
cacheflow/model_executor/parallel_utils/__init__.py
new
+12
@@ -0,0 +1,12 @@
1
+import cacheflow.model_executor.parallel_utils.parallel_state
2
+import cacheflow.model_executor.parallel_utils.tensor_parallel
3
+import cacheflow.model_executor.parallel_utils.utils
4
+
5
+# Alias parallel_state as mpu, its legacy name
6
+mpu = parallel_state
7
+
8
+__all__ = [
9
+ "parallel_state",
10
+ "tensor_parallel",
11
+ "utils",
12
+]
cacheflow/model_executor/parallel_utils/parallel_state.py
renamed
cacheflow/model_executor/parallel_utils/tensor_parallel/__init__.py
renamed
cacheflow/model_executor/parallel_utils/tensor_parallel/layers.py
renamed
+1
-1
@@ -9,7 +9,7 @@ import torch.nn.functional as F
9
import torch.nn.init as init
10
from torch.nn.parameter import Parameter
11
12
-from cacheflow.parallel_utils.parallel_state import (
12
+from cacheflow.model_executor.parallel_utils.parallel_state import (
13
get_tensor_model_parallel_rank,
14
get_tensor_model_parallel_world_size,
15
get_all_reduce_launcher,
cacheflow/model_executor/parallel_utils/tensor_parallel/mappings.py
renamed
+1
-1
@@ -2,7 +2,7 @@
2
3
import torch
4
5
-from cacheflow.parallel_utils.parallel_state import (
5
+from cacheflow.model_executor.parallel_utils.parallel_state import (
6
get_tensor_model_parallel_rank,
7
get_tensor_model_parallel_world_size,
8
get_tensor_model_parallel_group,
cacheflow/model_executor/parallel_utils/tensor_parallel/random.py
renamed
+2
-2
@@ -10,7 +10,7 @@ from torch import _C
10
from torch.cuda import _lazy_call, device as device_ctx_manager
11
from torch.utils.checkpoint import detach_variable
12
13
-from cacheflow.parallel_utils.parallel_state import (
13
+from cacheflow.model_executor.parallel_utils.parallel_state import (
14
get_data_parallel_rank,
15
get_tensor_model_parallel_group,
16
get_tensor_model_parallel_rank,
@@ -22,7 +22,7 @@ from .utils import (
22
gather_split_1d_tensor,
23
)
24
25
-from cacheflow.parallel_utils.utils import safely_set_viewless_tensor_data
25
+from cacheflow.model_executor.parallel_utils.utils import safely_set_viewless_tensor_data
26
27
# Default name for the model parallel rng tracker.
28
_MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
cacheflow/model_executor/parallel_utils/tensor_parallel/utils.py
renamed
+2
-2
@@ -3,8 +3,8 @@
3
import torch
4
from typing import List, Sequence
5
6
-from cacheflow.parallel_utils.utils import divide
7
-from cacheflow.parallel_utils import parallel_state
6
+from cacheflow.model_executor.parallel_utils.utils import divide
7
+from cacheflow.model_executor.parallel_utils import parallel_state
8
9
def split_tensor_along_last_dim(
10
tensor: torch.Tensor,
cacheflow/model_executor/parallel_utils/utils.py
renamed
+1
-1
@@ -4,7 +4,7 @@ import operator
4
5
import torch
6
7
-from cacheflow.parallel_utils import parallel_state
7
+from cacheflow.model_executor.parallel_utils import parallel_state
8
9
10
def ensure_divisibility(numerator, denominator):
cacheflow/model_executor/utils.py
new
+41
@@ -0,0 +1,41 @@
1
+import random
2
+from typing import Union
3
+
4
+import numpy as np
5
+import torch
6
+
7
+from cacheflow.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8
+from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
+
10
+
11
+_STR_DTYPE_TO_TORCH_DTYPE = {
12
+ 'half': torch.half,
13
+ 'float': torch.float,
14
+ 'float16': torch.float16,
15
+ 'float32': torch.float32,
16
+ 'bfloat16': torch.bfloat16,
17
+}
18
+
19
+
20
+def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
21
+ if isinstance(dtype, str):
22
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
23
+ else:
24
+ torch_dtype = dtype
25
+ return torch_dtype
26
+
27
+
28
+def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
29
+ torch_dtype = get_torch_dtype(dtype)
30
+ return torch.tensor([], dtype=torch_dtype).element_size()
31
+
32
+
33
+def set_random_seed(seed: int) -> None:
34
+ random.seed(seed)
35
+ np.random.seed(seed)
36
+ torch.manual_seed(seed)
37
+ if torch.cuda.is_available():
38
+ torch.cuda.manual_seed_all(seed)
39
+
40
+ if model_parallel_is_initialized():
41
+ model_parallel_cuda_manual_seed(seed)
cacheflow/model_executor/weight_utils.py
renamed
+27
-35
@@ -1,47 +1,26 @@
1
-import os
1
+import filelock
2
import glob
3
import json
4
-import filelock
5
-from typing import Union, Optional
4
+import os
5
+from typing import Iterator, List, Optional, Tuple
6
7
+from huggingface_hub import snapshot_download
8
import numpy as np
9
import torch
10
from tqdm.auto import tqdm
10
-from huggingface_hub import snapshot_download
11
-from cacheflow.parallel_utils.parallel_state import (
12
- get_tensor_model_parallel_rank)
13
-
14
-
15
-_STR_DTYPE_TO_TORCH_DTYPE = {
16
- 'half': torch.half,
17
- 'float': torch.float,
18
- 'float16': torch.float16,
19
- 'float32': torch.float32,
20
- 'bfloat16': torch.bfloat16,
21
-}
22
-
23
-
24
-def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
25
- if isinstance(dtype, str):
26
- torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
27
- else:
28
- torch_dtype = dtype
29
- return torch_dtype
30
-
31
-
32
-def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
33
- torch_dtype = get_torch_dtype(dtype)
34
- return torch.tensor([], dtype=torch_dtype).element_size()
11
12
13
class Disabledtqdm(tqdm):
14
+
15
def __init__(self, *args, **kwargs):
16
super().__init__(*args, **kwargs, disable=True)
17
18
42
-def hf_model_weights_iterator(model_name_or_path: str,
43
- cache_dir: Optional[str] = None,
44
- use_np_cache: bool = False):
19
+def hf_model_weights_iterator(
20
+ model_name_or_path: str,
21
+ cache_dir: Optional[str] = None,
22
+ use_np_cache: bool = False,
23
+) -> Iterator[Tuple[str, torch.Tensor]]:
24
# Prepare file lock directory to prevent multiple processes from
25
# downloading the same model weights at the same time.
26
lock_dir = cache_dir if cache_dir is not None else "/tmp"
@@ -95,10 +74,14 @@ def hf_model_weights_iterator(model_name_or_path: str,
74
yield name, param
75
76
98
-def load_tensor_parallel_weights(param, loaded_weight, param_name,
99
- column_parallel_weight_names,
100
- row_parallel_weight_names):
101
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
77
+def load_tensor_parallel_weights(
78
+ param: torch.Tensor,
79
+ loaded_weight: torch.Tensor,
80
+ param_name: str,
81
+ column_parallel_weight_names: List[str],
82
+ row_parallel_weight_names: List[str],
83
+ tensor_model_parallel_rank: int,
84
+) -> None:
85
for p in column_parallel_weight_names:
86
if p in param_name:
87
shard_size = param.shape[0]
@@ -116,3 +99,12 @@ def load_tensor_parallel_weights(param, loaded_weight, param_name,
99
break
100
assert param.shape == loaded_weight.shape
101
param.data.copy_(loaded_weight)
102
+
103
+
104
+def initialize_dummy_weights(
105
+ model: torch.nn.Module,
106
+ low: float = -1e-3,
107
+ high: float = 1e-3,
108
+) -> None:
109
+ for param in model.state_dict().values():
110
+ param.data.uniform_(low, high)
cacheflow/models/__init__.py
deleted
-10
@@ -1,10 +0,0 @@
1
-from cacheflow.models.input_metadata import InputMetadata
2
-from cacheflow.models.model_utils import get_memory_analyzer
3
-from cacheflow.models.model_utils import get_model
4
-
5
-
6
-__all__ = [
7
- 'InputMetadata',
8
- 'get_memory_analyzer',
9
- 'get_model',
10
-]
cacheflow/parallel_utils/__init__.py
deleted
-12
@@ -1,12 +0,0 @@
1
-import cacheflow.parallel_utils.parallel_state
2
-import cacheflow.parallel_utils.tensor_parallel
3
-import cacheflow.parallel_utils.utils
4
-
5
-# Alias parallel_state as mpu, its legacy name
6
-mpu = parallel_state
7
-
8
-__all__ = [
9
- "parallel_state",
10
- "tensor_parallel",
11
- "utils",
12
-]
cacheflow/utils.py
+1
-17
@@ -1,13 +1,8 @@
1
import enum
2
-import random
3
-import psutil
2
5
-import numpy as np
3
+import psutil
4
import torch
5
8
-from cacheflow.parallel_utils.parallel_state import model_parallel_is_initialized
9
-from cacheflow.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
10
-
6
7
class Device(enum.Enum):
8
GPU = enum.auto()
@@ -28,17 +23,6 @@ class Counter:
23
self.counter = 0
24
25
31
-def set_random_seed(seed: int):
32
- random.seed(seed)
33
- np.random.seed(seed)
34
- torch.manual_seed(seed)
35
- if torch.cuda.is_available():
36
- torch.cuda.manual_seed_all(seed)
37
-
38
- if model_parallel_is_initialized():
39
- model_parallel_cuda_manual_seed(seed)
40
-
41
-
26
def get_gpu_memory(gpu: int = 0) -> int:
27
return torch.cuda.get_device_properties(gpu).total_memory
28
cacheflow/worker/controller.py
+1
-1
@@ -5,7 +5,7 @@ try:
5
except ImportError:
6
ray = None
7
8
-from cacheflow.master.scheduler import Scheduler
8
+from cacheflow.core.scheduler import Scheduler
9
from cacheflow.sequence import SequenceGroupInputs
10
from cacheflow.worker.worker import Worker
11
cacheflow/worker/worker.py
+5
-8
@@ -2,18 +2,15 @@ from typing import Dict, List, Tuple, Optional
2
3
import torch
4
5
-from cacheflow.models import get_model
6
-from cacheflow.models import InputMetadata
5
+from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
6
+from cacheflow.model_executor.parallel_utils.parallel_state import (
7
+ initialize_model_parallel,
8
+ initialize_all_reduce_launcher,
9
+ get_tensor_model_parallel_world_size)
10
from cacheflow.sampling_params import SamplingParams
11
from cacheflow.sequence import SequenceGroupInputs
12
from cacheflow.sequence import SequenceOutputs
13
from cacheflow.worker.cache_engine import CacheEngine
11
-from cacheflow.parallel_utils.parallel_state import (
12
- initialize_model_parallel,
13
- initialize_all_reduce_launcher,
14
- get_tensor_model_parallel_world_size)
15
-from cacheflow.utils import set_random_seed
16
-
14
15
class Worker:
16
gradio_webserver.py
renamed
plot/plot_normalized_latency.py
deleted
-212
@@ -1,212 +0,0 @@
1
-import argparse
2
-import os
3
-import pickle
4
-from typing import Any, Dict, List, Optional, Tuple
5
-
6
-import matplotlib.pyplot as plt
7
-import numpy as np
8
-
9
-
10
-SYSTEMS = [
11
- 'orca-constant',
12
- 'orca-power2',
13
- 'orca-oracle',
14
- 'cacheflow',
15
-]
16
-
17
-SYSTEM_TO_LABEL = {
18
- 'orca-constant': 'Orca (Max)',
19
- 'orca-power2': 'Orca (Pow2)',
20
- 'orca-oracle': 'Orca (Oracle)',
21
- 'cacheflow': 'KVFlow',
22
-}
23
-
24
-SYSTEM_TO_COLOR = {
25
- 'orca-constant': 'red',
26
- 'orca-power2': 'orange',
27
- 'orca-oracle': 'green',
28
- 'cacheflow': 'blue',
29
-}
30
-
31
-SYSTEM_TO_MARKER = {
32
- 'orca-constant': 'x',
33
- 'orca-power2': '^',
34
- 'orca-oracle': 's',
35
- 'cacheflow': 'o',
36
-}
37
-
38
-
39
-def get_results(save_dir: str) -> List[Dict[str, Any]]:
40
- with open(os.path.join(save_dir, 'sequences.pkl'), 'rb') as f:
41
- results = pickle.load(f)
42
- return results
43
-
44
-
45
-def get_request_rate(save_dir: str) -> float:
46
- """Get request rate from save_dir name."""
47
- # Directory name format:
48
- # .../req-rate-{req_rate}/seed-{seed}/duration-{duration}
49
- save_dir = os.path.abspath(save_dir)
50
- dir_names = save_dir.split('/')
51
-
52
- request_rate = None
53
- for dir_name in dir_names:
54
- if dir_name.startswith('req-rate-'):
55
- if request_rate is not None:
56
- raise ValueError(f'Found multiple request rates in {save_dir}')
57
- request_rate = float(dir_name.split('-')[-1])
58
- if request_rate is None:
59
- raise ValueError(f'Cannot find request rate in {save_dir}')
60
- return request_rate
61
-
62
-
63
-def get_model(save_dir: str) -> Tuple[str, int]:
64
- save_dir = os.path.abspath(save_dir)
65
- dir_names = save_dir.split('/')
66
-
67
- model = None
68
- for dir_name in dir_names:
69
- if '-tp' in dir_name:
70
- if model is not None:
71
- raise ValueError(f'Found multiple models in {save_dir}')
72
- model = dir_name.split('-tp')[0]
73
- tp = int(dir_name.split('-tp')[-1])
74
- if model is None:
75
- raise ValueError(f'Cannot find model in {save_dir}')
76
- return model, tp
77
-
78
-
79
-def get_system(save_dir: str) -> str:
80
- save_dir = os.path.abspath(save_dir)
81
- dir_names = save_dir.split('/')
82
-
83
- for dir_name in dir_names:
84
- if dir_name.startswith('orca-'):
85
- return dir_name
86
- if dir_name == 'cacheflow':
87
- return dir_name
88
- raise ValueError(f'Cannot find system in {save_dir}')
89
-
90
-
91
-def get_sampling(save_dir: str) -> str:
92
- save_dir = os.path.abspath(save_dir)
93
- dir_names = save_dir.split('/')
94
-
95
- for dir_name in dir_names:
96
- if dir_name.startswith('n'):
97
- if dir_name.endswith('-beam'):
98
- return dir_name
99
- if dir_name[1:].isdigit():
100
- return dir_name
101
- raise ValueError(f'Cannot find sampling method in {save_dir}')
102
-
103
-
104
-def plot_normalized_latency(
105
- exp_dir: str,
106
- duration: int,
107
- seed: int,
108
- warmup: int,
109
- xlim: Optional[float],
110
- ylim: Optional[float],
111
- log_scale: bool,
112
- format: str,
113
-) -> None:
114
- # Get leaf directories.
115
- save_dirs = []
116
- for root, dirs, files in os.walk(exp_dir):
117
- if dirs:
118
- continue
119
- if 'sequences.pkl' not in files:
120
- continue
121
- if f'seed{seed}' not in root:
122
- continue
123
- if f'duration-{duration}' not in root:
124
- continue
125
- save_dirs.append(root)
126
-
127
- # Plot normalized latency.
128
- perf_per_system: Dict[str, Tuple[List[float], List[float]]] = {}
129
- for save_dir in save_dirs:
130
- per_seq_norm_latencies = []
131
- results = get_results(save_dir)
132
- for seq in results:
133
- arrival_time = seq['arrival_time']
134
- finish_time = seq['finish_time']
135
- output_len = seq['output_len']
136
- if arrival_time < warmup:
137
- continue
138
- latency = finish_time - arrival_time
139
- norm_latency = latency / output_len
140
- per_seq_norm_latencies.append(norm_latency)
141
-
142
- request_rate = get_request_rate(save_dir)
143
- normalized_latency = np.mean(per_seq_norm_latencies)
144
- system_name = get_system(save_dir)
145
- if system_name not in perf_per_system:
146
- perf_per_system[system_name] = ([], [])
147
- perf_per_system[system_name][0].append(request_rate)
148
- perf_per_system[system_name][1].append(normalized_latency)
149
-
150
- print('#seqs', len(per_seq_norm_latencies))
151
- print(f'{save_dir}: {normalized_latency:.3f} s')
152
-
153
-
154
- # Plot normalized latency.
155
- plt.figure(figsize=(6, 4))
156
- for system_name in reversed(SYSTEMS):
157
- if system_name not in perf_per_system:
158
- continue
159
- # Sort by request rate.
160
- request_rates, normalized_latencies = perf_per_system[system_name]
161
- request_rates, normalized_latencies = zip(*sorted(zip(request_rates, normalized_latencies)))
162
- label = SYSTEM_TO_LABEL[system_name]
163
- color = SYSTEM_TO_COLOR[system_name]
164
- marker = SYSTEM_TO_MARKER[system_name]
165
- plt.plot(request_rates, normalized_latencies, label=label, color=color, marker=marker)
166
-
167
- # plt.legend()
168
- plt.xlabel('Request rate (req/s)', fontsize=12)
169
- plt.ylabel('Normalized latency (s/token)', fontsize=12)
170
-
171
- if log_scale:
172
- plt.yscale('log')
173
- if xlim is not None:
174
- plt.xlim(left=0, right=xlim)
175
- if ylim is not None:
176
- if log_scale:
177
- plt.ylim(top=ylim)
178
- else:
179
- plt.ylim(bottom=0, top=ylim)
180
-
181
- handles, labels = plt.gca().get_legend_handles_labels()
182
- handles = reversed(handles)
183
- labels = reversed(labels)
184
-
185
- plt.legend(
186
- handles, labels,
187
- ncol=4, fontsize=12, loc='upper center', bbox_to_anchor=(0.5, 1.15),
188
- columnspacing=0.5, handletextpad=0.5, handlelength=1.5, frameon=False, borderpad=0)
189
-
190
- # Save figure.
191
- model, tp = get_model(exp_dir)
192
- sampling = get_sampling(exp_dir)
193
- figname = f'{model}-tp{tp}-{sampling}.{format}'
194
- os.makedirs('./figures', exist_ok=True)
195
- plt.savefig(os.path.join('figures', figname), bbox_inches='tight')
196
- print(f'Saved figure to ./figures/{figname}')
197
-
198
-
199
-if __name__ == '__main__':
200
- parser = argparse.ArgumentParser()
201
- parser.add_argument('exp_dir', type=str)
202
- parser.add_argument('--duration', type=int, required=True)
203
- parser.add_argument('--seed', type=int, default=0)
204
- parser.add_argument('--warmup', type=int, default=60)
205
- parser.add_argument('--xlim', type=float, required=False, default=None)
206
- parser.add_argument('--ylim', type=float, required=False, default=None)
207
- parser.add_argument('--log', action='store_true')
208
- parser.add_argument('--format', choices=['png', 'pdf'], default='png')
209
- args = parser.parse_args()
210
-
211
- plot_normalized_latency(
212
- args.exp_dir, args.duration, args.seed, args.warmup, args.xlim, args.ylim, args.log, args.format)
plot/plot_stats.py
deleted
-52
@@ -1,52 +0,0 @@
1
-import os
2
-import pickle
3
-
4
-import matplotlib.pyplot as plt
5
-
6
-STAT_NAMES = [
7
- 'input_lens',
8
- 'num_running',
9
- 'num_waiting',
10
- 'num_preemption',
11
- 'gpu_cache_usage',
12
- 'cpu_cache_usage',
13
- 'num_swapped',
14
- 'swap_in_lens',
15
- 'swap_out_lens',
16
-]
17
-
18
-
19
-def plot_stats(output_dir: str):
20
- # Get stats.
21
- with open(os.path.join(output_dir, 'stats.pkl'), 'rb') as f:
22
- stats = pickle.load(f)
23
- timestamps = stats['timestamps']
24
-
25
- # Draw one figure for each stat.
26
- num_stats = len(STAT_NAMES)
27
- COLORS = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'orange', 'purple', 'pink', 'brown', 'gray']
28
- fig, axs = plt.subplots(num_stats, 1, figsize=(10, 2 * num_stats))
29
- for i, stat in enumerate(STAT_NAMES):
30
- data = stats[stat]
31
- if stat in ['gpu_cache_usage', 'cpu_cache_usage']:
32
- data = [x * 100 for x in data]
33
- stat = stat + ' (%)'
34
- axs[i].plot(timestamps, data, color=COLORS[i % len(COLORS)])
35
- axs[i].set_ylabel(stat.replace('_', ' '), fontdict={'fontsize': 12})
36
- axs[i].set_ylim(bottom=0)
37
-
38
- plt.xlabel('Time (s)')
39
- plt.tight_layout()
40
- fig_path = os.path.join(output_dir, 'stats.png')
41
- plt.savefig(fig_path)
42
- print(f'Saved stats to {fig_path}')
43
-
44
-
45
-if __name__ == '__main__':
46
- import argparse
47
-
48
- parser = argparse.ArgumentParser()
49
- parser.add_argument('output_dir', type=str, help='Output directory.')
50
- args = parser.parse_args()
51
-
52
- plot_stats(args.output_dir)
simple_server.py
+1
-1
@@ -1,6 +1,6 @@
1
import argparse
2
3
-from cacheflow.master.server import (
3
+from cacheflow.core.server import (
4
add_server_arguments, process_server_arguments,
5
init_local_server_and_frontend_with_arguments)
6
from cacheflow.sampling_params import SamplingParams