@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c041ab578
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Refactor system architecture (#82)

Woosuk Kwon committed May 9, 2023 at 15:30 UTC 7c041ab5784760416f85d68eb8925a1d1f981932
40 files changed +195 -447
cacheflow/core/block_manager.py renamed
cacheflow/core/policy.py renamed
cacheflow/core/scheduler.py renamed
+2 -2
@@ -4,8 +4,8 @@ import pickle
4 import time
5 from typing import Any, Dict, List, Optional, Tuple
6
7 -from cacheflow.master.block_manager import BlockSpaceManager
8 -from cacheflow.master.policy import PolicyFactory
7 +from cacheflow.core.block_manager import BlockSpaceManager
8 +from cacheflow.core.policy import PolicyFactory
9 from cacheflow.sampling_params import SamplingParams
10 from cacheflow.sequence import Sequence
11 from cacheflow.sequence import SequenceGroup
cacheflow/core/server.py renamed
+5 -4
@@ -8,20 +8,21 @@ try:
8 except ImportError:
9 ray = None
10
11 +from cacheflow.core.scheduler import Scheduler
12 +from cacheflow.frontend.simple_frontend import SimpleFrontend
13 from cacheflow.logger import init_logger
12 -from cacheflow.master.scheduler import Scheduler
13 -from cacheflow.master.simple_frontend import SimpleFrontend
14 -from cacheflow.models import get_memory_analyzer
15 -from cacheflow.worker.controller import Controller, DeviceID
14 +from cacheflow.model_executor import get_memory_analyzer
15 from cacheflow.sequence import SequenceGroup
16 from cacheflow.sampling_params import SamplingParams
17 from cacheflow.utils import get_gpu_memory, get_cpu_memory
18 +from cacheflow.worker.controller import Controller, DeviceID
19
20
21 logger = init_logger(__name__)
22
23
24 class Server:
25 +
26 def __init__(
27 self,
28 model: str,
cacheflow/frontend/fastapi_frontend.py renamed
+7 -7
@@ -1,22 +1,22 @@
1 import argparse
2 import asyncio
3 +import json
4 import time
5 from typing import List, Dict, Optional
5 -import json
6
7 -import ray
8 -from transformers import AutoTokenizer
7 from fastapi import FastAPI, Request
8 from fastapi.responses import StreamingResponse
9 +import ray
10 +from transformers import AutoTokenizer
11 import uvicorn
12
13 +from cacheflow.core.server import (Server, add_server_arguments,
14 + process_server_arguments,
15 + initialize_cluster)
16 from cacheflow.sampling_params import SamplingParams
17 from cacheflow.sequence import Sequence, SequenceGroup
15 -from cacheflow.master.server import (Server, add_server_arguments,
16 - process_server_arguments,
17 - initialize_cluster)
18 -from cacheflow.worker.controller import DeviceID
18 from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
19 +from cacheflow.worker.controller import DeviceID
20
21 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
22 app = FastAPI()
cacheflow/frontend/simple_frontend.py renamed
cacheflow/model_executor/__init__.py new
+11
@@ -0,0 +1,11 @@
1 +from cacheflow.model_executor.input_metadata import InputMetadata
2 +from cacheflow.model_executor.model_loader import get_model, get_memory_analyzer
3 +from cacheflow.model_executor.utils import set_random_seed
4 +
5 +
6 +__all__ = [
7 + "InputMetadata",
8 + "get_model",
9 + "get_memory_analyzer",
10 + "set_random_seed",
11 +]
cacheflow/model_executor/input_metadata.py renamed
cacheflow/model_executor/layers/activation.py renamed
cacheflow/model_executor/layers/attention.py renamed
+1 -1
@@ -7,7 +7,7 @@ from xformers import ops as xops
7 from cacheflow import attention_ops
8 from cacheflow import cache_ops
9 from cacheflow import pos_encoding_ops
10 -from cacheflow.models import InputMetadata
10 +from cacheflow.model_executor.input_metadata import InputMetadata
11
12
13 class GPTCacheFlowAttention(nn.Module):
cacheflow/model_executor/layers/layernorm.py renamed
cacheflow/model_executor/layers/sampler.py renamed
+4 -3
@@ -3,10 +3,11 @@ from typing import Dict, List, Tuple
3 import torch
4 import torch.nn as nn
5
6 -from cacheflow.models import InputMetadata
6 +from cacheflow.model_executor.input_metadata import InputMetadata
7 +from cacheflow.model_executor.parallel_utils.tensor_parallel import (
8 + gather_from_tensor_model_parallel_region)
9 from cacheflow.sampling_params import SamplingParams
10 from cacheflow.sequence import SequenceOutputs
9 -from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_parallel_region
11
12
13 class Sampler(nn.Module):
@@ -27,7 +28,7 @@ class Sampler(nn.Module):
28 # Get the logits for the next tokens.
29 logits = torch.matmul(hidden_states, embedding.t())
30 logits = gather_from_tensor_model_parallel_region(logits)
30 - # Remove paddings in vocab.
31 + # Remove paddings in vocab (if any).
32 logits = logits[:, :self.vocab_size]
33
34 # Apply temperature scaling.
cacheflow/model_executor/memory_analyzer.py renamed
+1 -1
@@ -2,7 +2,7 @@ import torch
2 from transformers import AutoConfig
3
4 from cacheflow.logger import init_logger
5 -from cacheflow.models.utils import get_dtype_size
5 +from cacheflow.model_executor.utils import get_dtype_size
6
7
8 logger = init_logger(__name__)
cacheflow/model_executor/model_loader.py renamed
+8 -11
@@ -5,16 +5,13 @@ import torch.nn as nn
5 from transformers import AutoConfig
6 from transformers import PretrainedConfig
7
8 -from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9 -from cacheflow.models.memory_analyzer import GPT2MemoryAnalyzer
10 -from cacheflow.models.memory_analyzer import GPTNeoXMemoryAnalyzer
11 -from cacheflow.models.memory_analyzer import LlamaMemoryAnalyzer
12 -from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
13 -from cacheflow.models.gpt2 import GPT2LMHeadModel
14 -from cacheflow.models.gpt_neox import GPTNeoXForCausalLM
15 -from cacheflow.models.llama import LlamaForCausalLM
16 -from cacheflow.models.opt import OPTForCausalLM
17 -from cacheflow.models.utils import get_torch_dtype
8 +from cacheflow.model_executor.memory_analyzer import (
9 + CacheFlowMemoryAnalyzer, GPT2MemoryAnalyzer, GPTNeoXMemoryAnalyzer,
10 + LlamaMemoryAnalyzer, OPTMemoryAnalyzer)
11 +from cacheflow.model_executor.models import (
12 + GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
13 +from cacheflow.model_executor.utils import get_torch_dtype
14 +from cacheflow.model_executor.weight_utils import initialize_dummy_weights
15
16
17 _MODELS = {
@@ -77,7 +74,7 @@ def get_model(
74 model = model.cuda()
75 # NOTE(woosuk): For precise performance evaluation, we assign
76 # random values to the weights.
80 - model.initialize_dummy_weights()
77 + initialize_dummy_weights(model)
78 else:
79 # Create a model instance.
80 model = model_class(config)
cacheflow/model_executor/models/__init__.py new
+12
@@ -0,0 +1,12 @@
1 +from cacheflow.model_executor.models.gpt_neox import GPTNeoXForCausalLM
2 +from cacheflow.model_executor.models.gpt2 import GPT2LMHeadModel
3 +from cacheflow.model_executor.models.llama import LlamaForCausalLM
4 +from cacheflow.model_executor.models.opt import OPTForCausalLM
5 +
6 +
7 +__all__ = [
8 + "GPT2LMHeadModel",
9 + "GPTNeoXForCausalLM",
10 + "LlamaForCausalLM",
11 + "OPTForCausalLM",
12 +]
cacheflow/model_executor/models/gpt2.py renamed
+10 -14
@@ -5,16 +5,15 @@ import torch
5 from torch import nn
6 from transformers import GPT2Config
7
8 -from cacheflow.models import InputMetadata
9 -from cacheflow.models.attention import GPTCacheFlowAttention
10 -from cacheflow.models.sample import Sampler
11 -from cacheflow.models.utils import (hf_model_weights_iterator,
12 - load_tensor_parallel_weights)
13 -from cacheflow.parallel_utils.parallel_state import (
8 +from cacheflow.model_executor.input_metadata import InputMetadata
9 +from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
10 +from cacheflow.model_executor.layers.sampler import Sampler
11 +from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
12 + load_tensor_parallel_weights)
13 +from cacheflow.model_executor.parallel_utils.parallel_state import (
14 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 -from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
16 - ColumnParallelLinear,
17 - RowParallelLinear)
15 +from cacheflow.model_executor.parallel_utils.tensor_parallel import (
16 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
17 from cacheflow.sequence import SequenceOutputs
18
19 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -258,8 +257,5 @@ class GPT2LMHeadModel(nn.Module):
257 raise ValueError(f"Unexpected parameter name {name}")
258 load_tensor_parallel_weights(param, loaded_weight, name,
259 self._column_parallel_weights,
261 - self._row_parallel_weights)
262 -
263 - def initialize_dummy_weights(self) -> None:
264 - for param in self.state_dict().values():
265 - param.data.uniform_(-1e-3, 1e-3)
260 + self._row_parallel_weights,
261 + tensor_model_parallel_rank)
cacheflow/model_executor/models/gpt_neox.py renamed
+16 -19
@@ -3,17 +3,17 @@ from typing import Dict, List, Optional, Tuple
3
4 import torch
5 from torch import nn
6 -
7 -from cacheflow.models import InputMetadata
8 -from cacheflow.models.attention import GPTNeoXCacheFlowAttention
9 -from cacheflow.models.sample import Sampler
10 -from cacheflow.models.utils import (hf_model_weights_iterator,
11 - load_tensor_parallel_weights)
12 -from cacheflow.parallel_utils.parallel_state import (
6 +from transformers import GPTNeoXConfig
7 +
8 +from cacheflow.model_executor.input_metadata import InputMetadata
9 +from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
10 +from cacheflow.model_executor.layers.sampler import Sampler
11 +from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
12 + load_tensor_parallel_weights)
13 +from cacheflow.model_executor.parallel_utils.parallel_state import (
14 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
14 -from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
15 - ColumnParallelLinear,
16 - RowParallelLinear)
15 +from cacheflow.model_executor.parallel_utils.tensor_parallel import (
16 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
17 from cacheflow.sequence import SequenceOutputs
18
19 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -21,7 +21,7 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
21
22 class GPTNeoXAttention(nn.Module):
23
24 - def __init__(self, config):
24 + def __init__(self, config: GPTNeoXConfig):
25 super().__init__()
26 self.total_num_heads = config.num_attention_heads
27 self.hidden_size = config.hidden_size
@@ -63,7 +63,7 @@ class GPTNeoXAttention(nn.Module):
63
64
65 class GPTNeoXMLP(nn.Module):
66 - def __init__(self, config):
66 + def __init__(self, config: GPTNeoXConfig):
67 super().__init__()
68 self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
69 config.intermediate_size,
@@ -86,7 +86,7 @@ class GPTNeoXMLP(nn.Module):
86
87 class GPTNeoXLayer(nn.Module):
88
89 - def __init__(self, config):
89 + def __init__(self, config: GPTNeoXConfig):
90 super().__init__()
91 self.use_parallel_residual = config.use_parallel_residual
92 self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
@@ -129,7 +129,7 @@ class GPTNeoXLayer(nn.Module):
129
130
131 class GPTNeoXModel(nn.Module):
132 - def __init__(self, config):
132 + def __init__(self, config: GPTNeoXConfig):
133 super().__init__()
134 self.config = config
135
@@ -227,8 +227,5 @@ class GPTNeoXForCausalLM(nn.Module):
227 raise ValueError(f"Unexpected weight name: {name}")
228 load_tensor_parallel_weights(param, loaded_weight, name,
229 self._column_parallel_weights,
230 - self._row_parallel_weights)
231 -
232 - def initialize_dummy_weights(self) -> None:
233 - for param in self.state_dict().values():
234 - param.data.uniform_(-1e-3, 1e-3)
230 + self._row_parallel_weights,
231 + tensor_model_parallel_rank)
cacheflow/model_executor/models/llama.py renamed
+13 -16
@@ -5,18 +5,18 @@ import torch
5 from torch import nn
6 from transformers import LlamaConfig
7
8 -from cacheflow.models import InputMetadata
9 -from cacheflow.models.activation import SiluAndMul
10 -from cacheflow.models.attention import GPTNeoXCacheFlowAttention
11 -from cacheflow.models.layernorm import RMSNorm
12 -from cacheflow.models.sample import Sampler
13 -from cacheflow.models.utils import (hf_model_weights_iterator,
14 - load_tensor_parallel_weights)
15 -from cacheflow.parallel_utils.parallel_state import (
8 +from cacheflow.sequence import SequenceOutputs
9 +from cacheflow.model_executor.input_metadata import InputMetadata
10 +from cacheflow.model_executor.layers.activation import SiluAndMul
11 +from cacheflow.model_executor.layers.layernorm import RMSNorm
12 +from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
13 +from cacheflow.model_executor.layers.sampler import Sampler
14 +from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
15 + load_tensor_parallel_weights)
16 +from cacheflow.model_executor.parallel_utils.parallel_state import (
17 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17 -from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
18 - ColumnParallelLinear,
19 - RowParallelLinear)
18 +from cacheflow.model_executor.parallel_utils.tensor_parallel import (
19 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
20 from cacheflow.sequence import SequenceOutputs
21
22 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -263,8 +263,5 @@ class LlamaForCausalLM(nn.Module):
263 param = state_dict[name]
264 load_tensor_parallel_weights(param, loaded_weight, name,
265 self._column_parallel_weights,
266 - self._row_parallel_weights)
267 -
268 - def initialize_dummy_weights(self) -> None:
269 - for param in self.state_dict().values():
270 - param.data.uniform_(-1e-3, 1e-3)
266 + self._row_parallel_weights,
267 + tensor_model_parallel_rank)
cacheflow/model_executor/models/opt.py renamed
+10 -14
@@ -5,16 +5,15 @@ import torch
5 from torch import nn
6 from transformers import OPTConfig
7
8 -from cacheflow.models import InputMetadata
9 -from cacheflow.models.attention import GPTCacheFlowAttention
10 -from cacheflow.models.sample import Sampler
11 -from cacheflow.models.utils import (hf_model_weights_iterator,
12 - load_tensor_parallel_weights)
13 -from cacheflow.parallel_utils.parallel_state import (
8 +from cacheflow.model_executor.input_metadata import InputMetadata
9 +from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
10 +from cacheflow.model_executor.layers.sampler import Sampler
11 +from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
12 + load_tensor_parallel_weights)
13 +from cacheflow.model_executor.parallel_utils.parallel_state import (
14 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 -from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
16 - ColumnParallelLinear,
17 - RowParallelLinear)
15 +from cacheflow.model_executor.parallel_utils.tensor_parallel import (
16 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
17 from cacheflow.sequence import SequenceOutputs
18
19 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -288,8 +287,5 @@ class OPTForCausalLM(nn.Module):
287 param = state_dict[name]
288 load_tensor_parallel_weights(param, loaded_weight, name,
289 self._column_parallel_weights,
291 - self._row_parallel_weights)
292 -
293 - def initialize_dummy_weights(self) -> None:
294 - for param in self.state_dict().values():
295 - param.data.uniform_(-1e-3, 1e-3)
290 + self._row_parallel_weights,
291 + tensor_model_parallel_rank)
cacheflow/model_executor/parallel_utils/README.md renamed
cacheflow/model_executor/parallel_utils/__init__.py new
+12
@@ -0,0 +1,12 @@
1 +import cacheflow.model_executor.parallel_utils.parallel_state
2 +import cacheflow.model_executor.parallel_utils.tensor_parallel
3 +import cacheflow.model_executor.parallel_utils.utils
4 +
5 +# Alias parallel_state as mpu, its legacy name
6 +mpu = parallel_state
7 +
8 +__all__ = [
9 + "parallel_state",
10 + "tensor_parallel",
11 + "utils",
12 +]
cacheflow/model_executor/parallel_utils/parallel_state.py renamed
cacheflow/model_executor/parallel_utils/tensor_parallel/__init__.py renamed
cacheflow/model_executor/parallel_utils/tensor_parallel/layers.py renamed
+1 -1
@@ -9,7 +9,7 @@ import torch.nn.functional as F
9 import torch.nn.init as init
10 from torch.nn.parameter import Parameter
11
12 -from cacheflow.parallel_utils.parallel_state import (
12 +from cacheflow.model_executor.parallel_utils.parallel_state import (
13 get_tensor_model_parallel_rank,
14 get_tensor_model_parallel_world_size,
15 get_all_reduce_launcher,
cacheflow/model_executor/parallel_utils/tensor_parallel/mappings.py renamed
+1 -1
@@ -2,7 +2,7 @@
2
3 import torch
4
5 -from cacheflow.parallel_utils.parallel_state import (
5 +from cacheflow.model_executor.parallel_utils.parallel_state import (
6 get_tensor_model_parallel_rank,
7 get_tensor_model_parallel_world_size,
8 get_tensor_model_parallel_group,
cacheflow/model_executor/parallel_utils/tensor_parallel/random.py renamed
+2 -2
@@ -10,7 +10,7 @@ from torch import _C
10 from torch.cuda import _lazy_call, device as device_ctx_manager
11 from torch.utils.checkpoint import detach_variable
12
13 -from cacheflow.parallel_utils.parallel_state import (
13 +from cacheflow.model_executor.parallel_utils.parallel_state import (
14 get_data_parallel_rank,
15 get_tensor_model_parallel_group,
16 get_tensor_model_parallel_rank,
@@ -22,7 +22,7 @@ from .utils import (
22 gather_split_1d_tensor,
23 )
24
25 -from cacheflow.parallel_utils.utils import safely_set_viewless_tensor_data
25 +from cacheflow.model_executor.parallel_utils.utils import safely_set_viewless_tensor_data
26
27 # Default name for the model parallel rng tracker.
28 _MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
cacheflow/model_executor/parallel_utils/tensor_parallel/utils.py renamed
+2 -2
@@ -3,8 +3,8 @@
3 import torch
4 from typing import List, Sequence
5
6 -from cacheflow.parallel_utils.utils import divide
7 -from cacheflow.parallel_utils import parallel_state
6 +from cacheflow.model_executor.parallel_utils.utils import divide
7 +from cacheflow.model_executor.parallel_utils import parallel_state
8
9 def split_tensor_along_last_dim(
10 tensor: torch.Tensor,
cacheflow/model_executor/parallel_utils/utils.py renamed
+1 -1
@@ -4,7 +4,7 @@ import operator
4
5 import torch
6
7 -from cacheflow.parallel_utils import parallel_state
7 +from cacheflow.model_executor.parallel_utils import parallel_state
8
9
10 def ensure_divisibility(numerator, denominator):
cacheflow/model_executor/utils.py new
+41
@@ -0,0 +1,41 @@
1 +import random
2 +from typing import Union
3 +
4 +import numpy as np
5 +import torch
6 +
7 +from cacheflow.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8 +from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9 +
10 +
11 +_STR_DTYPE_TO_TORCH_DTYPE = {
12 + 'half': torch.half,
13 + 'float': torch.float,
14 + 'float16': torch.float16,
15 + 'float32': torch.float32,
16 + 'bfloat16': torch.bfloat16,
17 +}
18 +
19 +
20 +def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
21 + if isinstance(dtype, str):
22 + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
23 + else:
24 + torch_dtype = dtype
25 + return torch_dtype
26 +
27 +
28 +def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
29 + torch_dtype = get_torch_dtype(dtype)
30 + return torch.tensor([], dtype=torch_dtype).element_size()
31 +
32 +
33 +def set_random_seed(seed: int) -> None:
34 + random.seed(seed)
35 + np.random.seed(seed)
36 + torch.manual_seed(seed)
37 + if torch.cuda.is_available():
38 + torch.cuda.manual_seed_all(seed)
39 +
40 + if model_parallel_is_initialized():
41 + model_parallel_cuda_manual_seed(seed)
cacheflow/model_executor/weight_utils.py renamed
+27 -35
@@ -1,47 +1,26 @@
1 -import os
1 +import filelock
2 import glob
3 import json
4 -import filelock
5 -from typing import Union, Optional
4 +import os
5 +from typing import Iterator, List, Optional, Tuple
6
7 +from huggingface_hub import snapshot_download
8 import numpy as np
9 import torch
10 from tqdm.auto import tqdm
10 -from huggingface_hub import snapshot_download
11 -from cacheflow.parallel_utils.parallel_state import (
12 - get_tensor_model_parallel_rank)
13 -
14 -
15 -_STR_DTYPE_TO_TORCH_DTYPE = {
16 - 'half': torch.half,
17 - 'float': torch.float,
18 - 'float16': torch.float16,
19 - 'float32': torch.float32,
20 - 'bfloat16': torch.bfloat16,
21 -}
22 -
23 -
24 -def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
25 - if isinstance(dtype, str):
26 - torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
27 - else:
28 - torch_dtype = dtype
29 - return torch_dtype
30 -
31 -
32 -def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
33 - torch_dtype = get_torch_dtype(dtype)
34 - return torch.tensor([], dtype=torch_dtype).element_size()
11
12
13 class Disabledtqdm(tqdm):
14 +
15 def __init__(self, *args, **kwargs):
16 super().__init__(*args, **kwargs, disable=True)
17
18
42 -def hf_model_weights_iterator(model_name_or_path: str,
43 - cache_dir: Optional[str] = None,
44 - use_np_cache: bool = False):
19 +def hf_model_weights_iterator(
20 + model_name_or_path: str,
21 + cache_dir: Optional[str] = None,
22 + use_np_cache: bool = False,
23 +) -> Iterator[Tuple[str, torch.Tensor]]:
24 # Prepare file lock directory to prevent multiple processes from
25 # downloading the same model weights at the same time.
26 lock_dir = cache_dir if cache_dir is not None else "/tmp"
@@ -95,10 +74,14 @@ def hf_model_weights_iterator(model_name_or_path: str,
74 yield name, param
75
76
98 -def load_tensor_parallel_weights(param, loaded_weight, param_name,
99 - column_parallel_weight_names,
100 - row_parallel_weight_names):
101 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
77 +def load_tensor_parallel_weights(
78 + param: torch.Tensor,
79 + loaded_weight: torch.Tensor,
80 + param_name: str,
81 + column_parallel_weight_names: List[str],
82 + row_parallel_weight_names: List[str],
83 + tensor_model_parallel_rank: int,
84 +) -> None:
85 for p in column_parallel_weight_names:
86 if p in param_name:
87 shard_size = param.shape[0]
@@ -116,3 +99,12 @@ def load_tensor_parallel_weights(param, loaded_weight, param_name,
99 break
100 assert param.shape == loaded_weight.shape
101 param.data.copy_(loaded_weight)
102 +
103 +
104 +def initialize_dummy_weights(
105 + model: torch.nn.Module,
106 + low: float = -1e-3,
107 + high: float = 1e-3,
108 +) -> None:
109 + for param in model.state_dict().values():
110 + param.data.uniform_(low, high)
cacheflow/models/__init__.py deleted
-10
@@ -1,10 +0,0 @@
1 -from cacheflow.models.input_metadata import InputMetadata
2 -from cacheflow.models.model_utils import get_memory_analyzer
3 -from cacheflow.models.model_utils import get_model
4 -
5 -
6 -__all__ = [
7 - 'InputMetadata',
8 - 'get_memory_analyzer',
9 - 'get_model',
10 -]
cacheflow/parallel_utils/__init__.py deleted
-12
@@ -1,12 +0,0 @@
1 -import cacheflow.parallel_utils.parallel_state
2 -import cacheflow.parallel_utils.tensor_parallel
3 -import cacheflow.parallel_utils.utils
4 -
5 -# Alias parallel_state as mpu, its legacy name
6 -mpu = parallel_state
7 -
8 -__all__ = [
9 - "parallel_state",
10 - "tensor_parallel",
11 - "utils",
12 -]
cacheflow/utils.py
+1 -17
@@ -1,13 +1,8 @@
1 import enum
2 -import random
3 -import psutil
2
5 -import numpy as np
3 +import psutil
4 import torch
5
8 -from cacheflow.parallel_utils.parallel_state import model_parallel_is_initialized
9 -from cacheflow.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
10 -
6
7 class Device(enum.Enum):
8 GPU = enum.auto()
@@ -28,17 +23,6 @@ class Counter:
23 self.counter = 0
24
25
31 -def set_random_seed(seed: int):
32 - random.seed(seed)
33 - np.random.seed(seed)
34 - torch.manual_seed(seed)
35 - if torch.cuda.is_available():
36 - torch.cuda.manual_seed_all(seed)
37 -
38 - if model_parallel_is_initialized():
39 - model_parallel_cuda_manual_seed(seed)
40 -
41 -
26 def get_gpu_memory(gpu: int = 0) -> int:
27 return torch.cuda.get_device_properties(gpu).total_memory
28
cacheflow/worker/controller.py
+1 -1
@@ -5,7 +5,7 @@ try:
5 except ImportError:
6 ray = None
7
8 -from cacheflow.master.scheduler import Scheduler
8 +from cacheflow.core.scheduler import Scheduler
9 from cacheflow.sequence import SequenceGroupInputs
10 from cacheflow.worker.worker import Worker
11
cacheflow/worker/worker.py
+5 -8
@@ -2,18 +2,15 @@ from typing import Dict, List, Tuple, Optional
2
3 import torch
4
5 -from cacheflow.models import get_model
6 -from cacheflow.models import InputMetadata
5 +from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
6 +from cacheflow.model_executor.parallel_utils.parallel_state import (
7 + initialize_model_parallel,
8 + initialize_all_reduce_launcher,
9 + get_tensor_model_parallel_world_size)
10 from cacheflow.sampling_params import SamplingParams
11 from cacheflow.sequence import SequenceGroupInputs
12 from cacheflow.sequence import SequenceOutputs
13 from cacheflow.worker.cache_engine import CacheEngine
11 -from cacheflow.parallel_utils.parallel_state import (
12 - initialize_model_parallel,
13 - initialize_all_reduce_launcher,
14 - get_tensor_model_parallel_world_size)
15 -from cacheflow.utils import set_random_seed
16 -
14
15 class Worker:
16
gradio_webserver.py renamed
plot/plot_normalized_latency.py deleted
-212
@@ -1,212 +0,0 @@
1 -import argparse
2 -import os
3 -import pickle
4 -from typing import Any, Dict, List, Optional, Tuple
5 -
6 -import matplotlib.pyplot as plt
7 -import numpy as np
8 -
9 -
10 -SYSTEMS = [
11 - 'orca-constant',
12 - 'orca-power2',
13 - 'orca-oracle',
14 - 'cacheflow',
15 -]
16 -
17 -SYSTEM_TO_LABEL = {
18 - 'orca-constant': 'Orca (Max)',
19 - 'orca-power2': 'Orca (Pow2)',
20 - 'orca-oracle': 'Orca (Oracle)',
21 - 'cacheflow': 'KVFlow',
22 -}
23 -
24 -SYSTEM_TO_COLOR = {
25 - 'orca-constant': 'red',
26 - 'orca-power2': 'orange',
27 - 'orca-oracle': 'green',
28 - 'cacheflow': 'blue',
29 -}
30 -
31 -SYSTEM_TO_MARKER = {
32 - 'orca-constant': 'x',
33 - 'orca-power2': '^',
34 - 'orca-oracle': 's',
35 - 'cacheflow': 'o',
36 -}
37 -
38 -
39 -def get_results(save_dir: str) -> List[Dict[str, Any]]:
40 - with open(os.path.join(save_dir, 'sequences.pkl'), 'rb') as f:
41 - results = pickle.load(f)
42 - return results
43 -
44 -
45 -def get_request_rate(save_dir: str) -> float:
46 - """Get request rate from save_dir name."""
47 - # Directory name format:
48 - # .../req-rate-{req_rate}/seed-{seed}/duration-{duration}
49 - save_dir = os.path.abspath(save_dir)
50 - dir_names = save_dir.split('/')
51 -
52 - request_rate = None
53 - for dir_name in dir_names:
54 - if dir_name.startswith('req-rate-'):
55 - if request_rate is not None:
56 - raise ValueError(f'Found multiple request rates in {save_dir}')
57 - request_rate = float(dir_name.split('-')[-1])
58 - if request_rate is None:
59 - raise ValueError(f'Cannot find request rate in {save_dir}')
60 - return request_rate
61 -
62 -
63 -def get_model(save_dir: str) -> Tuple[str, int]:
64 - save_dir = os.path.abspath(save_dir)
65 - dir_names = save_dir.split('/')
66 -
67 - model = None
68 - for dir_name in dir_names:
69 - if '-tp' in dir_name:
70 - if model is not None:
71 - raise ValueError(f'Found multiple models in {save_dir}')
72 - model = dir_name.split('-tp')[0]
73 - tp = int(dir_name.split('-tp')[-1])
74 - if model is None:
75 - raise ValueError(f'Cannot find model in {save_dir}')
76 - return model, tp
77 -
78 -
79 -def get_system(save_dir: str) -> str:
80 - save_dir = os.path.abspath(save_dir)
81 - dir_names = save_dir.split('/')
82 -
83 - for dir_name in dir_names:
84 - if dir_name.startswith('orca-'):
85 - return dir_name
86 - if dir_name == 'cacheflow':
87 - return dir_name
88 - raise ValueError(f'Cannot find system in {save_dir}')
89 -
90 -
91 -def get_sampling(save_dir: str) -> str:
92 - save_dir = os.path.abspath(save_dir)
93 - dir_names = save_dir.split('/')
94 -
95 - for dir_name in dir_names:
96 - if dir_name.startswith('n'):
97 - if dir_name.endswith('-beam'):
98 - return dir_name
99 - if dir_name[1:].isdigit():
100 - return dir_name
101 - raise ValueError(f'Cannot find sampling method in {save_dir}')
102 -
103 -
104 -def plot_normalized_latency(
105 - exp_dir: str,
106 - duration: int,
107 - seed: int,
108 - warmup: int,
109 - xlim: Optional[float],
110 - ylim: Optional[float],
111 - log_scale: bool,
112 - format: str,
113 -) -> None:
114 - # Get leaf directories.
115 - save_dirs = []
116 - for root, dirs, files in os.walk(exp_dir):
117 - if dirs:
118 - continue
119 - if 'sequences.pkl' not in files:
120 - continue
121 - if f'seed{seed}' not in root:
122 - continue
123 - if f'duration-{duration}' not in root:
124 - continue
125 - save_dirs.append(root)
126 -
127 - # Plot normalized latency.
128 - perf_per_system: Dict[str, Tuple[List[float], List[float]]] = {}
129 - for save_dir in save_dirs:
130 - per_seq_norm_latencies = []
131 - results = get_results(save_dir)
132 - for seq in results:
133 - arrival_time = seq['arrival_time']
134 - finish_time = seq['finish_time']
135 - output_len = seq['output_len']
136 - if arrival_time < warmup:
137 - continue
138 - latency = finish_time - arrival_time
139 - norm_latency = latency / output_len
140 - per_seq_norm_latencies.append(norm_latency)
141 -
142 - request_rate = get_request_rate(save_dir)
143 - normalized_latency = np.mean(per_seq_norm_latencies)
144 - system_name = get_system(save_dir)
145 - if system_name not in perf_per_system:
146 - perf_per_system[system_name] = ([], [])
147 - perf_per_system[system_name][0].append(request_rate)
148 - perf_per_system[system_name][1].append(normalized_latency)
149 -
150 - print('#seqs', len(per_seq_norm_latencies))
151 - print(f'{save_dir}: {normalized_latency:.3f} s')
152 -
153 -
154 - # Plot normalized latency.
155 - plt.figure(figsize=(6, 4))
156 - for system_name in reversed(SYSTEMS):
157 - if system_name not in perf_per_system:
158 - continue
159 - # Sort by request rate.
160 - request_rates, normalized_latencies = perf_per_system[system_name]
161 - request_rates, normalized_latencies = zip(*sorted(zip(request_rates, normalized_latencies)))
162 - label = SYSTEM_TO_LABEL[system_name]
163 - color = SYSTEM_TO_COLOR[system_name]
164 - marker = SYSTEM_TO_MARKER[system_name]
165 - plt.plot(request_rates, normalized_latencies, label=label, color=color, marker=marker)
166 -
167 - # plt.legend()
168 - plt.xlabel('Request rate (req/s)', fontsize=12)
169 - plt.ylabel('Normalized latency (s/token)', fontsize=12)
170 -
171 - if log_scale:
172 - plt.yscale('log')
173 - if xlim is not None:
174 - plt.xlim(left=0, right=xlim)
175 - if ylim is not None:
176 - if log_scale:
177 - plt.ylim(top=ylim)
178 - else:
179 - plt.ylim(bottom=0, top=ylim)
180 -
181 - handles, labels = plt.gca().get_legend_handles_labels()
182 - handles = reversed(handles)
183 - labels = reversed(labels)
184 -
185 - plt.legend(
186 - handles, labels,
187 - ncol=4, fontsize=12, loc='upper center', bbox_to_anchor=(0.5, 1.15),
188 - columnspacing=0.5, handletextpad=0.5, handlelength=1.5, frameon=False, borderpad=0)
189 -
190 - # Save figure.
191 - model, tp = get_model(exp_dir)
192 - sampling = get_sampling(exp_dir)
193 - figname = f'{model}-tp{tp}-{sampling}.{format}'
194 - os.makedirs('./figures', exist_ok=True)
195 - plt.savefig(os.path.join('figures', figname), bbox_inches='tight')
196 - print(f'Saved figure to ./figures/{figname}')
197 -
198 -
199 -if __name__ == '__main__':
200 - parser = argparse.ArgumentParser()
201 - parser.add_argument('exp_dir', type=str)
202 - parser.add_argument('--duration', type=int, required=True)
203 - parser.add_argument('--seed', type=int, default=0)
204 - parser.add_argument('--warmup', type=int, default=60)
205 - parser.add_argument('--xlim', type=float, required=False, default=None)
206 - parser.add_argument('--ylim', type=float, required=False, default=None)
207 - parser.add_argument('--log', action='store_true')
208 - parser.add_argument('--format', choices=['png', 'pdf'], default='png')
209 - args = parser.parse_args()
210 -
211 - plot_normalized_latency(
212 - args.exp_dir, args.duration, args.seed, args.warmup, args.xlim, args.ylim, args.log, args.format)
plot/plot_stats.py deleted
-52
@@ -1,52 +0,0 @@
1 -import os
2 -import pickle
3 -
4 -import matplotlib.pyplot as plt
5 -
6 -STAT_NAMES = [
7 - 'input_lens',
8 - 'num_running',
9 - 'num_waiting',
10 - 'num_preemption',
11 - 'gpu_cache_usage',
12 - 'cpu_cache_usage',
13 - 'num_swapped',
14 - 'swap_in_lens',
15 - 'swap_out_lens',
16 -]
17 -
18 -
19 -def plot_stats(output_dir: str):
20 - # Get stats.
21 - with open(os.path.join(output_dir, 'stats.pkl'), 'rb') as f:
22 - stats = pickle.load(f)
23 - timestamps = stats['timestamps']
24 -
25 - # Draw one figure for each stat.
26 - num_stats = len(STAT_NAMES)
27 - COLORS = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'orange', 'purple', 'pink', 'brown', 'gray']
28 - fig, axs = plt.subplots(num_stats, 1, figsize=(10, 2 * num_stats))
29 - for i, stat in enumerate(STAT_NAMES):
30 - data = stats[stat]
31 - if stat in ['gpu_cache_usage', 'cpu_cache_usage']:
32 - data = [x * 100 for x in data]
33 - stat = stat + ' (%)'
34 - axs[i].plot(timestamps, data, color=COLORS[i % len(COLORS)])
35 - axs[i].set_ylabel(stat.replace('_', ' '), fontdict={'fontsize': 12})
36 - axs[i].set_ylim(bottom=0)
37 -
38 - plt.xlabel('Time (s)')
39 - plt.tight_layout()
40 - fig_path = os.path.join(output_dir, 'stats.png')
41 - plt.savefig(fig_path)
42 - print(f'Saved stats to {fig_path}')
43 -
44 -
45 -if __name__ == '__main__':
46 - import argparse
47 -
48 - parser = argparse.ArgumentParser()
49 - parser.add_argument('output_dir', type=str, help='Output directory.')
50 - args = parser.parse_args()
51 -
52 - plot_stats(args.output_dir)
simple_server.py
+1 -1
@@ -1,6 +1,6 @@
1 import argparse
2
3 -from cacheflow.master.server import (
3 +from cacheflow.core.server import (
4 add_server_arguments, process_server_arguments,
5 init_local_server_and_frontend_with_arguments)
6 from cacheflow.sampling_params import SamplingParams
test_cli_client.py renamed