A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Change the name to vLLM (#150)
Woosuk Kwon committed
Jun 17, 2023 at 03:07 UTC
0b98ba15c744f1dfb0ea4f2135e85ca23d572ae1
90 files changed
+342
-339
CONTRIBUTING.md
+6
-6
@@ -1,6 +1,6 @@
1
-# Contributing to CacheFlow
1
+# Contributing to vLLM
2
3
-Thank you for your interest in contributing to CacheFlow!
3
+Thank you for your interest in contributing to vLLM!
4
Our community is open to everyone and welcomes all kinds of contributions, no matter how small or large.
5
There are several ways you can contribute to the project:
6
@@ -11,9 +11,9 @@ There are several ways you can contribute to the project:
11
However, remember that contributions aren't just about code.
12
We believe in the power of community support; thus, answering queries, assisting others, and enhancing the documentation are highly regarded and beneficial contributions.
13
14
-Finally, one of the most impactful ways to support us is by raising awareness about CacheFlow.
14
+Finally, one of the most impactful ways to support us is by raising awareness about vLLM.
15
Talk about it in your blog posts, highlighting how it's driving your incredible projects.
16
-Express your support on Twitter if CacheFlow aids you, or simply offer your appreciation by starring our repository.
16
+Express your support on Twitter if vLLM aids you, or simply offer your appreciation by starring our repository.
17
18
19
## Setup for development
@@ -70,5 +70,5 @@ If a comment isn't clear or you disagree with a suggestion, feel free to ask for
70
71
### Thank You
72
73
-Finally, thank you for taking the time to read these guidelines and for your interest in contributing to CacheFlow.
74
-Your contributions make CacheFlow a great tool for everyone!
73
+Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM.
74
+Your contributions make vLLM a great tool for everyone!
README.md
+5
-5
@@ -1,4 +1,4 @@
1
-# CacheFlow
1
+# vLLM
2
3
## Build from source
4
@@ -28,7 +28,7 @@ python examples/simple_server.py --help
28
To start the server:
29
```bash
30
ray start --head
31
-python -m cacheflow.entrypoints.fastapi_server # --model <your_model>
31
+python -m vllm.entrypoints.fastapi_server # --model <your_model>
32
```
33
34
To test the server:
@@ -45,9 +45,9 @@ pip install gradio
45
46
Start the server:
47
```bash
48
-python -m cacheflow.http_frontend.fastapi_frontend
48
+python -m vllm.http_frontend.fastapi_frontend
49
# At another terminal
50
-python -m cacheflow.http_frontend.gradio_webserver
50
+python -m vllm.http_frontend.gradio_webserver
51
```
52
53
## Load LLaMA weights
@@ -62,5 +62,5 @@ Since LLaMA weight is not fully public, we cannot directly download the LLaMA we
62
2. For all the commands above, specify the model with `--model /output/path/llama-7b` to load the model. For example:
63
```bash
64
python simple_server.py --model /output/path/llama-7b
65
- python -m cacheflow.http_frontend.fastapi_frontend --model /output/path/llama-7b
65
+ python -m vllm.http_frontend.fastapi_frontend --model /output/path/llama-7b
66
```
benchmarks/README.md
+1
-1
@@ -1,4 +1,4 @@
1
-# Benchmarking CacheFlow
1
+# Benchmarking vLLM
2
3
## Downloading the ShareGPT dataset
4
benchmarks/benchmark_async_llm_server.py
+1
-1
@@ -11,7 +11,7 @@ def main(args: argparse.Namespace):
11
for i in range(args.n_threads)]
12
13
api_url = f"http://{args.host}:{args.port}/generate"
14
- headers = {"User-Agent": "CacheFlow Benchmark Client"}
14
+ headers = {"User-Agent": "vLLM Benchmark Client"}
15
ploads = [{
16
"prompt": p,
17
"max_tokens": args.max_tokens,
benchmarks/benchmark_latency.py
+1
-1
@@ -6,7 +6,7 @@ import numpy as np
6
import torch
7
from tqdm import tqdm
8
9
-from cacheflow import LLM, SamplingParams
9
+from vllm import LLM, SamplingParams
10
11
12
def main(args: argparse.Namespace):
benchmarks/benchmark_serving.py
+5
-5
@@ -1,8 +1,8 @@
1
"""Benchmark online serving throughput.
2
3
On the server side, run one of the following commands:
4
- (CacheFlow backend)
5
- python -m cacheflow.entrypoints.api_server \
4
+ (vLLM backend)
5
+ python -m vllm.entrypoints.api_server \
6
--disable-log-requests --model <your_model>
7
8
(TGI backend)
@@ -114,7 +114,7 @@ async def send_request(
114
request_start_time = time.time()
115
116
headers = {"User-Agent": "Benchmark Client"}
117
- if backend == "cacheflow":
117
+ if backend == "vllm":
118
pload = {
119
"prompt": prompt,
120
"n": 1,
@@ -213,8 +213,8 @@ def main(args: argparse.Namespace):
213
if __name__ == "__main__":
214
parser = argparse.ArgumentParser(
215
description="Benchmark the online serving throughput.")
216
- parser.add_argument("--backend", type=str, default="cacheflow",
217
- choices=["cacheflow", "tgi"])
216
+ parser.add_argument("--backend", type=str, default="vllm",
217
+ choices=["vllm", "tgi"])
218
parser.add_argument("--host", type=str, default="localhost")
219
parser.add_argument("--port", type=int, default=8001)
220
parser.add_argument("--dataset", type=str, required=True,
benchmarks/benchmark_throughput.py
+8
-7
@@ -5,12 +5,13 @@ import random
5
import time
6
from typing import List, Tuple
7
8
-from cacheflow import LLM, SamplingParams
8
import torch
9
from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
10
PreTrainedTokenizerBase)
11
from tqdm import tqdm
12
13
+from vllm import LLM, SamplingParams
14
+
15
16
def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
17
config = AutoConfig.from_pretrained(model_name)
@@ -70,7 +71,7 @@ def sample_requests(
71
return sampled_requests
72
73
73
-def run_cacheflow(
74
+def run_vllm(
75
requests: List[Tuple[str, int, int]],
76
model: str,
77
tensor_parallel_size: int,
@@ -172,8 +173,8 @@ def main(args: argparse.Namespace):
173
tokenizer = get_tokenizer(args.model)
174
requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
175
175
- if args.backend == "cacheflow":
176
- elapsed_time = run_cacheflow(
176
+ if args.backend == "vllm":
177
+ elapsed_time = run_vllm(
178
requests, args.model, args.tensor_parallel_size, args.seed, args.n,
179
args.use_beam_search)
180
elif args.backend == "hf":
@@ -192,8 +193,8 @@ def main(args: argparse.Namespace):
193
194
if __name__ == "__main__":
195
parser = argparse.ArgumentParser(description="Benchmark the throughput.")
195
- parser.add_argument("--backend", type=str, choices=["cacheflow", "hf"],
196
- default="cacheflow")
196
+ parser.add_argument("--backend", type=str, choices=["vllm", "hf"],
197
+ default="vllm")
198
parser.add_argument("--dataset", type=str, required=True,
199
help="Path to the dataset.")
200
parser.add_argument("--model", type=str, default="facebook/opt-125m")
@@ -207,7 +208,7 @@ if __name__ == "__main__":
208
parser.add_argument("--hf-max-batch-size", type=int, default=None,
209
help="Maximum batch size for HF backend.")
210
args = parser.parse_args()
210
- if args.backend == "cacheflow":
211
+ if args.backend == "vllm":
212
if args.hf_max_batch_size is not None:
213
raise ValueError("HF max batch size is only for HF backend.")
214
elif args.backend == "hf":
cacheflow/__init__.py
deleted
-18
@@ -1,18 +0,0 @@
1
-from cacheflow.engine.arg_utils import EngineArgs
2
-from cacheflow.engine.llm_engine import LLMEngine
3
-from cacheflow.engine.ray_utils import initialize_cluster
4
-from cacheflow.entrypoints.llm import LLM
5
-from cacheflow.outputs import CompletionOutput, RequestOutput
6
-from cacheflow.sampling_params import SamplingParams
7
-
8
-__version__ = "0.1.0"
9
-
10
-__all__ = [
11
- "LLM",
12
- "SamplingParams",
13
- "RequestOutput",
14
- "CompletionOutput",
15
- "LLMEngine",
16
- "EngineArgs",
17
- "initialize_cluster",
18
-]
cacheflow/model_executor/__init__.py
deleted
-10
@@ -1,10 +0,0 @@
1
-from cacheflow.model_executor.input_metadata import InputMetadata
2
-from cacheflow.model_executor.model_loader import get_model
3
-from cacheflow.model_executor.utils import set_random_seed
4
-
5
-
6
-__all__ = [
7
- "InputMetadata",
8
- "get_model",
9
- "set_random_seed",
10
-]
cacheflow/model_executor/models/__init__.py
deleted
-12
@@ -1,12 +0,0 @@
1
-from cacheflow.model_executor.models.gpt_neox import GPTNeoXForCausalLM
2
-from cacheflow.model_executor.models.gpt2 import GPT2LMHeadModel
3
-from cacheflow.model_executor.models.llama import LlamaForCausalLM
4
-from cacheflow.model_executor.models.opt import OPTForCausalLM
5
-
6
-
7
-__all__ = [
8
- "GPT2LMHeadModel",
9
- "GPTNeoXForCausalLM",
10
- "LlamaForCausalLM",
11
- "OPTForCausalLM",
12
-]
csrc/activation_kernels.cu
+3
-3
@@ -1,7 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
-namespace cacheflow {
4
+namespace vllm {
5
6
template<typename T>
7
__device__ __forceinline__ T silu(const T& x) {
@@ -22,7 +22,7 @@ __global__ void silu_and_mul_kernel(
22
}
23
}
24
25
-} // namespace cacheflow
25
+} // namespace vllm
26
27
void silu_and_mul(
28
torch::Tensor& out, // [num_tokens, d]
@@ -40,7 +40,7 @@ void silu_and_mul(
40
input.scalar_type(),
41
"silu_and_mul_kernel",
42
[&] {
43
- cacheflow::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
43
+ vllm::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
44
out.data_ptr<scalar_t>(),
45
input.data_ptr<scalar_t>(),
46
d);
csrc/attention/attention_generic.cuh
+3
-3
@@ -1,6 +1,6 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
3
- * Copyright (c) 2023, The CacheFlow team.
3
+ * Copyright (c) 2023, The vLLM team.
4
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5
*
6
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +19,7 @@
19
20
#include <stdint.h>
21
22
-namespace cacheflow {
22
+namespace vllm {
23
24
// A vector type to store Q, K, V elements.
25
template<typename T, int VEC_SIZE>
@@ -61,4 +61,4 @@ inline __device__ void zero(T& dst) {
61
dst = tmp.raw;
62
}
63
64
-} // namespace cacheflow
64
+} // namespace vllm
csrc/attention/attention_kernels.cu
+4
-4
@@ -1,6 +1,6 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
- * Copyright (c) 2023, The CacheFlow team.
3
+ * Copyright (c) 2023, The vLLM team.
4
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5
*
6
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,7 +27,7 @@
27
#define MAX(a, b) ((a) > (b) ? (a) : (b))
28
#define MIN(a, b) ((a) < (b) ? (a) : (b))
29
30
-namespace cacheflow {
30
+namespace vllm {
31
32
// Utility function for attention softmax.
33
template<int NUM_WARPS>
@@ -315,10 +315,10 @@ __global__ void single_query_cached_kv_attention_kernel(
315
}
316
}
317
318
-} // namespace cacheflow
318
+} // namespace vllm
319
320
#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
321
- cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
321
+ vllm::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
322
<<<grid, block, shared_mem_size, stream>>>( \
323
out_ptr, \
324
query_ptr, \
csrc/attention/attention_utils.cuh
+3
-3
@@ -1,6 +1,6 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
- * Copyright (c) 2023, The CacheFlow team.
3
+ * Copyright (c) 2023, The vLLM team.
4
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5
*
6
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +22,7 @@
22
#include <float.h>
23
#include <type_traits>
24
25
-namespace cacheflow {
25
+namespace vllm {
26
27
// Q*K^T operation.
28
template<int THREAD_GROUP_SIZE, typename Vec, int N>
@@ -52,4 +52,4 @@ struct Qk_dot {
52
}
53
};
54
55
-} // namespace cacheflow
55
+} // namespace vllm
csrc/attention/dtype_bfloat16.cuh
+3
-3
@@ -1,7 +1,7 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
* and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4
- * Copyright (c) 2023, The CacheFlow team.
4
+ * Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
7
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,7 +25,7 @@
25
#include <cuda_fp16.h>
26
#include <stdint.h>
27
28
-namespace cacheflow {
28
+namespace vllm {
29
30
// Define custom BF16 vector data types.
31
struct bf16_4_t {
@@ -420,4 +420,4 @@ inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
420
#endif
421
}
422
423
-} // namespace cacheflow
423
+} // namespace vllm
csrc/attention/dtype_float16.cuh
+3
-3
@@ -1,7 +1,7 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
* and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4
- * Copyright (c) 2023, The CacheFlow team.
4
+ * Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
7
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +23,7 @@
23
24
#include <stdint.h>
25
26
-namespace cacheflow {
26
+namespace vllm {
27
28
// FP16 vector types for Q, K, V.
29
template<>
@@ -441,4 +441,4 @@ inline __device__ Float8_ to_float(uint4 u) {
441
return tmp;
442
}
443
444
-} // namespace cacheflow
444
+} // namespace vllm
csrc/attention/dtype_float32.cuh
+3
-3
@@ -1,7 +1,7 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
* and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4
- * Copyright (c) 2023, The CacheFlow team.
4
+ * Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
7
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +22,7 @@
22
23
#include <stdint.h>
24
25
-namespace cacheflow {
25
+namespace vllm {
26
27
// Define custom FP32 vector data types.
28
struct Float4_ {
@@ -265,4 +265,4 @@ inline __device__ Float8_ to_float(Float8_ u) {
265
return u;
266
}
267
268
-} // namespace cacheflow
268
+} // namespace vllm
csrc/cache_kernels.cu
+9
-9
@@ -46,7 +46,7 @@ void swap_blocks(
46
}
47
}
48
49
-namespace cacheflow {
49
+namespace vllm {
50
51
// Grid: (num_layers, num_pairs)
52
template<typename scalar_t>
@@ -77,7 +77,7 @@ __global__ void copy_blocks_kernel(
77
}
78
}
79
80
-} // namespace cacheflow
80
+} // namespace vllm
81
82
void copy_blocks(
83
std::vector<torch::Tensor>& key_caches,
@@ -129,7 +129,7 @@ void copy_blocks(
129
at::ScalarType::Half,
130
at::ScalarType::BFloat16,
131
key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
132
- cacheflow::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
132
+ vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133
key_cache_ptrs_tensor.data_ptr<int64_t>(),
134
value_cache_ptrs_tensor.data_ptr<int64_t>(),
135
block_mapping_tensor.data_ptr<int>(),
@@ -137,7 +137,7 @@ void copy_blocks(
137
}));
138
}
139
140
-namespace cacheflow {
140
+namespace vllm {
141
142
template<typename scalar_t>
143
__global__ void reshape_and_cache_kernel(
@@ -181,7 +181,7 @@ __global__ void reshape_and_cache_kernel(
181
}
182
}
183
184
-} // namespace cacheflow
184
+} // namespace vllm
185
186
void reshape_and_cache(
187
torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -208,7 +208,7 @@ void reshape_and_cache(
208
key.scalar_type(),
209
"reshape_and_cache_kernel",
210
[&] {
211
- cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
211
+ vllm::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
212
key.data_ptr<scalar_t>(),
213
value.data_ptr<scalar_t>(),
214
key_cache.data_ptr<scalar_t>(),
@@ -223,7 +223,7 @@ void reshape_and_cache(
223
});
224
}
225
226
-namespace cacheflow {
226
+namespace vllm {
227
228
// Grid: (num_blocks, block_size).
229
template<typename scalar_t>
@@ -343,7 +343,7 @@ __global__ void gather_cached_kv_kernel_optimized(
343
}
344
}
345
346
-} // namespace cacheflow
346
+} // namespace vllm
347
348
void gather_cached_kv(
349
torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
@@ -370,7 +370,7 @@ void gather_cached_kv(
370
key.scalar_type(),
371
"gather_cached_kv_kernel_optimized",
372
[&] {
373
- cacheflow::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
373
+ vllm::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
374
key.data_ptr<scalar_t>(),
375
value.data_ptr<scalar_t>(),
376
key_cache.data_ptr<scalar_t>(),
csrc/layernorm_kernels.cu
+3
-3
@@ -3,7 +3,7 @@
3
4
#include "reduction_utils.cuh"
5
6
-namespace cacheflow {
6
+namespace vllm {
7
8
// TODO(woosuk): Further optimize this kernel.
9
template<typename scalar_t>
@@ -33,7 +33,7 @@ __global__ void rms_norm_kernel(
33
}
34
}
35
36
-} // namespace cacheflow
36
+} // namespace vllm
37
38
void rms_norm(
39
torch::Tensor& out, // [num_tokens, hidden_size]
@@ -52,7 +52,7 @@ void rms_norm(
52
input.scalar_type(),
53
"rms_norm_kernel",
54
[&] {
55
- cacheflow::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
55
+ vllm::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
56
out.data_ptr<scalar_t>(),
57
input.data_ptr<scalar_t>(),
58
weight.data_ptr<scalar_t>(),
csrc/pos_encoding_kernels.cu
+3
-3
@@ -1,7 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
-namespace cacheflow {
4
+namespace vllm {
5
6
template<typename scalar_t>
7
__global__ void rotary_embedding_neox_kernel(
@@ -46,7 +46,7 @@ __global__ void rotary_embedding_neox_kernel(
46
}
47
}
48
49
-} // namespace cacheflow
49
+} // namespace vllm
50
51
void rotary_embedding_neox(
52
torch::Tensor& positions, // [num_tokens]
@@ -70,7 +70,7 @@ void rotary_embedding_neox(
70
query.scalar_type(),
71
"rotary_embedding_neox",
72
[&] {
73
- cacheflow::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
73
+ vllm::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
74
positions.data_ptr<int64_t>(),
75
query.data_ptr<scalar_t>(),
76
key.data_ptr<scalar_t>(),
csrc/reduction_utils.cuh
+3
-3
@@ -1,6 +1,6 @@
1
/*
2
* Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
3
- * Copyright (c) 2023, The CacheFlow team.
3
+ * Copyright (c) 2023, The vLLM team.
4
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5
*
6
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +17,7 @@
17
*/
18
#pragma once
19
20
-namespace cacheflow {
20
+namespace vllm {
21
22
template<typename T>
23
__inline__ __device__ T warpReduceSum(T val) {
@@ -48,4 +48,4 @@ __inline__ __device__ T blockReduceSum(T val) {
48
return val;
49
}
50
51
-} // namespace cacheflow
51
+} // namespace vllm
docs/README.md
+1
-1
@@ -1,4 +1,4 @@
1
-# CacheFlow documents
1
+# vLLM documents
2
3
## Build the docs
4
docs/source/conf.py
+4
-4
@@ -17,9 +17,9 @@
17
18
# -- Project information -----------------------------------------------------
19
20
-project = 'CacheFlow'
21
-copyright = '2023, CacheFlow Team'
22
-author = 'the CacheFlow Team'
20
+project = 'vLLM'
21
+copyright = '2023, vLLM Team'
22
+author = 'the vLLM Team'
23
24
25
# -- General configuration ---------------------------------------------------
@@ -55,7 +55,7 @@ html_title = project
55
html_theme = 'sphinx_book_theme'
56
html_theme_options = {
57
'path_to_docs': 'docs/source',
58
- 'repository_url': 'https://github.com/WoosukKwon/cacheflow',
58
+ 'repository_url': 'https://github.com/WoosukKwon/vllm',
59
'use_repository_button': True,
60
}
61
docs/source/getting_started/installation.rst
+11
-11
@@ -1,8 +1,8 @@
1
Installation
2
============
3
4
-CacheFlow is a Python library that includes some C++ and CUDA code.
5
-CacheFlow can run on systems that meet the following requirements:
4
+vLLM is a Python library that includes some C++ and CUDA code.
5
+vLLM can run on systems that meet the following requirements:
6
7
* OS: Linux
8
* Python: 3.8 or higher
@@ -10,23 +10,23 @@ CacheFlow can run on systems that meet the following requirements:
10
* GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, etc.)
11
12
.. note::
13
- As of now, CacheFlow does not support CUDA 12.
13
+ As of now, vLLM does not support CUDA 12.
14
If you are using Hopper or Lovelace GPUs, please use CUDA 11.8.
15
16
.. tip::
17
- If you have trouble installing CacheFlow, we recommend using the NVIDIA PyTorch Docker image.
17
+ If you have trouble installing vLLM, we recommend using the NVIDIA PyTorch Docker image.
18
19
.. code-block:: console
20
21
$ # Pull the Docker image with CUDA 11.8.
22
$ docker run --gpus all -it --rm --shm-size=8g nvcr.io/nvidia/pytorch:22.12-py3
23
24
- Inside the Docker container, please execute :code:`pip uninstall torch` before installing CacheFlow.
24
+ Inside the Docker container, please execute :code:`pip uninstall torch` before installing vLLM.
25
26
Install with pip
27
----------------
28
29
-You can install CacheFlow using pip:
29
+You can install vLLM using pip:
30
31
.. code-block:: console
32
@@ -34,8 +34,8 @@ You can install CacheFlow using pip:
34
$ conda create -n myenv python=3.8 -y
35
$ conda activate myenv
36
37
- $ # Install CacheFlow.
38
- $ pip install cacheflow # This may take 5-10 minutes.
37
+ $ # Install vLLM.
38
+ $ pip install vllm # This may take 5-10 minutes.
39
40
41
.. _build_from_source:
@@ -43,10 +43,10 @@ You can install CacheFlow using pip:
43
Build from source
44
-----------------
45
46
-You can also build and install CacheFlow from source.
46
+You can also build and install vLLM from source.
47
48
.. code-block:: console
49
50
- $ git clone https://github.com/WoosukKwon/cacheflow.git
51
- $ cd cacheflow
50
+ $ git clone https://github.com/WoosukKwon/vllm.git
51
+ $ cd vllm
52
$ pip install -e . # This may take 5-10 minutes.
docs/source/getting_started/quickstart.rst
+1
-1
@@ -8,7 +8,7 @@ Placeholder.
8
9
.. code-block:: python
10
11
- from cacheflow import LLM, SamplingParams
11
+ from vllm import LLM, SamplingParams
12
13
# Sample prompts.
14
prompts = [
docs/source/index.rst
+2
-2
@@ -1,5 +1,5 @@
1
-Welcome to CacheFlow!
2
-=====================
1
+Welcome to vLLM!
2
+================
3
4
Documentation
5
-------------
docs/source/models/adding_model.rst
+11
-11
@@ -3,30 +3,30 @@
3
Adding a New Model
4
==================
5
6
-This document provides a high-level guide on integrating a `HuggingFace Transformers <https://github.com/huggingface/transformers>`_ model into CacheFlow.
6
+This document provides a high-level guide on integrating a `HuggingFace Transformers <https://github.com/huggingface/transformers>`_ model into vLLM.
7
8
.. note::
9
The complexity of adding a new model depends heavily on the model's architecture.
10
- The process is considerably straightforward if the model shares a similar architecture with an existing model in CacheFlow.
10
+ The process is considerably straightforward if the model shares a similar architecture with an existing model in vLLM.
11
However, for models that include new operators (e.g., a new attention mechanism), the process can be a bit more complex.
12
13
.. tip::
14
- If you are encountering issues while integrating your model into CacheFlow, feel free to open an issue on our `GitHub <https://github.com/WoosukKwon/cacheflow/issues>`_ repository.
14
+ If you are encountering issues while integrating your model into vLLM, feel free to open an issue on our `GitHub <https://github.com/WoosukKwon/vllm/issues>`_ repository.
15
We will be happy to help you out!
16
17
18
-0. Fork the CacheFlow repository
18
+0. Fork the vLLM repository
19
--------------------------------
20
21
-Start by forking our `GitHub <https://github.com/WoosukKwon/cacheflow/issues>`_ repository and then :ref:`build it from source <build_from_source>`.
21
+Start by forking our `GitHub <https://github.com/WoosukKwon/vllm/issues>`_ repository and then :ref:`build it from source <build_from_source>`.
22
This gives you the ability to modify the codebase and test your model.
23
24
25
1. Bring your model code
26
------------------------
27
28
-Clone the PyTorch model code from the HuggingFace Transformers repository and put it into the `cacheflow/model_executor/models <https://github.com/WoosukKwon/cacheflow/tree/main/cacheflow/model_executor/models>`_ directory.
29
-For instance, CacheFlow's `OPT model <https://github.com/WoosukKwon/cacheflow/blob/main/cacheflow/model_executor/models/opt.py>`_ was adpated from the HuggingFace's `modeling_opt.py <https://github.com/huggingface/transformers/blob/main/src/transformers/models/opt/modeling_opt.py>`_ file.
28
+Clone the PyTorch model code from the HuggingFace Transformers repository and put it into the `vllm/model_executor/models <https://github.com/WoosukKwon/vllm/tree/main/vllm/model_executor/models>`_ directory.
29
+For instance, vLLM's `OPT model <https://github.com/WoosukKwon/vllm/blob/main/vllm/model_executor/models/opt.py>`_ was adpated from the HuggingFace's `modeling_opt.py <https://github.com/huggingface/transformers/blob/main/src/transformers/models/opt/modeling_opt.py>`_ file.
30
31
.. warning::
32
When copying the model code, make sure to review and adhere to the code's copyright and licensing terms.
@@ -62,11 +62,11 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
62
+) -> Dict[int, SequenceOutputs]:
63
64
3. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
65
-4. Replace the attention operation with either :code:`GPTCacheFlowAttention` or :code:`GPTNeoXCacheFlowAttention`, depending on the model's architecture.
65
+4. Replace the attention operation with either :code:`GPTPagedAttention` or :code:`GPTNeoXPagedAttention`, depending on the model's architecture.
66
67
.. note::
68
- Currently, CacheFlow supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
69
- If your model employs a different attention mechanism, you will need to implement a new attention layer in CacheFlow.
68
+ Currently, vLLM supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
69
+ If your model employs a different attention mechanism, you will need to implement a new attention layer in vLLM.
70
71
72
3. (Optional) Implement tensor parallelism support
@@ -91,4 +91,4 @@ While the process is straightforward for most layers, the tensor-parallel layers
91
5. Register your model
92
----------------------
93
94
-Finally, include your :code:`*ForCausalLM` class in `cacheflow/model_executor/models/__init__.py <https://github.com/WoosukKwon/cacheflow/blob/main/cacheflow/model_executor/models/__init__.py>`_ and register it to the :code:`_MODEL_REGISTRY` in `cacheflow/model_executor/model_loader.py <https://github.com/WoosukKwon/cacheflow/blob/main/cacheflow/model_executor/model_loader.py>`_.
94
+Finally, include your :code:`*ForCausalLM` class in `vllm/model_executor/models/__init__.py <https://github.com/WoosukKwon/vllm/blob/main/vllm/model_executor/models/__init__.py>`_ and register it to the :code:`_MODEL_REGISTRY` in `vllm/model_executor/model_loader.py <https://github.com/WoosukKwon/vllm/blob/main/vllm/model_executor/model_loader.py>`_.
docs/source/models/supported_models.rst
+6
-6
@@ -3,8 +3,8 @@
3
Supported Models
4
================
5
6
-CacheFlow supports a variety of generative Transformer models in `HuggingFace Transformers <https://github.com/huggingface/transformers>`_.
7
-The following is the list of model architectures that are currently supported by CacheFlow.
6
+vLLM supports a variety of generative Transformer models in `HuggingFace Transformers <https://github.com/huggingface/transformers>`_.
7
+The following is the list of model architectures that are currently supported by vLLM.
8
Alongside each architecture, we include some popular models that use it.
9
10
.. list-table::
@@ -22,19 +22,19 @@ Alongside each architecture, we include some popular models that use it.
22
* - :code:`OPTForCausalLM`
23
- OPT, OPT-IML
24
25
-If your model uses one of the above model architectures, you can seamlessly run your model with CacheFlow.
25
+If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
26
Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` for instructions on how to implement support for your model.
27
-Alternatively, you can raise an issue on our `GitHub <https://github.com/WoosukKwon/cacheflow/issues>`_ project.
27
+Alternatively, you can raise an issue on our `GitHub <https://github.com/WoosukKwon/vllm/issues>`_ project.
28
29
.. tip::
30
The easiest way to check if your model is supported is to run the program below:
31
32
.. code-block:: python
33
34
- from cacheflow import LLM
34
+ from vllm import LLM
35
36
llm = LLM(model=...) # Name or path of your model
37
output = llm.generate("Hello, my name is")
38
print(output)
39
40
- If CacheFlow successfully generates text, it indicates that your model is supported.
40
+ If vLLM successfully generates text, it indicates that your model is supported.
examples/api_client.py
+1
-1
@@ -1,4 +1,4 @@
1
-"""Example Python client for cacheflow.entrypoints.api_server"""
1
+"""Example Python client for vllm.entrypoints.api_server"""
2
3
import argparse
4
import json
examples/gradio_webserver.py
+2
-2
@@ -6,7 +6,7 @@ import requests
6
7
8
def http_bot(prompt):
9
- headers = {"User-Agent": "Cacheflow Client"}
9
+ headers = {"User-Agent": "vLLM Client"}
10
pload = {
11
"prompt": prompt,
12
"stream": True,
@@ -24,7 +24,7 @@ def http_bot(prompt):
24
def build_demo():
25
with gr.Blocks() as demo:
26
gr.Markdown(
27
- "# Cacheflow text completion demo\n"
27
+ "# vLLM text completion demo\n"
28
)
29
inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")
30
outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
examples/llm_engine_example.py
+1
-1
@@ -1,6 +1,6 @@
1
import argparse
2
3
-from cacheflow import EngineArgs, LLMEngine, SamplingParams
3
+from vllm import EngineArgs, LLMEngine, SamplingParams
4
5
6
def main(args: argparse.Namespace):
examples/offline_inference.py
+1
-1
@@ -1,4 +1,4 @@
1
-from cacheflow import LLM, SamplingParams
1
+from vllm import LLM, SamplingParams
2
3
4
# Sample prompts.
examples/openai_client.py
+1
-1
@@ -1,6 +1,6 @@
1
import openai
2
3
-# Modify OpenAI's API key and API base to use CacheFlow's API server.
3
+# Modify OpenAI's API key and API base to use vLLM's API server.
4
openai.api_key = "EMPTY"
5
openai.api_base = "http://localhost:8000/v1"
6
model = "facebook/opt-125m"
mypy.ini
+2
-2
@@ -3,6 +3,6 @@ python_version = 3.8
3
4
ignore_missing_imports = True
5
6
-files = cacheflow
6
+files = vllm
7
# TODO(woosuk): Include the code from Megatron and HuggingFace.
8
-exclude = cacheflow/model_executor/parallel_utils/|cacheflow/model_executor/models/
8
+exclude = vllm/model_executor/parallel_utils/|vllm/model_executor/models/
setup.py
+13
-13
@@ -75,7 +75,7 @@ ext_modules = []
75
76
# Cache operations.
77
cache_extension = CUDAExtension(
78
- name="cacheflow.cache_ops",
78
+ name="vllm.cache_ops",
79
sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
80
extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
81
)
@@ -83,7 +83,7 @@ ext_modules.append(cache_extension)
83
84
# Attention kernels.
85
attention_extension = CUDAExtension(
86
- name="cacheflow.attention_ops",
86
+ name="vllm.attention_ops",
87
sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
88
extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
89
)
@@ -91,7 +91,7 @@ ext_modules.append(attention_extension)
91
92
# Positional encoding kernels.
93
positional_encoding_extension = CUDAExtension(
94
- name="cacheflow.pos_encoding_ops",
94
+ name="vllm.pos_encoding_ops",
95
sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
96
extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
97
)
@@ -99,7 +99,7 @@ ext_modules.append(positional_encoding_extension)
99
100
# Layer normalization kernels.
101
layernorm_extension = CUDAExtension(
102
- name="cacheflow.layernorm_ops",
102
+ name="vllm.layernorm_ops",
103
sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
104
extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
105
)
@@ -107,7 +107,7 @@ ext_modules.append(layernorm_extension)
107
108
# Activation kernels.
109
activation_extension = CUDAExtension(
110
- name="cacheflow.activation_ops",
110
+ name="vllm.activation_ops",
111
sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
112
extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
113
)
@@ -144,18 +144,18 @@ def get_requirements() -> List[str]:
144
145
146
setuptools.setup(
147
- name="cacheflow",
148
- version=find_version(get_path("cacheflow", "__init__.py")),
149
- author="CacheFlow Team",
150
- author_email="cacheflow@gmail.com",
147
+ name="vllm",
148
+ version=find_version(get_path("vllm", "__init__.py")),
149
+ author="vLLM Team",
150
+ author_email="vllm@gmail.com", # FIXME
151
license="Apache 2.0",
152
- description="CacheFlow: A high-performance LLM Serving System",
152
+ description="vLLM: Easy, Fast, and Cheap LLM Serving with PagedAttention", # FIXME
153
long_description=read_readme(),
154
long_description_content_type="text/markdown",
155
- url="https://github.com/WoosukKwon/cacheflow",
155
+ url="https://github.com/WoosukKwon/vllm",
156
project_urls={
157
- "Homepage": "https://github.com/WoosukKwon/cacheflow",
158
- "Documentation": "https://cacheflow.readthedocs.io/en/latest/",
157
+ "Homepage": "https://github.com/WoosukKwon/vllm",
158
+ "Documentation": "https://vllm.readthedocs.io/en/latest/", # FIXME
159
},
160
classifiers=[
161
"Programming Language :: Python :: 3.8",
tests/kernels/test_activation.py
+1
-1
@@ -1,7 +1,7 @@
1
import torch
2
import torch.nn.functional as F
3
4
-from cacheflow import activation_ops
4
+from vllm import activation_ops
5
6
7
def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
tests/kernels/test_attention.py
+1
-1
@@ -5,7 +5,7 @@ import torch
5
from xformers import ops as xops
6
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
7
8
-from cacheflow import attention_ops
8
+from vllm import attention_ops
9
10
MAX_SEQ_LEN = 4096
11
TEST_SEED = 0
tests/kernels/test_cache.py
+1
-1
@@ -2,7 +2,7 @@ import random
2
3
import torch
4
5
-from cacheflow import cache_ops
5
+from vllm import cache_ops
6
7
8
@torch.inference_mode()
tests/kernels/test_layernorm.py
+1
-1
@@ -1,7 +1,7 @@
1
import torch
2
import torch.nn as nn
3
4
-from cacheflow import layernorm_ops
4
+from vllm import layernorm_ops
5
6
7
class RefRMSNorm(nn.Module):
tests/kernels/test_pos_encoding.py
+1
-1
@@ -4,7 +4,7 @@ import torch
4
import torch.nn as nn
5
import torch.nn.functional as F
6
7
-from cacheflow import pos_encoding_ops
7
+from vllm import pos_encoding_ops
8
9
10
def rotate_half(x: torch.Tensor) -> torch.Tensor:
vllm/__init__.py
new
+21
@@ -0,0 +1,21 @@
1
+from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
2
+from vllm.engine.async_llm_engine import AsyncLLMEngine
3
+from vllm.engine.llm_engine import LLMEngine
4
+from vllm.engine.ray_utils import initialize_cluster
5
+from vllm.entrypoints.llm import LLM
6
+from vllm.outputs import CompletionOutput, RequestOutput
7
+from vllm.sampling_params import SamplingParams
8
+
9
+__version__ = "0.1.0"
10
+
11
+__all__ = [
12
+ "LLM",
13
+ "SamplingParams",
14
+ "RequestOutput",
15
+ "CompletionOutput",
16
+ "LLMEngine",
17
+ "EngineArgs",
18
+ "AsyncLLMEngine",
19
+ "AsyncEngineArgs",
20
+ "initialize_cluster",
21
+]
vllm/block.py
renamed
+1
-1
@@ -1,7 +1,7 @@
1
"""Token blocks."""
2
from typing import List
3
4
-from cacheflow.utils import Device
4
+from vllm.utils import Device
5
6
_BLANK_TOKEN_ID = -1
7
vllm/config.py
renamed
+3
-3
@@ -3,8 +3,8 @@ from typing import Optional
3
import torch
4
from transformers import AutoConfig, PretrainedConfig
5
6
-from cacheflow.logger import init_logger
7
-from cacheflow.utils import get_cpu_memory
6
+from vllm.logger import init_logger
7
+from vllm.utils import get_cpu_memory
8
9
logger = init_logger(__name__)
10
@@ -87,7 +87,7 @@ class CacheConfig:
87
Args:
88
block_size: Size of a cache block in number of tokens.
89
gpu_memory_utilization: Fraction of GPU memory to use for the
90
- CacheFlow execution.
90
+ vLLM execution.
91
swap_space: Size of the CPU swap space per GPU (in GiB).
92
"""
93
def __init__(
vllm/core/__init__.py
renamed
vllm/core/block_manager.py
renamed
+3
-3
@@ -1,9 +1,9 @@
1
"""A block manager that manages token blocks."""
2
from typing import Dict, List, Optional, Set, Tuple
3
4
-from cacheflow.block import PhysicalTokenBlock
5
-from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
6
-from cacheflow.utils import Device
4
+from vllm.block import PhysicalTokenBlock
5
+from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
6
+from vllm.utils import Device
7
8
9
class BlockAllocator:
vllm/core/policy.py
renamed
+1
-1
@@ -1,6 +1,6 @@
1
from typing import List
2
3
-from cacheflow.sequence import SequenceGroup
3
+from vllm.sequence import SequenceGroup
4
5
6
class Policy:
vllm/core/scheduler.py
renamed
+7
-7
@@ -2,13 +2,13 @@ import enum
2
import time
3
from typing import Dict, List, Optional, Tuple
4
5
-from cacheflow.config import CacheConfig, SchedulerConfig
6
-from cacheflow.core.block_manager import BlockSpaceManager
7
-from cacheflow.core.policy import PolicyFactory
8
-from cacheflow.logger import init_logger
9
-from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
10
- SequenceGroupMetadata, SequenceOutputs,
11
- SequenceStatus)
5
+from vllm.config import CacheConfig, SchedulerConfig
6
+from vllm.core.block_manager import BlockSpaceManager
7
+from vllm.core.policy import PolicyFactory
8
+from vllm.logger import init_logger
9
+from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
10
+ SequenceGroupMetadata, SequenceOutputs,
11
+ SequenceStatus)
12
13
logger = init_logger(__name__)
14
vllm/engine/__init__.py
renamed
vllm/engine/arg_utils.py
renamed
+5
-5
@@ -3,13 +3,13 @@ import dataclasses
3
from dataclasses import dataclass
4
from typing import Optional, Tuple
5
6
-from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7
- SchedulerConfig)
6
+from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
+ SchedulerConfig)
8
9
10
@dataclass
11
class EngineArgs:
12
- """Arguments for CacheFlow engine."""
12
+ """Arguments for vLLM engine."""
13
model: str
14
download_dir: Optional[str] = None
15
use_np_weights: bool = False
@@ -33,7 +33,7 @@ class EngineArgs:
33
def add_cli_args(
34
parser: argparse.ArgumentParser,
35
) -> argparse.ArgumentParser:
36
- """Shared CLI arguments for CacheFlow engine."""
36
+ """Shared CLI arguments for vLLM engine."""
37
# Model arguments
38
parser.add_argument('--model', type=str, default='facebook/opt-125m',
39
help='name or path of the huggingface model to use')
@@ -118,7 +118,7 @@ class EngineArgs:
118
119
@dataclass
120
class AsyncEngineArgs(EngineArgs):
121
- """Arguments for asynchronous CacheFlow engine."""
121
+ """Arguments for asynchronous vLLM engine."""
122
engine_use_ray: bool = False
123
disable_log_requests: bool = False
124
vllm/engine/async_llm_engine.py
renamed
+9
-9
@@ -2,12 +2,12 @@ import asyncio
2
import time
3
from typing import Dict, List, Optional
4
5
-from cacheflow.engine.arg_utils import AsyncEngineArgs
6
-from cacheflow.engine.llm_engine import LLMEngine
7
-from cacheflow.engine.ray_utils import initialize_cluster, ray
8
-from cacheflow.logger import init_logger
9
-from cacheflow.outputs import RequestOutput
10
-from cacheflow.sampling_params import SamplingParams
5
+from vllm.engine.arg_utils import AsyncEngineArgs
6
+from vllm.engine.llm_engine import LLMEngine
7
+from vllm.engine.ray_utils import initialize_cluster, ray
8
+from vllm.logger import init_logger
9
+from vllm.outputs import RequestOutput
10
+from vllm.sampling_params import SamplingParams
11
12
logger = init_logger(__name__)
13
@@ -104,7 +104,7 @@ class AsyncLLMEngine:
104
arrival_time = time.time()
105
106
# Create an event to notify us that there is new output from the
107
- # cacheflow engine.
107
+ # vLLM engine.
108
request_event = asyncio.Event()
109
self.request_events[request_id] = request_event
110
@@ -114,7 +114,7 @@ class AsyncLLMEngine:
114
f"sampling params: {sampling_params}, "
115
f"prompt token ids: {prompt_token_ids}.")
116
117
- # Add the request into the cacheflow engine's waiting queue.
117
+ # Add the request into the vLLM engine's waiting queue.
118
if self.engine_use_ray:
119
await self.engine.add_request.remote(
120
request_id, prompt, sampling_params,
@@ -126,7 +126,7 @@ class AsyncLLMEngine:
126
prompt_token_ids=prompt_token_ids,
127
arrival_time=arrival_time)
128
129
- # The cacheflow engine does not have a background loop that keeps
129
+ # The vLLM engine does not have a background loop that keeps
130
# processing incoming requests. Therefore, we need to keep kicking
131
# the engine to process the requests.
132
while True:
vllm/engine/llm_engine.py
renamed
+13
-14
@@ -1,19 +1,18 @@
1
import time
2
from typing import Any, List, Optional
3
4
-from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5
- SchedulerConfig)
6
-from cacheflow.core.scheduler import Scheduler
7
-from cacheflow.engine.arg_utils import EngineArgs
8
-from cacheflow.engine.ray_utils import DeviceID, initialize_cluster, ray
9
-from cacheflow.engine.tokenizer_utils import (detokenize_incrementally,
10
- get_tokenizer)
11
-from cacheflow.logger import init_logger
12
-from cacheflow.outputs import RequestOutput
13
-from cacheflow.sampling_params import SamplingParams
14
-from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
15
-from cacheflow.utils import Counter
16
-from cacheflow.worker.worker import Worker
4
+from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
5
+ SchedulerConfig)
6
+from vllm.core.scheduler import Scheduler
7
+from vllm.engine.arg_utils import EngineArgs
8
+from vllm.engine.ray_utils import DeviceID, initialize_cluster, ray
9
+from vllm.engine.tokenizer_utils import detokenize_incrementally, get_tokenizer
10
+from vllm.logger import init_logger
11
+from vllm.outputs import RequestOutput
12
+from vllm.sampling_params import SamplingParams
13
+from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
14
+from vllm.utils import Counter
15
+from vllm.worker.worker import Worker
16
17
logger = init_logger(__name__)
18
@@ -21,7 +20,7 @@ logger = init_logger(__name__)
20
class LLMEngine:
21
"""An LLM engine that receives requests and generates texts.
22
24
- This is the main class for the CacheFlow LLM engine. It receives requests
23
+ This is the main class for the vLLM engine. It receives requests
24
from clients and generates texts from the LLM. It includes a tokenizer, a
25
language model (possibly distributed across multiple GPUs), and GPU memory
26
space allocated for intermediate states (aka KV cache). This class utilizes
vllm/engine/ray_utils.py
renamed
+1
-1
@@ -6,7 +6,7 @@ try:
6
except ImportError:
7
ray = None
8
9
-from cacheflow.config import ParallelConfig
9
+from vllm.config import ParallelConfig
10
11
DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), device id
12
vllm/engine/tokenizer_utils.py
renamed
+1
-1
@@ -3,7 +3,7 @@ from typing import List, Tuple, Union
3
from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
5
6
-from cacheflow.logger import init_logger
6
+from vllm.logger import init_logger
7
8
logger = init_logger(__name__)
9
vllm/entrypoints/__init__.py
renamed
vllm/entrypoints/api_server.py
renamed
+4
-4
@@ -6,10 +6,10 @@ from fastapi import BackgroundTasks, FastAPI, Request
6
from fastapi.responses import Response, StreamingResponse
7
import uvicorn
8
9
-from cacheflow.engine.arg_utils import AsyncEngineArgs
10
-from cacheflow.engine.async_llm_engine import AsyncLLMEngine
11
-from cacheflow.sampling_params import SamplingParams
12
-from cacheflow.utils import random_uuid
9
+from vllm.engine.arg_utils import AsyncEngineArgs
10
+from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.sampling_params import SamplingParams
12
+from vllm.utils import random_uuid
13
14
TIMEOUT_KEEP_ALIVE = 5 # seconds.
15
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
vllm/entrypoints/llm.py
renamed
+5
-5
@@ -3,11 +3,11 @@ from typing import List, Optional, Union
3
from tqdm import tqdm
4
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6
-from cacheflow.engine.arg_utils import EngineArgs
7
-from cacheflow.engine.llm_engine import LLMEngine
8
-from cacheflow.outputs import RequestOutput
9
-from cacheflow.sampling_params import SamplingParams
10
-from cacheflow.utils import Counter
6
+from vllm.engine.arg_utils import EngineArgs
7
+from vllm.engine.llm_engine import LLMEngine
8
+from vllm.outputs import RequestOutput
9
+from vllm.sampling_params import SamplingParams
10
+from vllm.utils import Counter
11
12
13
class LLM:
vllm/entrypoints/openai/__init__.py
renamed
vllm/entrypoints/openai/api_server.py
renamed
+13
-13
@@ -13,17 +13,17 @@ from fastapi.middleware.cors import CORSMiddleware
13
from fastapi.responses import JSONResponse, StreamingResponse
14
import uvicorn
15
16
-from cacheflow.engine.arg_utils import AsyncEngineArgs
17
-from cacheflow.engine.async_llm_engine import AsyncLLMEngine
18
-from cacheflow.engine.tokenizer_utils import get_tokenizer
19
-from cacheflow.entrypoints.openai.protocol import (
16
+from vllm.engine.arg_utils import AsyncEngineArgs
17
+from vllm.engine.async_llm_engine import AsyncLLMEngine
18
+from vllm.engine.tokenizer_utils import get_tokenizer
19
+from vllm.entrypoints.openai.protocol import (
20
CompletionRequest, CompletionResponse, CompletionResponseChoice,
21
CompletionResponseStreamChoice, CompletionStreamResponse, ErrorResponse,
22
LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
23
-from cacheflow.logger import init_logger
24
-from cacheflow.outputs import RequestOutput
25
-from cacheflow.sampling_params import SamplingParams
26
-from cacheflow.utils import random_uuid
23
+from vllm.logger import init_logger
24
+from vllm.outputs import RequestOutput
25
+from vllm.sampling_params import SamplingParams
26
+from vllm.utils import random_uuid
27
28
TIMEOUT_KEEP_ALIVE = 5 # seconds
29
@@ -93,11 +93,11 @@ async def create_completion(raw_request: Request):
93
for the API specification. This API mimics the OpenAI Completion API.
94
95
NOTE: Currently we do not support the following features:
96
- - echo (since the cacheflow engine does not currently support
96
+ - echo (since the vLLM engine does not currently support
97
getting the logprobs of prompt tokens)
98
- suffix (the language models we currently support do not support
99
suffix)
100
- - logit_bias (to be supported in cacheflow engine)
100
+ - logit_bias (to be supported by vLLM engine)
101
"""
102
request = CompletionRequest(**await raw_request.json())
103
logger.info(f"Received completion request: {request}")
@@ -107,7 +107,7 @@ async def create_completion(raw_request: Request):
107
return error_check_ret
108
109
if request.echo:
110
- # We do not support echo since the cacheflow engine does not
110
+ # We do not support echo since the vLLM engine does not
111
# currently support getting the logprobs of prompt tokens.
112
return create_error_response(HTTPStatus.BAD_REQUEST,
113
"echo is not currently supported")
@@ -118,7 +118,7 @@ async def create_completion(raw_request: Request):
118
"suffix is not currently supported")
119
120
if request.logit_bias is not None:
121
- # TODO: support logit_bias in cacheflow engine.
121
+ # TODO: support logit_bias in vLLM engine.
122
return create_error_response(HTTPStatus.BAD_REQUEST,
123
"logit_bias is not currently supported")
124
@@ -274,7 +274,7 @@ async def create_completion(raw_request: Request):
274
275
if __name__ == "__main__":
276
parser = argparse.ArgumentParser(
277
- description="CacheFlow OpenAI-Compatible RESTful API server."
277
+ description="vLLM OpenAI-Compatible RESTful API server."
278
)
279
parser.add_argument("--host", type=str, default="localhost", help="host name")
280
parser.add_argument("--port", type=int, default=8000, help="port number")
vllm/entrypoints/openai/protocol.py
renamed
+3
-3
@@ -4,7 +4,7 @@ from typing import Dict, List, Literal, Optional, Union
4
5
from pydantic import BaseModel, Field
6
7
-from cacheflow.utils import random_uuid
7
+from vllm.utils import random_uuid
8
9
10
class ErrorResponse(BaseModel):
@@ -34,7 +34,7 @@ class ModelCard(BaseModel):
34
id: str
35
object: str = "model"
36
created: int = Field(default_factory=lambda: int(time.time()))
37
- owned_by: str = "cacheflow"
37
+ owned_by: str = "vllm"
38
root: Optional[str] = None
39
parent: Optional[str] = None
40
permission: List[ModelPermission] = Field(default_factory=list)
@@ -82,7 +82,7 @@ class CompletionRequest(BaseModel):
82
best_of: Optional[int] = None
83
logit_bias: Optional[Dict[str, float]] = None
84
user: Optional[str] = None
85
- # Additional parameters supported by cacheflow
85
+ # Additional parameters supported by vLLM
86
top_k: Optional[int] = -1
87
ignore_eos: Optional[bool] = False
88
use_beam_search: Optional[bool] = False
vllm/logger.py
renamed
+1
-1
@@ -22,7 +22,7 @@ class NewLineFormatter(logging.Formatter):
22
return msg
23
24
25
-_root_logger = logging.getLogger("cacheflow")
25
+_root_logger = logging.getLogger("vllm")
26
_default_handler = None
27
28
vllm/model_executor/__init__.py
new
+10
@@ -0,0 +1,10 @@
1
+from vllm.model_executor.input_metadata import InputMetadata
2
+from vllm.model_executor.model_loader import get_model
3
+from vllm.model_executor.utils import set_random_seed
4
+
5
+
6
+__all__ = [
7
+ "InputMetadata",
8
+ "get_model",
9
+ "set_random_seed",
10
+]
vllm/model_executor/input_metadata.py
renamed
+2
-2
@@ -3,8 +3,8 @@ from typing import Dict, List, Tuple
3
import torch
4
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
5
6
-from cacheflow.sampling_params import SamplingParams
7
-from cacheflow.sequence import SequenceData
6
+from vllm.sampling_params import SamplingParams
7
+from vllm.sequence import SequenceData
8
9
10
class InputMetadata:
vllm/model_executor/layers/__init__.py
renamed
vllm/model_executor/layers/activation.py
renamed
+1
-1
@@ -2,7 +2,7 @@
2
import torch
3
import torch.nn as nn
4
5
-from cacheflow import activation_ops
5
+from vllm import activation_ops
6
7
_ACTIVATION_REGISTRY = {
8
"gelu": nn.GELU(),
vllm/model_executor/layers/attention.py
renamed
+8
-8
@@ -5,16 +5,16 @@ import torch
5
import torch.nn as nn
6
from xformers import ops as xops
7
8
-from cacheflow import attention_ops
9
-from cacheflow import cache_ops
10
-from cacheflow import pos_encoding_ops
11
-from cacheflow.model_executor.input_metadata import InputMetadata
8
+from vllm import attention_ops
9
+from vllm import cache_ops
10
+from vllm import pos_encoding_ops
11
+from vllm.model_executor.input_metadata import InputMetadata
12
13
_SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
14
15
16
-class GPTCacheFlowAttention(nn.Module):
17
- """GPT-style multi-head attention.
16
+class PagedAttention(nn.Module):
17
+ """GPT-style multi-head PagedAttention.
18
19
This class takes flattened 1D query, key, and value tensors as input. The
20
input 1D tensors can be split into three parts: the prompt tokens, the
@@ -164,8 +164,8 @@ class GPTCacheFlowAttention(nn.Module):
164
return output.view(-1, self.num_heads * self.head_size)
165
166
167
-class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
168
- """Attention with GPT-NeoX style rotary embedding."""
167
+class PagedAttentionWithRoPE(PagedAttention):
168
+ """PagedAttention with GPT-NeoX style rotary embedding."""
169
170
def __init__(
171
self,
vllm/model_executor/layers/layernorm.py
renamed
+1
-1
@@ -2,7 +2,7 @@
2
import torch
3
import torch.nn as nn
4
5
-from cacheflow import layernorm_ops
5
+from vllm import layernorm_ops
6
7
8
class RMSNorm(nn.Module):
vllm/model_executor/layers/sampler.py
renamed
+4
-4
@@ -5,11 +5,11 @@ import numpy as np
5
import torch
6
import torch.nn as nn
7
8
-from cacheflow.model_executor.input_metadata import InputMetadata
9
-from cacheflow.model_executor.parallel_utils.tensor_parallel import (
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.parallel_utils.tensor_parallel import (
10
gather_from_tensor_model_parallel_region)
11
-from cacheflow.sampling_params import SamplingParams
12
-from cacheflow.sequence import SequenceOutputs
11
+from vllm.sampling_params import SamplingParams
12
+from vllm.sequence import SequenceOutputs
13
14
15
class Sampler(nn.Module):
vllm/model_executor/model_loader.py
renamed
+4
-4
@@ -5,10 +5,10 @@ import torch
5
import torch.nn as nn
6
from transformers import PretrainedConfig
7
8
-from cacheflow.config import ModelConfig
9
-from cacheflow.model_executor.models import (
10
- GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
11
-from cacheflow.model_executor.weight_utils import initialize_dummy_weights
8
+from vllm.config import ModelConfig
9
+from vllm.model_executor.models import (GPT2LMHeadModel, GPTNeoXForCausalLM,
10
+ LlamaForCausalLM, OPTForCausalLM)
11
+from vllm.model_executor.weight_utils import initialize_dummy_weights
12
13
# TODO(woosuk): Lazy-load the model classes.
14
_MODEL_REGISTRY = {
vllm/model_executor/models/__init__.py
new
+12
@@ -0,0 +1,12 @@
1
+from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
2
+from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
3
+from vllm.model_executor.models.llama import LlamaForCausalLM
4
+from vllm.model_executor.models.opt import OPTForCausalLM
5
+
6
+
7
+__all__ = [
8
+ "GPT2LMHeadModel",
9
+ "GPTNeoXForCausalLM",
10
+ "LlamaForCausalLM",
11
+ "OPTForCausalLM",
12
+]
vllm/model_executor/models/gpt2.py
renamed
+16
-16
@@ -1,6 +1,6 @@
1
# coding=utf-8
2
# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3
-# Copyright 2023 The CacheFlow team.
3
+# Copyright 2023 The vLLM team.
4
# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
5
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
6
#
@@ -26,17 +26,17 @@ import torch
26
from torch import nn
27
from transformers import GPT2Config
28
29
-from cacheflow.model_executor.input_metadata import InputMetadata
30
-from cacheflow.model_executor.layers.activation import get_act_fn
31
-from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
32
-from cacheflow.model_executor.layers.sampler import Sampler
33
-from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
34
- load_tensor_parallel_weights)
35
-from cacheflow.model_executor.parallel_utils.parallel_state import (
29
+from vllm.model_executor.input_metadata import InputMetadata
30
+from vllm.model_executor.layers.activation import get_act_fn
31
+from vllm.model_executor.layers.attention import PagedAttention
32
+from vllm.model_executor.layers.sampler import Sampler
33
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
+ load_tensor_parallel_weights)
35
+from vllm.model_executor.parallel_utils.parallel_state import (
36
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
-from cacheflow.model_executor.parallel_utils.tensor_parallel import (
37
+from vllm.model_executor.parallel_utils.tensor_parallel import (
38
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39
-from cacheflow.sequence import SequenceOutputs
39
+from vllm.sequence import SequenceOutputs
40
41
KVCache = Tuple[torch.Tensor, torch.Tensor]
42
@@ -53,14 +53,14 @@ class GPT2Attention(nn.Module):
53
self.head_dim = self.hidden_size // total_num_heads
54
self.scale = self.head_dim ** -0.5
55
56
- self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size, bias=True,
57
- gather_output=False,
56
+ self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
57
+ bias=True, gather_output=False,
58
perform_initialization=False)
59
- self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size, bias=True,
60
- input_is_parallel=True,
59
+ self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
60
+ bias=True, input_is_parallel=True,
61
perform_initialization=False)
62
- self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
63
- scale=self.scale)
62
+ self.attn = PagedAttention(self.num_heads, self.head_dim,
63
+ scale=self.scale)
64
65
def forward(
66
self,
vllm/model_executor/models/gpt_neox.py
renamed
+13
-12
@@ -1,6 +1,6 @@
1
# coding=utf-8
2
# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
3
-# Copyright 2023 The CacheFlow team.
3
+# Copyright 2023 The vLLM team.
4
# Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved.
5
#
6
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,17 +25,17 @@ import torch
25
from torch import nn
26
from transformers import GPTNeoXConfig
27
28
-from cacheflow.model_executor.input_metadata import InputMetadata
29
-from cacheflow.model_executor.layers.activation import get_act_fn
30
-from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
31
-from cacheflow.model_executor.layers.sampler import Sampler
32
-from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
33
- load_tensor_parallel_weights)
34
-from cacheflow.model_executor.parallel_utils.parallel_state import (
28
+from vllm.model_executor.input_metadata import InputMetadata
29
+from vllm.model_executor.layers.activation import get_act_fn
30
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
31
+from vllm.model_executor.layers.sampler import Sampler
32
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
33
+ load_tensor_parallel_weights)
34
+from vllm.model_executor.parallel_utils.parallel_state import (
35
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
36
-from cacheflow.model_executor.parallel_utils.tensor_parallel import (
36
+from vllm.model_executor.parallel_utils.tensor_parallel import (
37
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38
-from cacheflow.sequence import SequenceOutputs
38
+from vllm.sequence import SequenceOutputs
39
40
KVCache = Tuple[torch.Tensor, torch.Tensor]
41
@@ -63,8 +63,8 @@ class GPTNeoXAttention(nn.Module):
63
scaling = self.head_size ** -0.5
64
rotary_dim = int(self.head_size * config.rotary_pct)
65
assert rotary_dim % 2 == 0
66
- self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_size,
67
- scaling, rotary_dim)
66
+ self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
67
+ scaling, rotary_dim)
68
69
def forward(
70
self,
@@ -149,6 +149,7 @@ class GPTNeoXLayer(nn.Module):
149
150
151
class GPTNeoXModel(nn.Module):
152
+
153
def __init__(self, config: GPTNeoXConfig):
154
super().__init__()
155
self.config = config
vllm/model_executor/models/llama.py
renamed
+14
-14
@@ -1,6 +1,6 @@
1
# coding=utf-8
2
# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
-# Copyright 2023 The CacheFlow team.
3
+# Copyright 2023 The vLLM team.
4
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5
#
6
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
@@ -30,19 +30,19 @@ import torch
30
from torch import nn
31
from transformers import LlamaConfig
32
33
-from cacheflow.sequence import SequenceOutputs
34
-from cacheflow.model_executor.input_metadata import InputMetadata
35
-from cacheflow.model_executor.layers.activation import SiluAndMul
36
-from cacheflow.model_executor.layers.layernorm import RMSNorm
37
-from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
38
-from cacheflow.model_executor.layers.sampler import Sampler
39
-from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
40
- load_tensor_parallel_weights)
41
-from cacheflow.model_executor.parallel_utils.parallel_state import (
33
+from vllm.sequence import SequenceOutputs
34
+from vllm.model_executor.input_metadata import InputMetadata
35
+from vllm.model_executor.layers.activation import SiluAndMul
36
+from vllm.model_executor.layers.layernorm import RMSNorm
37
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38
+from vllm.model_executor.layers.sampler import Sampler
39
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
40
+ load_tensor_parallel_weights)
41
+from vllm.model_executor.parallel_utils.parallel_state import (
42
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43
-from cacheflow.model_executor.parallel_utils.tensor_parallel import (
43
+from vllm.model_executor.parallel_utils.tensor_parallel import (
44
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
45
-from cacheflow.sequence import SequenceOutputs
45
+from vllm.sequence import SequenceOutputs
46
47
KVCache = Tuple[torch.Tensor, torch.Tensor]
48
@@ -104,8 +104,8 @@ class LlamaAttention(nn.Module):
104
input_is_parallel=True,
105
perform_initialization=False,
106
)
107
- self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_dim,
108
- self.scaling, rotary_dim=self.head_dim)
107
+ self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_dim,
108
+ self.scaling, rotary_dim=self.head_dim)
109
110
def forward(
111
self,
vllm/model_executor/models/opt.py
renamed
+12
-12
@@ -1,6 +1,6 @@
1
# coding=utf-8
2
# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
3
-# Copyright 2023 The CacheFlow team.
3
+# Copyright 2023 The vLLM team.
4
# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.
5
#
6
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,17 +25,17 @@ import torch
25
from torch import nn
26
from transformers import OPTConfig
27
28
-from cacheflow.model_executor.input_metadata import InputMetadata
29
-from cacheflow.model_executor.layers.activation import get_act_fn
30
-from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
31
-from cacheflow.model_executor.layers.sampler import Sampler
32
-from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
33
- load_tensor_parallel_weights)
34
-from cacheflow.model_executor.parallel_utils.parallel_state import (
28
+from vllm.model_executor.input_metadata import InputMetadata
29
+from vllm.model_executor.layers.activation import get_act_fn
30
+from vllm.model_executor.layers.attention import PagedAttention
31
+from vllm.model_executor.layers.sampler import Sampler
32
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
33
+ load_tensor_parallel_weights)
34
+from vllm.model_executor.parallel_utils.parallel_state import (
35
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
36
-from cacheflow.model_executor.parallel_utils.tensor_parallel import (
36
+from vllm.model_executor.parallel_utils.tensor_parallel import (
37
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38
-from cacheflow.sequence import SequenceOutputs
38
+from vllm.sequence import SequenceOutputs
39
40
KVCache = Tuple[torch.Tensor, torch.Tensor]
41
@@ -75,8 +75,8 @@ class OPTAttention(nn.Module):
75
self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
76
input_is_parallel=True,
77
perform_initialization=False)
78
- self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
79
- scale=self.scaling)
78
+ self.attn = PagedAttention(self.num_heads, self.head_dim,
79
+ scale=self.scaling)
80
81
def forward(
82
self,
vllm/model_executor/parallel_utils/README.md
renamed
vllm/model_executor/parallel_utils/__init__.py
renamed
+2
-2
@@ -1,5 +1,5 @@
1
-import cacheflow.model_executor.parallel_utils.parallel_state
2
-import cacheflow.model_executor.parallel_utils.tensor_parallel
1
+import vllm.model_executor.parallel_utils.parallel_state
2
+import vllm.model_executor.parallel_utils.tensor_parallel
3
4
# Alias parallel_state as mpu, its legacy name
5
mpu = parallel_state
vllm/model_executor/parallel_utils/parallel_state.py
renamed
+1
-1
@@ -1,4 +1,4 @@
1
-# Copyright 2023 The CacheFlow team.
1
+# Copyright 2023 The vLLM team.
2
# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
vllm/model_executor/parallel_utils/tensor_parallel/__init__.py
renamed
vllm/model_executor/parallel_utils/tensor_parallel/layers.py
renamed
+2
-2
@@ -1,4 +1,4 @@
1
-# Copyright 2023 The CacheFlow team.
1
+# Copyright 2023 The vLLM team.
2
# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
@@ -11,7 +11,7 @@ import torch.nn.functional as F
11
import torch.nn.init as init
12
from torch.nn.parameter import Parameter
13
14
-from cacheflow.model_executor.parallel_utils.parallel_state import (
14
+from vllm.model_executor.parallel_utils.parallel_state import (
15
get_tensor_model_parallel_rank,
16
get_tensor_model_parallel_world_size,
17
get_all_reduce_launcher,
vllm/model_executor/parallel_utils/tensor_parallel/mappings.py
renamed
+2
-2
@@ -1,10 +1,10 @@
1
-# Copyright 2023 The CacheFlow team.
1
+# Copyright 2023 The vLLM team.
2
# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/mappings.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5
import torch
6
7
-from cacheflow.model_executor.parallel_utils.parallel_state import (
7
+from vllm.model_executor.parallel_utils.parallel_state import (
8
get_tensor_model_parallel_rank,
9
get_tensor_model_parallel_world_size,
10
get_tensor_model_parallel_group,
vllm/model_executor/parallel_utils/tensor_parallel/random.py
renamed
+2
-2
@@ -1,4 +1,4 @@
1
-# Copyright 2023 The CacheFlow team.
1
+# Copyright 2023 The vLLM team.
2
# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/random.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
@@ -11,7 +11,7 @@ import torch
11
from torch import _C
12
from torch.cuda import _lazy_call, device as device_ctx_manager
13
14
-from cacheflow.model_executor.parallel_utils.parallel_state import (
14
+from vllm.model_executor.parallel_utils.parallel_state import (
15
get_tensor_model_parallel_rank,
16
)
17
vllm/model_executor/parallel_utils/tensor_parallel/utils.py
renamed
+1
-1
@@ -1,4 +1,4 @@
1
-# Copyright 2023 The CacheFlow team.
1
+# Copyright 2023 The vLLM team.
2
# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
vllm/model_executor/utils.py
renamed
+2
-2
@@ -4,8 +4,8 @@ import random
4
import numpy as np
5
import torch
6
7
-from cacheflow.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8
-from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
7
+from vllm.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8
+from vllm.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
10
11
def set_random_seed(seed: int) -> None:
vllm/model_executor/weight_utils.py
renamed
vllm/outputs.py
renamed
+1
-1
@@ -1,6 +1,6 @@
1
from typing import Dict, List, Optional
2
3
-from cacheflow.sequence import SequenceGroup, SequenceStatus
3
+from vllm.sequence import SequenceGroup, SequenceStatus
4
5
6
class CompletionOutput:
vllm/sampling_params.py
renamed
vllm/sequence.py
renamed
+2
-2
@@ -2,8 +2,8 @@ import copy
2
import enum
3
from typing import Dict, List, Optional, Union
4
5
-from cacheflow.block import LogicalTokenBlock
6
-from cacheflow.sampling_params import SamplingParams
5
+from vllm.block import LogicalTokenBlock
6
+from vllm.sampling_params import SamplingParams
7
8
9
class SequenceStatus(enum.Enum):
vllm/utils.py
renamed
vllm/worker/__init__.py
renamed
vllm/worker/cache_engine.py
renamed
+2
-2
@@ -3,8 +3,8 @@ from typing import Dict, List, Tuple
3
4
import torch
5
6
-from cacheflow import cache_ops
7
-from cacheflow.config import CacheConfig, ModelConfig, ParallelConfig
6
+from vllm import cache_ops
7
+from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
9
KVCache = Tuple[torch.Tensor, torch.Tensor]
10
vllm/worker/worker.py
renamed
+8
-9
@@ -3,16 +3,15 @@ from typing import Dict, List, Tuple
3
4
import torch
5
6
-from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7
- SchedulerConfig)
8
-from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
9
-from cacheflow.model_executor.parallel_utils.parallel_state import (
6
+from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
+ SchedulerConfig)
8
+from vllm.model_executor import get_model, InputMetadata, set_random_seed
9
+from vllm.model_executor.parallel_utils.parallel_state import (
10
initialize_model_parallel, initialize_all_reduce_launcher)
11
-from cacheflow.sampling_params import SamplingParams
12
-from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
13
- SequenceOutputs)
14
-from cacheflow.worker.cache_engine import CacheEngine
15
-from cacheflow.utils import get_gpu_memory
11
+from vllm.sampling_params import SamplingParams
12
+from vllm.sequence import SequenceData, SequenceGroupMetadata, SequenceOutputs
13
+from vllm.worker.cache_engine import CacheEngine
14
+from vllm.utils import get_gpu_memory
15
16
17
class Worker: