@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b98ba15c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Change the name to vLLM (#150)

Woosuk Kwon committed Jun 17, 2023 at 03:07 UTC 0b98ba15c744f1dfb0ea4f2135e85ca23d572ae1
90 files changed +342 -339
CONTRIBUTING.md
+6 -6
@@ -1,6 +1,6 @@
1 -# Contributing to CacheFlow
1 +# Contributing to vLLM
2
3 -Thank you for your interest in contributing to CacheFlow!
3 +Thank you for your interest in contributing to vLLM!
4 Our community is open to everyone and welcomes all kinds of contributions, no matter how small or large.
5 There are several ways you can contribute to the project:
6
@@ -11,9 +11,9 @@ There are several ways you can contribute to the project:
11 However, remember that contributions aren't just about code.
12 We believe in the power of community support; thus, answering queries, assisting others, and enhancing the documentation are highly regarded and beneficial contributions.
13
14 -Finally, one of the most impactful ways to support us is by raising awareness about CacheFlow.
14 +Finally, one of the most impactful ways to support us is by raising awareness about vLLM.
15 Talk about it in your blog posts, highlighting how it's driving your incredible projects.
16 -Express your support on Twitter if CacheFlow aids you, or simply offer your appreciation by starring our repository.
16 +Express your support on Twitter if vLLM aids you, or simply offer your appreciation by starring our repository.
17
18
19 ## Setup for development
@@ -70,5 +70,5 @@ If a comment isn't clear or you disagree with a suggestion, feel free to ask for
70
71 ### Thank You
72
73 -Finally, thank you for taking the time to read these guidelines and for your interest in contributing to CacheFlow.
74 -Your contributions make CacheFlow a great tool for everyone!
73 +Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM.
74 +Your contributions make vLLM a great tool for everyone!
README.md
+5 -5
@@ -1,4 +1,4 @@
1 -# CacheFlow
1 +# vLLM
2
3 ## Build from source
4
@@ -28,7 +28,7 @@ python examples/simple_server.py --help
28 To start the server:
29 ```bash
30 ray start --head
31 -python -m cacheflow.entrypoints.fastapi_server # --model <your_model>
31 +python -m vllm.entrypoints.fastapi_server # --model <your_model>
32 ```
33
34 To test the server:
@@ -45,9 +45,9 @@ pip install gradio
45
46 Start the server:
47 ```bash
48 -python -m cacheflow.http_frontend.fastapi_frontend
48 +python -m vllm.http_frontend.fastapi_frontend
49 # At another terminal
50 -python -m cacheflow.http_frontend.gradio_webserver
50 +python -m vllm.http_frontend.gradio_webserver
51 ```
52
53 ## Load LLaMA weights
@@ -62,5 +62,5 @@ Since LLaMA weight is not fully public, we cannot directly download the LLaMA we
62 2. For all the commands above, specify the model with `--model /output/path/llama-7b` to load the model. For example:
63 ```bash
64 python simple_server.py --model /output/path/llama-7b
65 - python -m cacheflow.http_frontend.fastapi_frontend --model /output/path/llama-7b
65 + python -m vllm.http_frontend.fastapi_frontend --model /output/path/llama-7b
66 ```
benchmarks/README.md
+1 -1
@@ -1,4 +1,4 @@
1 -# Benchmarking CacheFlow
1 +# Benchmarking vLLM
2
3 ## Downloading the ShareGPT dataset
4
benchmarks/benchmark_async_llm_server.py
+1 -1
@@ -11,7 +11,7 @@ def main(args: argparse.Namespace):
11 for i in range(args.n_threads)]
12
13 api_url = f"http://{args.host}:{args.port}/generate"
14 - headers = {"User-Agent": "CacheFlow Benchmark Client"}
14 + headers = {"User-Agent": "vLLM Benchmark Client"}
15 ploads = [{
16 "prompt": p,
17 "max_tokens": args.max_tokens,
benchmarks/benchmark_latency.py
+1 -1
@@ -6,7 +6,7 @@ import numpy as np
6 import torch
7 from tqdm import tqdm
8
9 -from cacheflow import LLM, SamplingParams
9 +from vllm import LLM, SamplingParams
10
11
12 def main(args: argparse.Namespace):
benchmarks/benchmark_serving.py
+5 -5
@@ -1,8 +1,8 @@
1 """Benchmark online serving throughput.
2
3 On the server side, run one of the following commands:
4 - (CacheFlow backend)
5 - python -m cacheflow.entrypoints.api_server \
4 + (vLLM backend)
5 + python -m vllm.entrypoints.api_server \
6 --disable-log-requests --model <your_model>
7
8 (TGI backend)
@@ -114,7 +114,7 @@ async def send_request(
114 request_start_time = time.time()
115
116 headers = {"User-Agent": "Benchmark Client"}
117 - if backend == "cacheflow":
117 + if backend == "vllm":
118 pload = {
119 "prompt": prompt,
120 "n": 1,
@@ -213,8 +213,8 @@ def main(args: argparse.Namespace):
213 if __name__ == "__main__":
214 parser = argparse.ArgumentParser(
215 description="Benchmark the online serving throughput.")
216 - parser.add_argument("--backend", type=str, default="cacheflow",
217 - choices=["cacheflow", "tgi"])
216 + parser.add_argument("--backend", type=str, default="vllm",
217 + choices=["vllm", "tgi"])
218 parser.add_argument("--host", type=str, default="localhost")
219 parser.add_argument("--port", type=int, default=8001)
220 parser.add_argument("--dataset", type=str, required=True,
benchmarks/benchmark_throughput.py
+8 -7
@@ -5,12 +5,13 @@ import random
5 import time
6 from typing import List, Tuple
7
8 -from cacheflow import LLM, SamplingParams
8 import torch
9 from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
10 PreTrainedTokenizerBase)
11 from tqdm import tqdm
12
13 +from vllm import LLM, SamplingParams
14 +
15
16 def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
17 config = AutoConfig.from_pretrained(model_name)
@@ -70,7 +71,7 @@ def sample_requests(
71 return sampled_requests
72
73
73 -def run_cacheflow(
74 +def run_vllm(
75 requests: List[Tuple[str, int, int]],
76 model: str,
77 tensor_parallel_size: int,
@@ -172,8 +173,8 @@ def main(args: argparse.Namespace):
173 tokenizer = get_tokenizer(args.model)
174 requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
175
175 - if args.backend == "cacheflow":
176 - elapsed_time = run_cacheflow(
176 + if args.backend == "vllm":
177 + elapsed_time = run_vllm(
178 requests, args.model, args.tensor_parallel_size, args.seed, args.n,
179 args.use_beam_search)
180 elif args.backend == "hf":
@@ -192,8 +193,8 @@ def main(args: argparse.Namespace):
193
194 if __name__ == "__main__":
195 parser = argparse.ArgumentParser(description="Benchmark the throughput.")
195 - parser.add_argument("--backend", type=str, choices=["cacheflow", "hf"],
196 - default="cacheflow")
196 + parser.add_argument("--backend", type=str, choices=["vllm", "hf"],
197 + default="vllm")
198 parser.add_argument("--dataset", type=str, required=True,
199 help="Path to the dataset.")
200 parser.add_argument("--model", type=str, default="facebook/opt-125m")
@@ -207,7 +208,7 @@ if __name__ == "__main__":
208 parser.add_argument("--hf-max-batch-size", type=int, default=None,
209 help="Maximum batch size for HF backend.")
210 args = parser.parse_args()
210 - if args.backend == "cacheflow":
211 + if args.backend == "vllm":
212 if args.hf_max_batch_size is not None:
213 raise ValueError("HF max batch size is only for HF backend.")
214 elif args.backend == "hf":
cacheflow/__init__.py deleted
-18
@@ -1,18 +0,0 @@
1 -from cacheflow.engine.arg_utils import EngineArgs
2 -from cacheflow.engine.llm_engine import LLMEngine
3 -from cacheflow.engine.ray_utils import initialize_cluster
4 -from cacheflow.entrypoints.llm import LLM
5 -from cacheflow.outputs import CompletionOutput, RequestOutput
6 -from cacheflow.sampling_params import SamplingParams
7 -
8 -__version__ = "0.1.0"
9 -
10 -__all__ = [
11 - "LLM",
12 - "SamplingParams",
13 - "RequestOutput",
14 - "CompletionOutput",
15 - "LLMEngine",
16 - "EngineArgs",
17 - "initialize_cluster",
18 -]
cacheflow/model_executor/__init__.py deleted
-10
@@ -1,10 +0,0 @@
1 -from cacheflow.model_executor.input_metadata import InputMetadata
2 -from cacheflow.model_executor.model_loader import get_model
3 -from cacheflow.model_executor.utils import set_random_seed
4 -
5 -
6 -__all__ = [
7 - "InputMetadata",
8 - "get_model",
9 - "set_random_seed",
10 -]
cacheflow/model_executor/models/__init__.py deleted
-12
@@ -1,12 +0,0 @@
1 -from cacheflow.model_executor.models.gpt_neox import GPTNeoXForCausalLM
2 -from cacheflow.model_executor.models.gpt2 import GPT2LMHeadModel
3 -from cacheflow.model_executor.models.llama import LlamaForCausalLM
4 -from cacheflow.model_executor.models.opt import OPTForCausalLM
5 -
6 -
7 -__all__ = [
8 - "GPT2LMHeadModel",
9 - "GPTNeoXForCausalLM",
10 - "LlamaForCausalLM",
11 - "OPTForCausalLM",
12 -]
csrc/activation_kernels.cu
+3 -3
@@ -1,7 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 -namespace cacheflow {
4 +namespace vllm {
5
6 template<typename T>
7 __device__ __forceinline__ T silu(const T& x) {
@@ -22,7 +22,7 @@ __global__ void silu_and_mul_kernel(
22 }
23 }
24
25 -} // namespace cacheflow
25 +} // namespace vllm
26
27 void silu_and_mul(
28 torch::Tensor& out, // [num_tokens, d]
@@ -40,7 +40,7 @@ void silu_and_mul(
40 input.scalar_type(),
41 "silu_and_mul_kernel",
42 [&] {
43 - cacheflow::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
43 + vllm::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
44 out.data_ptr<scalar_t>(),
45 input.data_ptr<scalar_t>(),
46 d);
csrc/attention/attention_generic.cuh
+3 -3
@@ -1,6 +1,6 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
3 - * Copyright (c) 2023, The CacheFlow team.
3 + * Copyright (c) 2023, The vLLM team.
4 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +19,7 @@
19
20 #include <stdint.h>
21
22 -namespace cacheflow {
22 +namespace vllm {
23
24 // A vector type to store Q, K, V elements.
25 template<typename T, int VEC_SIZE>
@@ -61,4 +61,4 @@ inline __device__ void zero(T& dst) {
61 dst = tmp.raw;
62 }
63
64 -} // namespace cacheflow
64 +} // namespace vllm
csrc/attention/attention_kernels.cu
+4 -4
@@ -1,6 +1,6 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 - * Copyright (c) 2023, The CacheFlow team.
3 + * Copyright (c) 2023, The vLLM team.
4 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,7 +27,7 @@
27 #define MAX(a, b) ((a) > (b) ? (a) : (b))
28 #define MIN(a, b) ((a) < (b) ? (a) : (b))
29
30 -namespace cacheflow {
30 +namespace vllm {
31
32 // Utility function for attention softmax.
33 template<int NUM_WARPS>
@@ -315,10 +315,10 @@ __global__ void single_query_cached_kv_attention_kernel(
315 }
316 }
317
318 -} // namespace cacheflow
318 +} // namespace vllm
319
320 #define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
321 - cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
321 + vllm::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
322 <<<grid, block, shared_mem_size, stream>>>( \
323 out_ptr, \
324 query_ptr, \
csrc/attention/attention_utils.cuh
+3 -3
@@ -1,6 +1,6 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 - * Copyright (c) 2023, The CacheFlow team.
3 + * Copyright (c) 2023, The vLLM team.
4 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +22,7 @@
22 #include <float.h>
23 #include <type_traits>
24
25 -namespace cacheflow {
25 +namespace vllm {
26
27 // Q*K^T operation.
28 template<int THREAD_GROUP_SIZE, typename Vec, int N>
@@ -52,4 +52,4 @@ struct Qk_dot {
52 }
53 };
54
55 -} // namespace cacheflow
55 +} // namespace vllm
csrc/attention/dtype_bfloat16.cuh
+3 -3
@@ -1,7 +1,7 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4 - * Copyright (c) 2023, The CacheFlow team.
4 + * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,7 +25,7 @@
25 #include <cuda_fp16.h>
26 #include <stdint.h>
27
28 -namespace cacheflow {
28 +namespace vllm {
29
30 // Define custom BF16 vector data types.
31 struct bf16_4_t {
@@ -420,4 +420,4 @@ inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
420 #endif
421 }
422
423 -} // namespace cacheflow
423 +} // namespace vllm
csrc/attention/dtype_float16.cuh
+3 -3
@@ -1,7 +1,7 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4 - * Copyright (c) 2023, The CacheFlow team.
4 + * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +23,7 @@
23
24 #include <stdint.h>
25
26 -namespace cacheflow {
26 +namespace vllm {
27
28 // FP16 vector types for Q, K, V.
29 template<>
@@ -441,4 +441,4 @@ inline __device__ Float8_ to_float(uint4 u) {
441 return tmp;
442 }
443
444 -} // namespace cacheflow
444 +} // namespace vllm
csrc/attention/dtype_float32.cuh
+3 -3
@@ -1,7 +1,7 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4 - * Copyright (c) 2023, The CacheFlow team.
4 + * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +22,7 @@
22
23 #include <stdint.h>
24
25 -namespace cacheflow {
25 +namespace vllm {
26
27 // Define custom FP32 vector data types.
28 struct Float4_ {
@@ -265,4 +265,4 @@ inline __device__ Float8_ to_float(Float8_ u) {
265 return u;
266 }
267
268 -} // namespace cacheflow
268 +} // namespace vllm
csrc/cache_kernels.cu
+9 -9
@@ -46,7 +46,7 @@ void swap_blocks(
46 }
47 }
48
49 -namespace cacheflow {
49 +namespace vllm {
50
51 // Grid: (num_layers, num_pairs)
52 template<typename scalar_t>
@@ -77,7 +77,7 @@ __global__ void copy_blocks_kernel(
77 }
78 }
79
80 -} // namespace cacheflow
80 +} // namespace vllm
81
82 void copy_blocks(
83 std::vector<torch::Tensor>& key_caches,
@@ -129,7 +129,7 @@ void copy_blocks(
129 at::ScalarType::Half,
130 at::ScalarType::BFloat16,
131 key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
132 - cacheflow::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
132 + vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133 key_cache_ptrs_tensor.data_ptr<int64_t>(),
134 value_cache_ptrs_tensor.data_ptr<int64_t>(),
135 block_mapping_tensor.data_ptr<int>(),
@@ -137,7 +137,7 @@ void copy_blocks(
137 }));
138 }
139
140 -namespace cacheflow {
140 +namespace vllm {
141
142 template<typename scalar_t>
143 __global__ void reshape_and_cache_kernel(
@@ -181,7 +181,7 @@ __global__ void reshape_and_cache_kernel(
181 }
182 }
183
184 -} // namespace cacheflow
184 +} // namespace vllm
185
186 void reshape_and_cache(
187 torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -208,7 +208,7 @@ void reshape_and_cache(
208 key.scalar_type(),
209 "reshape_and_cache_kernel",
210 [&] {
211 - cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
211 + vllm::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
212 key.data_ptr<scalar_t>(),
213 value.data_ptr<scalar_t>(),
214 key_cache.data_ptr<scalar_t>(),
@@ -223,7 +223,7 @@ void reshape_and_cache(
223 });
224 }
225
226 -namespace cacheflow {
226 +namespace vllm {
227
228 // Grid: (num_blocks, block_size).
229 template<typename scalar_t>
@@ -343,7 +343,7 @@ __global__ void gather_cached_kv_kernel_optimized(
343 }
344 }
345
346 -} // namespace cacheflow
346 +} // namespace vllm
347
348 void gather_cached_kv(
349 torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
@@ -370,7 +370,7 @@ void gather_cached_kv(
370 key.scalar_type(),
371 "gather_cached_kv_kernel_optimized",
372 [&] {
373 - cacheflow::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
373 + vllm::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
374 key.data_ptr<scalar_t>(),
375 value.data_ptr<scalar_t>(),
376 key_cache.data_ptr<scalar_t>(),
csrc/layernorm_kernels.cu
+3 -3
@@ -3,7 +3,7 @@
3
4 #include "reduction_utils.cuh"
5
6 -namespace cacheflow {
6 +namespace vllm {
7
8 // TODO(woosuk): Further optimize this kernel.
9 template<typename scalar_t>
@@ -33,7 +33,7 @@ __global__ void rms_norm_kernel(
33 }
34 }
35
36 -} // namespace cacheflow
36 +} // namespace vllm
37
38 void rms_norm(
39 torch::Tensor& out, // [num_tokens, hidden_size]
@@ -52,7 +52,7 @@ void rms_norm(
52 input.scalar_type(),
53 "rms_norm_kernel",
54 [&] {
55 - cacheflow::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
55 + vllm::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
56 out.data_ptr<scalar_t>(),
57 input.data_ptr<scalar_t>(),
58 weight.data_ptr<scalar_t>(),
csrc/pos_encoding_kernels.cu
+3 -3
@@ -1,7 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 -namespace cacheflow {
4 +namespace vllm {
5
6 template<typename scalar_t>
7 __global__ void rotary_embedding_neox_kernel(
@@ -46,7 +46,7 @@ __global__ void rotary_embedding_neox_kernel(
46 }
47 }
48
49 -} // namespace cacheflow
49 +} // namespace vllm
50
51 void rotary_embedding_neox(
52 torch::Tensor& positions, // [num_tokens]
@@ -70,7 +70,7 @@ void rotary_embedding_neox(
70 query.scalar_type(),
71 "rotary_embedding_neox",
72 [&] {
73 - cacheflow::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
73 + vllm::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
74 positions.data_ptr<int64_t>(),
75 query.data_ptr<scalar_t>(),
76 key.data_ptr<scalar_t>(),
csrc/reduction_utils.cuh
+3 -3
@@ -1,6 +1,6 @@
1 /*
2 * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
3 - * Copyright (c) 2023, The CacheFlow team.
3 + * Copyright (c) 2023, The vLLM team.
4 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +17,7 @@
17 */
18 #pragma once
19
20 -namespace cacheflow {
20 +namespace vllm {
21
22 template<typename T>
23 __inline__ __device__ T warpReduceSum(T val) {
@@ -48,4 +48,4 @@ __inline__ __device__ T blockReduceSum(T val) {
48 return val;
49 }
50
51 -} // namespace cacheflow
51 +} // namespace vllm
docs/README.md
+1 -1
@@ -1,4 +1,4 @@
1 -# CacheFlow documents
1 +# vLLM documents
2
3 ## Build the docs
4
docs/source/conf.py
+4 -4
@@ -17,9 +17,9 @@
17
18 # -- Project information -----------------------------------------------------
19
20 -project = 'CacheFlow'
21 -copyright = '2023, CacheFlow Team'
22 -author = 'the CacheFlow Team'
20 +project = 'vLLM'
21 +copyright = '2023, vLLM Team'
22 +author = 'the vLLM Team'
23
24
25 # -- General configuration ---------------------------------------------------
@@ -55,7 +55,7 @@ html_title = project
55 html_theme = 'sphinx_book_theme'
56 html_theme_options = {
57 'path_to_docs': 'docs/source',
58 - 'repository_url': 'https://github.com/WoosukKwon/cacheflow',
58 + 'repository_url': 'https://github.com/WoosukKwon/vllm',
59 'use_repository_button': True,
60 }
61
docs/source/getting_started/installation.rst
+11 -11
@@ -1,8 +1,8 @@
1 Installation
2 ============
3
4 -CacheFlow is a Python library that includes some C++ and CUDA code.
5 -CacheFlow can run on systems that meet the following requirements:
4 +vLLM is a Python library that includes some C++ and CUDA code.
5 +vLLM can run on systems that meet the following requirements:
6
7 * OS: Linux
8 * Python: 3.8 or higher
@@ -10,23 +10,23 @@ CacheFlow can run on systems that meet the following requirements:
10 * GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, etc.)
11
12 .. note::
13 - As of now, CacheFlow does not support CUDA 12.
13 + As of now, vLLM does not support CUDA 12.
14 If you are using Hopper or Lovelace GPUs, please use CUDA 11.8.
15
16 .. tip::
17 - If you have trouble installing CacheFlow, we recommend using the NVIDIA PyTorch Docker image.
17 + If you have trouble installing vLLM, we recommend using the NVIDIA PyTorch Docker image.
18
19 .. code-block:: console
20
21 $ # Pull the Docker image with CUDA 11.8.
22 $ docker run --gpus all -it --rm --shm-size=8g nvcr.io/nvidia/pytorch:22.12-py3
23
24 - Inside the Docker container, please execute :code:`pip uninstall torch` before installing CacheFlow.
24 + Inside the Docker container, please execute :code:`pip uninstall torch` before installing vLLM.
25
26 Install with pip
27 ----------------
28
29 -You can install CacheFlow using pip:
29 +You can install vLLM using pip:
30
31 .. code-block:: console
32
@@ -34,8 +34,8 @@ You can install CacheFlow using pip:
34 $ conda create -n myenv python=3.8 -y
35 $ conda activate myenv
36
37 - $ # Install CacheFlow.
38 - $ pip install cacheflow # This may take 5-10 minutes.
37 + $ # Install vLLM.
38 + $ pip install vllm # This may take 5-10 minutes.
39
40
41 .. _build_from_source:
@@ -43,10 +43,10 @@ You can install CacheFlow using pip:
43 Build from source
44 -----------------
45
46 -You can also build and install CacheFlow from source.
46 +You can also build and install vLLM from source.
47
48 .. code-block:: console
49
50 - $ git clone https://github.com/WoosukKwon/cacheflow.git
51 - $ cd cacheflow
50 + $ git clone https://github.com/WoosukKwon/vllm.git
51 + $ cd vllm
52 $ pip install -e . # This may take 5-10 minutes.
docs/source/getting_started/quickstart.rst
+1 -1
@@ -8,7 +8,7 @@ Placeholder.
8
9 .. code-block:: python
10
11 - from cacheflow import LLM, SamplingParams
11 + from vllm import LLM, SamplingParams
12
13 # Sample prompts.
14 prompts = [
docs/source/index.rst
+2 -2
@@ -1,5 +1,5 @@
1 -Welcome to CacheFlow!
2 -=====================
1 +Welcome to vLLM!
2 +================
3
4 Documentation
5 -------------
docs/source/models/adding_model.rst
+11 -11
@@ -3,30 +3,30 @@
3 Adding a New Model
4 ==================
5
6 -This document provides a high-level guide on integrating a `HuggingFace Transformers <https://github.com/huggingface/transformers>`_ model into CacheFlow.
6 +This document provides a high-level guide on integrating a `HuggingFace Transformers <https://github.com/huggingface/transformers>`_ model into vLLM.
7
8 .. note::
9 The complexity of adding a new model depends heavily on the model's architecture.
10 - The process is considerably straightforward if the model shares a similar architecture with an existing model in CacheFlow.
10 + The process is considerably straightforward if the model shares a similar architecture with an existing model in vLLM.
11 However, for models that include new operators (e.g., a new attention mechanism), the process can be a bit more complex.
12
13 .. tip::
14 - If you are encountering issues while integrating your model into CacheFlow, feel free to open an issue on our `GitHub <https://github.com/WoosukKwon/cacheflow/issues>`_ repository.
14 + If you are encountering issues while integrating your model into vLLM, feel free to open an issue on our `GitHub <https://github.com/WoosukKwon/vllm/issues>`_ repository.
15 We will be happy to help you out!
16
17
18 -0. Fork the CacheFlow repository
18 +0. Fork the vLLM repository
19 --------------------------------
20
21 -Start by forking our `GitHub <https://github.com/WoosukKwon/cacheflow/issues>`_ repository and then :ref:`build it from source <build_from_source>`.
21 +Start by forking our `GitHub <https://github.com/WoosukKwon/vllm/issues>`_ repository and then :ref:`build it from source <build_from_source>`.
22 This gives you the ability to modify the codebase and test your model.
23
24
25 1. Bring your model code
26 ------------------------
27
28 -Clone the PyTorch model code from the HuggingFace Transformers repository and put it into the `cacheflow/model_executor/models <https://github.com/WoosukKwon/cacheflow/tree/main/cacheflow/model_executor/models>`_ directory.
29 -For instance, CacheFlow's `OPT model <https://github.com/WoosukKwon/cacheflow/blob/main/cacheflow/model_executor/models/opt.py>`_ was adpated from the HuggingFace's `modeling_opt.py <https://github.com/huggingface/transformers/blob/main/src/transformers/models/opt/modeling_opt.py>`_ file.
28 +Clone the PyTorch model code from the HuggingFace Transformers repository and put it into the `vllm/model_executor/models <https://github.com/WoosukKwon/vllm/tree/main/vllm/model_executor/models>`_ directory.
29 +For instance, vLLM's `OPT model <https://github.com/WoosukKwon/vllm/blob/main/vllm/model_executor/models/opt.py>`_ was adpated from the HuggingFace's `modeling_opt.py <https://github.com/huggingface/transformers/blob/main/src/transformers/models/opt/modeling_opt.py>`_ file.
30
31 .. warning::
32 When copying the model code, make sure to review and adhere to the code's copyright and licensing terms.
@@ -62,11 +62,11 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
62 +) -> Dict[int, SequenceOutputs]:
63
64 3. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
65 -4. Replace the attention operation with either :code:`GPTCacheFlowAttention` or :code:`GPTNeoXCacheFlowAttention`, depending on the model's architecture.
65 +4. Replace the attention operation with either :code:`GPTPagedAttention` or :code:`GPTNeoXPagedAttention`, depending on the model's architecture.
66
67 .. note::
68 - Currently, CacheFlow supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
69 - If your model employs a different attention mechanism, you will need to implement a new attention layer in CacheFlow.
68 + Currently, vLLM supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
69 + If your model employs a different attention mechanism, you will need to implement a new attention layer in vLLM.
70
71
72 3. (Optional) Implement tensor parallelism support
@@ -91,4 +91,4 @@ While the process is straightforward for most layers, the tensor-parallel layers
91 5. Register your model
92 ----------------------
93
94 -Finally, include your :code:`*ForCausalLM` class in `cacheflow/model_executor/models/__init__.py <https://github.com/WoosukKwon/cacheflow/blob/main/cacheflow/model_executor/models/__init__.py>`_ and register it to the :code:`_MODEL_REGISTRY` in `cacheflow/model_executor/model_loader.py <https://github.com/WoosukKwon/cacheflow/blob/main/cacheflow/model_executor/model_loader.py>`_.
94 +Finally, include your :code:`*ForCausalLM` class in `vllm/model_executor/models/__init__.py <https://github.com/WoosukKwon/vllm/blob/main/vllm/model_executor/models/__init__.py>`_ and register it to the :code:`_MODEL_REGISTRY` in `vllm/model_executor/model_loader.py <https://github.com/WoosukKwon/vllm/blob/main/vllm/model_executor/model_loader.py>`_.
docs/source/models/supported_models.rst
+6 -6
@@ -3,8 +3,8 @@
3 Supported Models
4 ================
5
6 -CacheFlow supports a variety of generative Transformer models in `HuggingFace Transformers <https://github.com/huggingface/transformers>`_.
7 -The following is the list of model architectures that are currently supported by CacheFlow.
6 +vLLM supports a variety of generative Transformer models in `HuggingFace Transformers <https://github.com/huggingface/transformers>`_.
7 +The following is the list of model architectures that are currently supported by vLLM.
8 Alongside each architecture, we include some popular models that use it.
9
10 .. list-table::
@@ -22,19 +22,19 @@ Alongside each architecture, we include some popular models that use it.
22 * - :code:`OPTForCausalLM`
23 - OPT, OPT-IML
24
25 -If your model uses one of the above model architectures, you can seamlessly run your model with CacheFlow.
25 +If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
26 Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` for instructions on how to implement support for your model.
27 -Alternatively, you can raise an issue on our `GitHub <https://github.com/WoosukKwon/cacheflow/issues>`_ project.
27 +Alternatively, you can raise an issue on our `GitHub <https://github.com/WoosukKwon/vllm/issues>`_ project.
28
29 .. tip::
30 The easiest way to check if your model is supported is to run the program below:
31
32 .. code-block:: python
33
34 - from cacheflow import LLM
34 + from vllm import LLM
35
36 llm = LLM(model=...) # Name or path of your model
37 output = llm.generate("Hello, my name is")
38 print(output)
39
40 - If CacheFlow successfully generates text, it indicates that your model is supported.
40 + If vLLM successfully generates text, it indicates that your model is supported.
examples/api_client.py
+1 -1
@@ -1,4 +1,4 @@
1 -"""Example Python client for cacheflow.entrypoints.api_server"""
1 +"""Example Python client for vllm.entrypoints.api_server"""
2
3 import argparse
4 import json
examples/gradio_webserver.py
+2 -2
@@ -6,7 +6,7 @@ import requests
6
7
8 def http_bot(prompt):
9 - headers = {"User-Agent": "Cacheflow Client"}
9 + headers = {"User-Agent": "vLLM Client"}
10 pload = {
11 "prompt": prompt,
12 "stream": True,
@@ -24,7 +24,7 @@ def http_bot(prompt):
24 def build_demo():
25 with gr.Blocks() as demo:
26 gr.Markdown(
27 - "# Cacheflow text completion demo\n"
27 + "# vLLM text completion demo\n"
28 )
29 inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")
30 outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
examples/llm_engine_example.py
+1 -1
@@ -1,6 +1,6 @@
1 import argparse
2
3 -from cacheflow import EngineArgs, LLMEngine, SamplingParams
3 +from vllm import EngineArgs, LLMEngine, SamplingParams
4
5
6 def main(args: argparse.Namespace):
examples/offline_inference.py
+1 -1
@@ -1,4 +1,4 @@
1 -from cacheflow import LLM, SamplingParams
1 +from vllm import LLM, SamplingParams
2
3
4 # Sample prompts.
examples/openai_client.py
+1 -1
@@ -1,6 +1,6 @@
1 import openai
2
3 -# Modify OpenAI's API key and API base to use CacheFlow's API server.
3 +# Modify OpenAI's API key and API base to use vLLM's API server.
4 openai.api_key = "EMPTY"
5 openai.api_base = "http://localhost:8000/v1"
6 model = "facebook/opt-125m"
mypy.ini
+2 -2
@@ -3,6 +3,6 @@ python_version = 3.8
3
4 ignore_missing_imports = True
5
6 -files = cacheflow
6 +files = vllm
7 # TODO(woosuk): Include the code from Megatron and HuggingFace.
8 -exclude = cacheflow/model_executor/parallel_utils/|cacheflow/model_executor/models/
8 +exclude = vllm/model_executor/parallel_utils/|vllm/model_executor/models/
setup.py
+13 -13
@@ -75,7 +75,7 @@ ext_modules = []
75
76 # Cache operations.
77 cache_extension = CUDAExtension(
78 - name="cacheflow.cache_ops",
78 + name="vllm.cache_ops",
79 sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
80 extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
81 )
@@ -83,7 +83,7 @@ ext_modules.append(cache_extension)
83
84 # Attention kernels.
85 attention_extension = CUDAExtension(
86 - name="cacheflow.attention_ops",
86 + name="vllm.attention_ops",
87 sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
88 extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
89 )
@@ -91,7 +91,7 @@ ext_modules.append(attention_extension)
91
92 # Positional encoding kernels.
93 positional_encoding_extension = CUDAExtension(
94 - name="cacheflow.pos_encoding_ops",
94 + name="vllm.pos_encoding_ops",
95 sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
96 extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
97 )
@@ -99,7 +99,7 @@ ext_modules.append(positional_encoding_extension)
99
100 # Layer normalization kernels.
101 layernorm_extension = CUDAExtension(
102 - name="cacheflow.layernorm_ops",
102 + name="vllm.layernorm_ops",
103 sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
104 extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
105 )
@@ -107,7 +107,7 @@ ext_modules.append(layernorm_extension)
107
108 # Activation kernels.
109 activation_extension = CUDAExtension(
110 - name="cacheflow.activation_ops",
110 + name="vllm.activation_ops",
111 sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
112 extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
113 )
@@ -144,18 +144,18 @@ def get_requirements() -> List[str]:
144
145
146 setuptools.setup(
147 - name="cacheflow",
148 - version=find_version(get_path("cacheflow", "__init__.py")),
149 - author="CacheFlow Team",
150 - author_email="cacheflow@gmail.com",
147 + name="vllm",
148 + version=find_version(get_path("vllm", "__init__.py")),
149 + author="vLLM Team",
150 + author_email="vllm@gmail.com", # FIXME
151 license="Apache 2.0",
152 - description="CacheFlow: A high-performance LLM Serving System",
152 + description="vLLM: Easy, Fast, and Cheap LLM Serving with PagedAttention", # FIXME
153 long_description=read_readme(),
154 long_description_content_type="text/markdown",
155 - url="https://github.com/WoosukKwon/cacheflow",
155 + url="https://github.com/WoosukKwon/vllm",
156 project_urls={
157 - "Homepage": "https://github.com/WoosukKwon/cacheflow",
158 - "Documentation": "https://cacheflow.readthedocs.io/en/latest/",
157 + "Homepage": "https://github.com/WoosukKwon/vllm",
158 + "Documentation": "https://vllm.readthedocs.io/en/latest/", # FIXME
159 },
160 classifiers=[
161 "Programming Language :: Python :: 3.8",
tests/kernels/test_activation.py
+1 -1
@@ -1,7 +1,7 @@
1 import torch
2 import torch.nn.functional as F
3
4 -from cacheflow import activation_ops
4 +from vllm import activation_ops
5
6
7 def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
tests/kernels/test_attention.py
+1 -1
@@ -5,7 +5,7 @@ import torch
5 from xformers import ops as xops
6 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
7
8 -from cacheflow import attention_ops
8 +from vllm import attention_ops
9
10 MAX_SEQ_LEN = 4096
11 TEST_SEED = 0
tests/kernels/test_cache.py
+1 -1
@@ -2,7 +2,7 @@ import random
2
3 import torch
4
5 -from cacheflow import cache_ops
5 +from vllm import cache_ops
6
7
8 @torch.inference_mode()
tests/kernels/test_layernorm.py
+1 -1
@@ -1,7 +1,7 @@
1 import torch
2 import torch.nn as nn
3
4 -from cacheflow import layernorm_ops
4 +from vllm import layernorm_ops
5
6
7 class RefRMSNorm(nn.Module):
tests/kernels/test_pos_encoding.py
+1 -1
@@ -4,7 +4,7 @@ import torch
4 import torch.nn as nn
5 import torch.nn.functional as F
6
7 -from cacheflow import pos_encoding_ops
7 +from vllm import pos_encoding_ops
8
9
10 def rotate_half(x: torch.Tensor) -> torch.Tensor:
vllm/__init__.py new
+21
@@ -0,0 +1,21 @@
1 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
2 +from vllm.engine.async_llm_engine import AsyncLLMEngine
3 +from vllm.engine.llm_engine import LLMEngine
4 +from vllm.engine.ray_utils import initialize_cluster
5 +from vllm.entrypoints.llm import LLM
6 +from vllm.outputs import CompletionOutput, RequestOutput
7 +from vllm.sampling_params import SamplingParams
8 +
9 +__version__ = "0.1.0"
10 +
11 +__all__ = [
12 + "LLM",
13 + "SamplingParams",
14 + "RequestOutput",
15 + "CompletionOutput",
16 + "LLMEngine",
17 + "EngineArgs",
18 + "AsyncLLMEngine",
19 + "AsyncEngineArgs",
20 + "initialize_cluster",
21 +]
vllm/block.py renamed
+1 -1
@@ -1,7 +1,7 @@
1 """Token blocks."""
2 from typing import List
3
4 -from cacheflow.utils import Device
4 +from vllm.utils import Device
5
6 _BLANK_TOKEN_ID = -1
7
vllm/config.py renamed
+3 -3
@@ -3,8 +3,8 @@ from typing import Optional
3 import torch
4 from transformers import AutoConfig, PretrainedConfig
5
6 -from cacheflow.logger import init_logger
7 -from cacheflow.utils import get_cpu_memory
6 +from vllm.logger import init_logger
7 +from vllm.utils import get_cpu_memory
8
9 logger = init_logger(__name__)
10
@@ -87,7 +87,7 @@ class CacheConfig:
87 Args:
88 block_size: Size of a cache block in number of tokens.
89 gpu_memory_utilization: Fraction of GPU memory to use for the
90 - CacheFlow execution.
90 + vLLM execution.
91 swap_space: Size of the CPU swap space per GPU (in GiB).
92 """
93 def __init__(
vllm/core/__init__.py renamed
vllm/core/block_manager.py renamed
+3 -3
@@ -1,9 +1,9 @@
1 """A block manager that manages token blocks."""
2 from typing import Dict, List, Optional, Set, Tuple
3
4 -from cacheflow.block import PhysicalTokenBlock
5 -from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
6 -from cacheflow.utils import Device
4 +from vllm.block import PhysicalTokenBlock
5 +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
6 +from vllm.utils import Device
7
8
9 class BlockAllocator:
vllm/core/policy.py renamed
+1 -1
@@ -1,6 +1,6 @@
1 from typing import List
2
3 -from cacheflow.sequence import SequenceGroup
3 +from vllm.sequence import SequenceGroup
4
5
6 class Policy:
vllm/core/scheduler.py renamed
+7 -7
@@ -2,13 +2,13 @@ import enum
2 import time
3 from typing import Dict, List, Optional, Tuple
4
5 -from cacheflow.config import CacheConfig, SchedulerConfig
6 -from cacheflow.core.block_manager import BlockSpaceManager
7 -from cacheflow.core.policy import PolicyFactory
8 -from cacheflow.logger import init_logger
9 -from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
10 - SequenceGroupMetadata, SequenceOutputs,
11 - SequenceStatus)
5 +from vllm.config import CacheConfig, SchedulerConfig
6 +from vllm.core.block_manager import BlockSpaceManager
7 +from vllm.core.policy import PolicyFactory
8 +from vllm.logger import init_logger
9 +from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
10 + SequenceGroupMetadata, SequenceOutputs,
11 + SequenceStatus)
12
13 logger = init_logger(__name__)
14
vllm/engine/__init__.py renamed
vllm/engine/arg_utils.py renamed
+5 -5
@@ -3,13 +3,13 @@ import dataclasses
3 from dataclasses import dataclass
4 from typing import Optional, Tuple
5
6 -from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig)
6 +from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 + SchedulerConfig)
8
9
10 @dataclass
11 class EngineArgs:
12 - """Arguments for CacheFlow engine."""
12 + """Arguments for vLLM engine."""
13 model: str
14 download_dir: Optional[str] = None
15 use_np_weights: bool = False
@@ -33,7 +33,7 @@ class EngineArgs:
33 def add_cli_args(
34 parser: argparse.ArgumentParser,
35 ) -> argparse.ArgumentParser:
36 - """Shared CLI arguments for CacheFlow engine."""
36 + """Shared CLI arguments for vLLM engine."""
37 # Model arguments
38 parser.add_argument('--model', type=str, default='facebook/opt-125m',
39 help='name or path of the huggingface model to use')
@@ -118,7 +118,7 @@ class EngineArgs:
118
119 @dataclass
120 class AsyncEngineArgs(EngineArgs):
121 - """Arguments for asynchronous CacheFlow engine."""
121 + """Arguments for asynchronous vLLM engine."""
122 engine_use_ray: bool = False
123 disable_log_requests: bool = False
124
vllm/engine/async_llm_engine.py renamed
+9 -9
@@ -2,12 +2,12 @@ import asyncio
2 import time
3 from typing import Dict, List, Optional
4
5 -from cacheflow.engine.arg_utils import AsyncEngineArgs
6 -from cacheflow.engine.llm_engine import LLMEngine
7 -from cacheflow.engine.ray_utils import initialize_cluster, ray
8 -from cacheflow.logger import init_logger
9 -from cacheflow.outputs import RequestOutput
10 -from cacheflow.sampling_params import SamplingParams
5 +from vllm.engine.arg_utils import AsyncEngineArgs
6 +from vllm.engine.llm_engine import LLMEngine
7 +from vllm.engine.ray_utils import initialize_cluster, ray
8 +from vllm.logger import init_logger
9 +from vllm.outputs import RequestOutput
10 +from vllm.sampling_params import SamplingParams
11
12 logger = init_logger(__name__)
13
@@ -104,7 +104,7 @@ class AsyncLLMEngine:
104 arrival_time = time.time()
105
106 # Create an event to notify us that there is new output from the
107 - # cacheflow engine.
107 + # vLLM engine.
108 request_event = asyncio.Event()
109 self.request_events[request_id] = request_event
110
@@ -114,7 +114,7 @@ class AsyncLLMEngine:
114 f"sampling params: {sampling_params}, "
115 f"prompt token ids: {prompt_token_ids}.")
116
117 - # Add the request into the cacheflow engine's waiting queue.
117 + # Add the request into the vLLM engine's waiting queue.
118 if self.engine_use_ray:
119 await self.engine.add_request.remote(
120 request_id, prompt, sampling_params,
@@ -126,7 +126,7 @@ class AsyncLLMEngine:
126 prompt_token_ids=prompt_token_ids,
127 arrival_time=arrival_time)
128
129 - # The cacheflow engine does not have a background loop that keeps
129 + # The vLLM engine does not have a background loop that keeps
130 # processing incoming requests. Therefore, we need to keep kicking
131 # the engine to process the requests.
132 while True:
vllm/engine/llm_engine.py renamed
+13 -14
@@ -1,19 +1,18 @@
1 import time
2 from typing import Any, List, Optional
3
4 -from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5 - SchedulerConfig)
6 -from cacheflow.core.scheduler import Scheduler
7 -from cacheflow.engine.arg_utils import EngineArgs
8 -from cacheflow.engine.ray_utils import DeviceID, initialize_cluster, ray
9 -from cacheflow.engine.tokenizer_utils import (detokenize_incrementally,
10 - get_tokenizer)
11 -from cacheflow.logger import init_logger
12 -from cacheflow.outputs import RequestOutput
13 -from cacheflow.sampling_params import SamplingParams
14 -from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
15 -from cacheflow.utils import Counter
16 -from cacheflow.worker.worker import Worker
4 +from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
5 + SchedulerConfig)
6 +from vllm.core.scheduler import Scheduler
7 +from vllm.engine.arg_utils import EngineArgs
8 +from vllm.engine.ray_utils import DeviceID, initialize_cluster, ray
9 +from vllm.engine.tokenizer_utils import detokenize_incrementally, get_tokenizer
10 +from vllm.logger import init_logger
11 +from vllm.outputs import RequestOutput
12 +from vllm.sampling_params import SamplingParams
13 +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
14 +from vllm.utils import Counter
15 +from vllm.worker.worker import Worker
16
17 logger = init_logger(__name__)
18
@@ -21,7 +20,7 @@ logger = init_logger(__name__)
20 class LLMEngine:
21 """An LLM engine that receives requests and generates texts.
22
24 - This is the main class for the CacheFlow LLM engine. It receives requests
23 + This is the main class for the vLLM engine. It receives requests
24 from clients and generates texts from the LLM. It includes a tokenizer, a
25 language model (possibly distributed across multiple GPUs), and GPU memory
26 space allocated for intermediate states (aka KV cache). This class utilizes
vllm/engine/ray_utils.py renamed
+1 -1
@@ -6,7 +6,7 @@ try:
6 except ImportError:
7 ray = None
8
9 -from cacheflow.config import ParallelConfig
9 +from vllm.config import ParallelConfig
10
11 DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), device id
12
vllm/engine/tokenizer_utils.py renamed
+1 -1
@@ -3,7 +3,7 @@ from typing import List, Tuple, Union
3 from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
5
6 -from cacheflow.logger import init_logger
6 +from vllm.logger import init_logger
7
8 logger = init_logger(__name__)
9
vllm/entrypoints/__init__.py renamed
vllm/entrypoints/api_server.py renamed
+4 -4
@@ -6,10 +6,10 @@ from fastapi import BackgroundTasks, FastAPI, Request
6 from fastapi.responses import Response, StreamingResponse
7 import uvicorn
8
9 -from cacheflow.engine.arg_utils import AsyncEngineArgs
10 -from cacheflow.engine.async_llm_engine import AsyncLLMEngine
11 -from cacheflow.sampling_params import SamplingParams
12 -from cacheflow.utils import random_uuid
9 +from vllm.engine.arg_utils import AsyncEngineArgs
10 +from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.sampling_params import SamplingParams
12 +from vllm.utils import random_uuid
13
14 TIMEOUT_KEEP_ALIVE = 5 # seconds.
15 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
vllm/entrypoints/llm.py renamed
+5 -5
@@ -3,11 +3,11 @@ from typing import List, Optional, Union
3 from tqdm import tqdm
4 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6 -from cacheflow.engine.arg_utils import EngineArgs
7 -from cacheflow.engine.llm_engine import LLMEngine
8 -from cacheflow.outputs import RequestOutput
9 -from cacheflow.sampling_params import SamplingParams
10 -from cacheflow.utils import Counter
6 +from vllm.engine.arg_utils import EngineArgs
7 +from vllm.engine.llm_engine import LLMEngine
8 +from vllm.outputs import RequestOutput
9 +from vllm.sampling_params import SamplingParams
10 +from vllm.utils import Counter
11
12
13 class LLM:
vllm/entrypoints/openai/__init__.py renamed
vllm/entrypoints/openai/api_server.py renamed
+13 -13
@@ -13,17 +13,17 @@ from fastapi.middleware.cors import CORSMiddleware
13 from fastapi.responses import JSONResponse, StreamingResponse
14 import uvicorn
15
16 -from cacheflow.engine.arg_utils import AsyncEngineArgs
17 -from cacheflow.engine.async_llm_engine import AsyncLLMEngine
18 -from cacheflow.engine.tokenizer_utils import get_tokenizer
19 -from cacheflow.entrypoints.openai.protocol import (
16 +from vllm.engine.arg_utils import AsyncEngineArgs
17 +from vllm.engine.async_llm_engine import AsyncLLMEngine
18 +from vllm.engine.tokenizer_utils import get_tokenizer
19 +from vllm.entrypoints.openai.protocol import (
20 CompletionRequest, CompletionResponse, CompletionResponseChoice,
21 CompletionResponseStreamChoice, CompletionStreamResponse, ErrorResponse,
22 LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
23 -from cacheflow.logger import init_logger
24 -from cacheflow.outputs import RequestOutput
25 -from cacheflow.sampling_params import SamplingParams
26 -from cacheflow.utils import random_uuid
23 +from vllm.logger import init_logger
24 +from vllm.outputs import RequestOutput
25 +from vllm.sampling_params import SamplingParams
26 +from vllm.utils import random_uuid
27
28 TIMEOUT_KEEP_ALIVE = 5 # seconds
29
@@ -93,11 +93,11 @@ async def create_completion(raw_request: Request):
93 for the API specification. This API mimics the OpenAI Completion API.
94
95 NOTE: Currently we do not support the following features:
96 - - echo (since the cacheflow engine does not currently support
96 + - echo (since the vLLM engine does not currently support
97 getting the logprobs of prompt tokens)
98 - suffix (the language models we currently support do not support
99 suffix)
100 - - logit_bias (to be supported in cacheflow engine)
100 + - logit_bias (to be supported by vLLM engine)
101 """
102 request = CompletionRequest(**await raw_request.json())
103 logger.info(f"Received completion request: {request}")
@@ -107,7 +107,7 @@ async def create_completion(raw_request: Request):
107 return error_check_ret
108
109 if request.echo:
110 - # We do not support echo since the cacheflow engine does not
110 + # We do not support echo since the vLLM engine does not
111 # currently support getting the logprobs of prompt tokens.
112 return create_error_response(HTTPStatus.BAD_REQUEST,
113 "echo is not currently supported")
@@ -118,7 +118,7 @@ async def create_completion(raw_request: Request):
118 "suffix is not currently supported")
119
120 if request.logit_bias is not None:
121 - # TODO: support logit_bias in cacheflow engine.
121 + # TODO: support logit_bias in vLLM engine.
122 return create_error_response(HTTPStatus.BAD_REQUEST,
123 "logit_bias is not currently supported")
124
@@ -274,7 +274,7 @@ async def create_completion(raw_request: Request):
274
275 if __name__ == "__main__":
276 parser = argparse.ArgumentParser(
277 - description="CacheFlow OpenAI-Compatible RESTful API server."
277 + description="vLLM OpenAI-Compatible RESTful API server."
278 )
279 parser.add_argument("--host", type=str, default="localhost", help="host name")
280 parser.add_argument("--port", type=int, default=8000, help="port number")
vllm/entrypoints/openai/protocol.py renamed
+3 -3
@@ -4,7 +4,7 @@ from typing import Dict, List, Literal, Optional, Union
4
5 from pydantic import BaseModel, Field
6
7 -from cacheflow.utils import random_uuid
7 +from vllm.utils import random_uuid
8
9
10 class ErrorResponse(BaseModel):
@@ -34,7 +34,7 @@ class ModelCard(BaseModel):
34 id: str
35 object: str = "model"
36 created: int = Field(default_factory=lambda: int(time.time()))
37 - owned_by: str = "cacheflow"
37 + owned_by: str = "vllm"
38 root: Optional[str] = None
39 parent: Optional[str] = None
40 permission: List[ModelPermission] = Field(default_factory=list)
@@ -82,7 +82,7 @@ class CompletionRequest(BaseModel):
82 best_of: Optional[int] = None
83 logit_bias: Optional[Dict[str, float]] = None
84 user: Optional[str] = None
85 - # Additional parameters supported by cacheflow
85 + # Additional parameters supported by vLLM
86 top_k: Optional[int] = -1
87 ignore_eos: Optional[bool] = False
88 use_beam_search: Optional[bool] = False
vllm/logger.py renamed
+1 -1
@@ -22,7 +22,7 @@ class NewLineFormatter(logging.Formatter):
22 return msg
23
24
25 -_root_logger = logging.getLogger("cacheflow")
25 +_root_logger = logging.getLogger("vllm")
26 _default_handler = None
27
28
vllm/model_executor/__init__.py new
+10
@@ -0,0 +1,10 @@
1 +from vllm.model_executor.input_metadata import InputMetadata
2 +from vllm.model_executor.model_loader import get_model
3 +from vllm.model_executor.utils import set_random_seed
4 +
5 +
6 +__all__ = [
7 + "InputMetadata",
8 + "get_model",
9 + "set_random_seed",
10 +]
vllm/model_executor/input_metadata.py renamed
+2 -2
@@ -3,8 +3,8 @@ from typing import Dict, List, Tuple
3 import torch
4 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
5
6 -from cacheflow.sampling_params import SamplingParams
7 -from cacheflow.sequence import SequenceData
6 +from vllm.sampling_params import SamplingParams
7 +from vllm.sequence import SequenceData
8
9
10 class InputMetadata:
vllm/model_executor/layers/__init__.py renamed
vllm/model_executor/layers/activation.py renamed
+1 -1
@@ -2,7 +2,7 @@
2 import torch
3 import torch.nn as nn
4
5 -from cacheflow import activation_ops
5 +from vllm import activation_ops
6
7 _ACTIVATION_REGISTRY = {
8 "gelu": nn.GELU(),
vllm/model_executor/layers/attention.py renamed
+8 -8
@@ -5,16 +5,16 @@ import torch
5 import torch.nn as nn
6 from xformers import ops as xops
7
8 -from cacheflow import attention_ops
9 -from cacheflow import cache_ops
10 -from cacheflow import pos_encoding_ops
11 -from cacheflow.model_executor.input_metadata import InputMetadata
8 +from vllm import attention_ops
9 +from vllm import cache_ops
10 +from vllm import pos_encoding_ops
11 +from vllm.model_executor.input_metadata import InputMetadata
12
13 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
14
15
16 -class GPTCacheFlowAttention(nn.Module):
17 - """GPT-style multi-head attention.
16 +class PagedAttention(nn.Module):
17 + """GPT-style multi-head PagedAttention.
18
19 This class takes flattened 1D query, key, and value tensors as input. The
20 input 1D tensors can be split into three parts: the prompt tokens, the
@@ -164,8 +164,8 @@ class GPTCacheFlowAttention(nn.Module):
164 return output.view(-1, self.num_heads * self.head_size)
165
166
167 -class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
168 - """Attention with GPT-NeoX style rotary embedding."""
167 +class PagedAttentionWithRoPE(PagedAttention):
168 + """PagedAttention with GPT-NeoX style rotary embedding."""
169
170 def __init__(
171 self,
vllm/model_executor/layers/layernorm.py renamed
+1 -1
@@ -2,7 +2,7 @@
2 import torch
3 import torch.nn as nn
4
5 -from cacheflow import layernorm_ops
5 +from vllm import layernorm_ops
6
7
8 class RMSNorm(nn.Module):
vllm/model_executor/layers/sampler.py renamed
+4 -4
@@ -5,11 +5,11 @@ import numpy as np
5 import torch
6 import torch.nn as nn
7
8 -from cacheflow.model_executor.input_metadata import InputMetadata
9 -from cacheflow.model_executor.parallel_utils.tensor_parallel import (
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.parallel_utils.tensor_parallel import (
10 gather_from_tensor_model_parallel_region)
11 -from cacheflow.sampling_params import SamplingParams
12 -from cacheflow.sequence import SequenceOutputs
11 +from vllm.sampling_params import SamplingParams
12 +from vllm.sequence import SequenceOutputs
13
14
15 class Sampler(nn.Module):
vllm/model_executor/model_loader.py renamed
+4 -4
@@ -5,10 +5,10 @@ import torch
5 import torch.nn as nn
6 from transformers import PretrainedConfig
7
8 -from cacheflow.config import ModelConfig
9 -from cacheflow.model_executor.models import (
10 - GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
11 -from cacheflow.model_executor.weight_utils import initialize_dummy_weights
8 +from vllm.config import ModelConfig
9 +from vllm.model_executor.models import (GPT2LMHeadModel, GPTNeoXForCausalLM,
10 + LlamaForCausalLM, OPTForCausalLM)
11 +from vllm.model_executor.weight_utils import initialize_dummy_weights
12
13 # TODO(woosuk): Lazy-load the model classes.
14 _MODEL_REGISTRY = {
vllm/model_executor/models/__init__.py new
+12
@@ -0,0 +1,12 @@
1 +from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
2 +from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
3 +from vllm.model_executor.models.llama import LlamaForCausalLM
4 +from vllm.model_executor.models.opt import OPTForCausalLM
5 +
6 +
7 +__all__ = [
8 + "GPT2LMHeadModel",
9 + "GPTNeoXForCausalLM",
10 + "LlamaForCausalLM",
11 + "OPTForCausalLM",
12 +]
vllm/model_executor/models/gpt2.py renamed
+16 -16
@@ -1,6 +1,6 @@
1 # coding=utf-8
2 # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3 -# Copyright 2023 The CacheFlow team.
3 +# Copyright 2023 The vLLM team.
4 # Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
5 # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
6 #
@@ -26,17 +26,17 @@ import torch
26 from torch import nn
27 from transformers import GPT2Config
28
29 -from cacheflow.model_executor.input_metadata import InputMetadata
30 -from cacheflow.model_executor.layers.activation import get_act_fn
31 -from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
32 -from cacheflow.model_executor.layers.sampler import Sampler
33 -from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
34 - load_tensor_parallel_weights)
35 -from cacheflow.model_executor.parallel_utils.parallel_state import (
29 +from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.model_executor.layers.activation import get_act_fn
31 +from vllm.model_executor.layers.attention import PagedAttention
32 +from vllm.model_executor.layers.sampler import Sampler
33 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 + load_tensor_parallel_weights)
35 +from vllm.model_executor.parallel_utils.parallel_state import (
36 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 -from cacheflow.model_executor.parallel_utils.tensor_parallel import (
37 +from vllm.model_executor.parallel_utils.tensor_parallel import (
38 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39 -from cacheflow.sequence import SequenceOutputs
39 +from vllm.sequence import SequenceOutputs
40
41 KVCache = Tuple[torch.Tensor, torch.Tensor]
42
@@ -53,14 +53,14 @@ class GPT2Attention(nn.Module):
53 self.head_dim = self.hidden_size // total_num_heads
54 self.scale = self.head_dim ** -0.5
55
56 - self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size, bias=True,
57 - gather_output=False,
56 + self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
57 + bias=True, gather_output=False,
58 perform_initialization=False)
59 - self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size, bias=True,
60 - input_is_parallel=True,
59 + self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
60 + bias=True, input_is_parallel=True,
61 perform_initialization=False)
62 - self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
63 - scale=self.scale)
62 + self.attn = PagedAttention(self.num_heads, self.head_dim,
63 + scale=self.scale)
64
65 def forward(
66 self,
vllm/model_executor/models/gpt_neox.py renamed
+13 -12
@@ -1,6 +1,6 @@
1 # coding=utf-8
2 # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
3 -# Copyright 2023 The CacheFlow team.
3 +# Copyright 2023 The vLLM team.
4 # Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved.
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,17 +25,17 @@ import torch
25 from torch import nn
26 from transformers import GPTNeoXConfig
27
28 -from cacheflow.model_executor.input_metadata import InputMetadata
29 -from cacheflow.model_executor.layers.activation import get_act_fn
30 -from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
31 -from cacheflow.model_executor.layers.sampler import Sampler
32 -from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
33 - load_tensor_parallel_weights)
34 -from cacheflow.model_executor.parallel_utils.parallel_state import (
28 +from vllm.model_executor.input_metadata import InputMetadata
29 +from vllm.model_executor.layers.activation import get_act_fn
30 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
31 +from vllm.model_executor.layers.sampler import Sampler
32 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
33 + load_tensor_parallel_weights)
34 +from vllm.model_executor.parallel_utils.parallel_state import (
35 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
36 -from cacheflow.model_executor.parallel_utils.tensor_parallel import (
36 +from vllm.model_executor.parallel_utils.tensor_parallel import (
37 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38 -from cacheflow.sequence import SequenceOutputs
38 +from vllm.sequence import SequenceOutputs
39
40 KVCache = Tuple[torch.Tensor, torch.Tensor]
41
@@ -63,8 +63,8 @@ class GPTNeoXAttention(nn.Module):
63 scaling = self.head_size ** -0.5
64 rotary_dim = int(self.head_size * config.rotary_pct)
65 assert rotary_dim % 2 == 0
66 - self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_size,
67 - scaling, rotary_dim)
66 + self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
67 + scaling, rotary_dim)
68
69 def forward(
70 self,
@@ -149,6 +149,7 @@ class GPTNeoXLayer(nn.Module):
149
150
151 class GPTNeoXModel(nn.Module):
152 +
153 def __init__(self, config: GPTNeoXConfig):
154 super().__init__()
155 self.config = config
vllm/model_executor/models/llama.py renamed
+14 -14
@@ -1,6 +1,6 @@
1 # coding=utf-8
2 # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 -# Copyright 2023 The CacheFlow team.
3 +# Copyright 2023 The vLLM team.
4 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5 #
6 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
@@ -30,19 +30,19 @@ import torch
30 from torch import nn
31 from transformers import LlamaConfig
32
33 -from cacheflow.sequence import SequenceOutputs
34 -from cacheflow.model_executor.input_metadata import InputMetadata
35 -from cacheflow.model_executor.layers.activation import SiluAndMul
36 -from cacheflow.model_executor.layers.layernorm import RMSNorm
37 -from cacheflow.model_executor.layers.attention import GPTNeoXCacheFlowAttention
38 -from cacheflow.model_executor.layers.sampler import Sampler
39 -from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
40 - load_tensor_parallel_weights)
41 -from cacheflow.model_executor.parallel_utils.parallel_state import (
33 +from vllm.sequence import SequenceOutputs
34 +from vllm.model_executor.input_metadata import InputMetadata
35 +from vllm.model_executor.layers.activation import SiluAndMul
36 +from vllm.model_executor.layers.layernorm import RMSNorm
37 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38 +from vllm.model_executor.layers.sampler import Sampler
39 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
40 + load_tensor_parallel_weights)
41 +from vllm.model_executor.parallel_utils.parallel_state import (
42 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43 -from cacheflow.model_executor.parallel_utils.tensor_parallel import (
43 +from vllm.model_executor.parallel_utils.tensor_parallel import (
44 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
45 -from cacheflow.sequence import SequenceOutputs
45 +from vllm.sequence import SequenceOutputs
46
47 KVCache = Tuple[torch.Tensor, torch.Tensor]
48
@@ -104,8 +104,8 @@ class LlamaAttention(nn.Module):
104 input_is_parallel=True,
105 perform_initialization=False,
106 )
107 - self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_dim,
108 - self.scaling, rotary_dim=self.head_dim)
107 + self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_dim,
108 + self.scaling, rotary_dim=self.head_dim)
109
110 def forward(
111 self,
vllm/model_executor/models/opt.py renamed
+12 -12
@@ -1,6 +1,6 @@
1 # coding=utf-8
2 # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
3 -# Copyright 2023 The CacheFlow team.
3 +# Copyright 2023 The vLLM team.
4 # Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,17 +25,17 @@ import torch
25 from torch import nn
26 from transformers import OPTConfig
27
28 -from cacheflow.model_executor.input_metadata import InputMetadata
29 -from cacheflow.model_executor.layers.activation import get_act_fn
30 -from cacheflow.model_executor.layers.attention import GPTCacheFlowAttention
31 -from cacheflow.model_executor.layers.sampler import Sampler
32 -from cacheflow.model_executor.weight_utils import (hf_model_weights_iterator,
33 - load_tensor_parallel_weights)
34 -from cacheflow.model_executor.parallel_utils.parallel_state import (
28 +from vllm.model_executor.input_metadata import InputMetadata
29 +from vllm.model_executor.layers.activation import get_act_fn
30 +from vllm.model_executor.layers.attention import PagedAttention
31 +from vllm.model_executor.layers.sampler import Sampler
32 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
33 + load_tensor_parallel_weights)
34 +from vllm.model_executor.parallel_utils.parallel_state import (
35 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
36 -from cacheflow.model_executor.parallel_utils.tensor_parallel import (
36 +from vllm.model_executor.parallel_utils.tensor_parallel import (
37 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38 -from cacheflow.sequence import SequenceOutputs
38 +from vllm.sequence import SequenceOutputs
39
40 KVCache = Tuple[torch.Tensor, torch.Tensor]
41
@@ -75,8 +75,8 @@ class OPTAttention(nn.Module):
75 self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
76 input_is_parallel=True,
77 perform_initialization=False)
78 - self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
79 - scale=self.scaling)
78 + self.attn = PagedAttention(self.num_heads, self.head_dim,
79 + scale=self.scaling)
80
81 def forward(
82 self,
vllm/model_executor/parallel_utils/README.md renamed
vllm/model_executor/parallel_utils/__init__.py renamed
+2 -2
@@ -1,5 +1,5 @@
1 -import cacheflow.model_executor.parallel_utils.parallel_state
2 -import cacheflow.model_executor.parallel_utils.tensor_parallel
1 +import vllm.model_executor.parallel_utils.parallel_state
2 +import vllm.model_executor.parallel_utils.tensor_parallel
3
4 # Alias parallel_state as mpu, its legacy name
5 mpu = parallel_state
vllm/model_executor/parallel_utils/parallel_state.py renamed
+1 -1
@@ -1,4 +1,4 @@
1 -# Copyright 2023 The CacheFlow team.
1 +# Copyright 2023 The vLLM team.
2 # Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
vllm/model_executor/parallel_utils/tensor_parallel/__init__.py renamed
vllm/model_executor/parallel_utils/tensor_parallel/layers.py renamed
+2 -2
@@ -1,4 +1,4 @@
1 -# Copyright 2023 The CacheFlow team.
1 +# Copyright 2023 The vLLM team.
2 # Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
@@ -11,7 +11,7 @@ import torch.nn.functional as F
11 import torch.nn.init as init
12 from torch.nn.parameter import Parameter
13
14 -from cacheflow.model_executor.parallel_utils.parallel_state import (
14 +from vllm.model_executor.parallel_utils.parallel_state import (
15 get_tensor_model_parallel_rank,
16 get_tensor_model_parallel_world_size,
17 get_all_reduce_launcher,
vllm/model_executor/parallel_utils/tensor_parallel/mappings.py renamed
+2 -2
@@ -1,10 +1,10 @@
1 -# Copyright 2023 The CacheFlow team.
1 +# Copyright 2023 The vLLM team.
2 # Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/mappings.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5 import torch
6
7 -from cacheflow.model_executor.parallel_utils.parallel_state import (
7 +from vllm.model_executor.parallel_utils.parallel_state import (
8 get_tensor_model_parallel_rank,
9 get_tensor_model_parallel_world_size,
10 get_tensor_model_parallel_group,
vllm/model_executor/parallel_utils/tensor_parallel/random.py renamed
+2 -2
@@ -1,4 +1,4 @@
1 -# Copyright 2023 The CacheFlow team.
1 +# Copyright 2023 The vLLM team.
2 # Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/random.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
@@ -11,7 +11,7 @@ import torch
11 from torch import _C
12 from torch.cuda import _lazy_call, device as device_ctx_manager
13
14 -from cacheflow.model_executor.parallel_utils.parallel_state import (
14 +from vllm.model_executor.parallel_utils.parallel_state import (
15 get_tensor_model_parallel_rank,
16 )
17
vllm/model_executor/parallel_utils/tensor_parallel/utils.py renamed
+1 -1
@@ -1,4 +1,4 @@
1 -# Copyright 2023 The CacheFlow team.
1 +# Copyright 2023 The vLLM team.
2 # Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
vllm/model_executor/utils.py renamed
+2 -2
@@ -4,8 +4,8 @@ import random
4 import numpy as np
5 import torch
6
7 -from cacheflow.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8 -from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
7 +from vllm.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8 +from vllm.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
10
11 def set_random_seed(seed: int) -> None:
vllm/model_executor/weight_utils.py renamed
vllm/outputs.py renamed
+1 -1
@@ -1,6 +1,6 @@
1 from typing import Dict, List, Optional
2
3 -from cacheflow.sequence import SequenceGroup, SequenceStatus
3 +from vllm.sequence import SequenceGroup, SequenceStatus
4
5
6 class CompletionOutput:
vllm/sampling_params.py renamed
vllm/sequence.py renamed
+2 -2
@@ -2,8 +2,8 @@ import copy
2 import enum
3 from typing import Dict, List, Optional, Union
4
5 -from cacheflow.block import LogicalTokenBlock
6 -from cacheflow.sampling_params import SamplingParams
5 +from vllm.block import LogicalTokenBlock
6 +from vllm.sampling_params import SamplingParams
7
8
9 class SequenceStatus(enum.Enum):
vllm/utils.py renamed
vllm/worker/__init__.py renamed
vllm/worker/cache_engine.py renamed
+2 -2
@@ -3,8 +3,8 @@ from typing import Dict, List, Tuple
3
4 import torch
5
6 -from cacheflow import cache_ops
7 -from cacheflow.config import CacheConfig, ModelConfig, ParallelConfig
6 +from vllm import cache_ops
7 +from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
9 KVCache = Tuple[torch.Tensor, torch.Tensor]
10
vllm/worker/worker.py renamed
+8 -9
@@ -3,16 +3,15 @@ from typing import Dict, List, Tuple
3
4 import torch
5
6 -from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig)
8 -from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
9 -from cacheflow.model_executor.parallel_utils.parallel_state import (
6 +from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 + SchedulerConfig)
8 +from vllm.model_executor import get_model, InputMetadata, set_random_seed
9 +from vllm.model_executor.parallel_utils.parallel_state import (
10 initialize_model_parallel, initialize_all_reduce_launcher)
11 -from cacheflow.sampling_params import SamplingParams
12 -from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
13 - SequenceOutputs)
14 -from cacheflow.worker.cache_engine import CacheEngine
15 -from cacheflow.utils import get_gpu_memory
11 +from vllm.sampling_params import SamplingParams
12 +from vllm.sequence import SequenceData, SequenceGroupMetadata, SequenceOutputs
13 +from vllm.worker.cache_engine import CacheEngine
14 +from vllm.utils import get_gpu_memory
15
16
17 class Worker: