| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """vLLM: a high-throughput and memory-efficient inference engine for LLMs.""" |
| 4 | |
| 5 | # The version.py should be independent library, and we always import the |
| 6 | # version library first. Such assumption is critical for some customization. |
| 7 | from .version import __version__, __version_tuple__ # isort:skip |
| 8 | |
| 9 | import typing |
| 10 | |
| 11 | # The environment variables override should be imported before any other |
| 12 | # modules to ensure that the environment variables are set before any |
| 13 | # other modules are imported. |
| 14 | import vllm.env_override # noqa: F401 |
| 15 | |
| 16 | MODULE_ATTRS = { |
| 17 | "AsyncEngineArgs": ".engine.arg_utils:AsyncEngineArgs", |
| 18 | "EngineArgs": ".engine.arg_utils:EngineArgs", |
| 19 | "AsyncLLMEngine": ".engine.async_llm_engine:AsyncLLMEngine", |
| 20 | "LLMEngine": ".engine.llm_engine:LLMEngine", |
| 21 | "LLM": ".entrypoints.llm:LLM", |
| 22 | "initialize_ray_cluster": ".v1.executor.ray_utils:initialize_ray_cluster", |
| 23 | "PromptType": ".inputs:PromptType", |
| 24 | "TextPrompt": ".inputs:TextPrompt", |
| 25 | "TokensPrompt": ".inputs:TokensPrompt", |
| 26 | "ModelRegistry": ".model_executor.models:ModelRegistry", |
| 27 | "SamplingParams": ".sampling_params:SamplingParams", |
| 28 | "PoolingParams": ".pooling_params:PoolingParams", |
| 29 | "ClassificationOutput": ".outputs:ClassificationOutput", |
| 30 | "ClassificationRequestOutput": ".outputs:ClassificationRequestOutput", |
| 31 | "CompletionOutput": ".outputs:CompletionOutput", |
| 32 | "EmbeddingOutput": ".outputs:EmbeddingOutput", |
| 33 | "EmbeddingRequestOutput": ".outputs:EmbeddingRequestOutput", |
| 34 | "PoolingOutput": ".outputs:PoolingOutput", |
| 35 | "PoolingRequestOutput": ".outputs:PoolingRequestOutput", |
| 36 | "RequestOutput": ".outputs:RequestOutput", |
| 37 | "ScoringOutput": ".outputs:ScoringOutput", |
| 38 | "ScoringRequestOutput": ".outputs:ScoringRequestOutput", |
| 39 | } |
| 40 | |
| 41 | if typing.TYPE_CHECKING: |
| 42 | from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs |
| 43 | from vllm.engine.async_llm_engine import AsyncLLMEngine |
| 44 | from vllm.engine.llm_engine import LLMEngine |
| 45 | from vllm.entrypoints.llm import LLM |
| 46 | from vllm.inputs import PromptType, TextPrompt, TokensPrompt |
| 47 | from vllm.model_executor.models import ModelRegistry |
| 48 | from vllm.outputs import ( |
| 49 | ClassificationOutput, |
| 50 | ClassificationRequestOutput, |
| 51 | CompletionOutput, |
| 52 | EmbeddingOutput, |
| 53 | EmbeddingRequestOutput, |
| 54 | PoolingOutput, |
| 55 | PoolingRequestOutput, |
| 56 | RequestOutput, |
| 57 | ScoringOutput, |
| 58 | ScoringRequestOutput, |
| 59 | ) |
| 60 | from vllm.pooling_params import PoolingParams |
| 61 | from vllm.sampling_params import SamplingParams |
| 62 | from vllm.v1.executor.ray_utils import initialize_ray_cluster |
| 63 | else: |
| 64 | |
| 65 | def __getattr__(name: str) -> typing.Any: |
| 66 | from importlib import import_module |
| 67 | |
| 68 | if name in MODULE_ATTRS: |
| 69 | module_name, attr_name = MODULE_ATTRS[name].split(":") |
| 70 | module = import_module(module_name, __package__) |
| 71 | return getattr(module, attr_name) |
| 72 | else: |
| 73 | raise AttributeError(f"module {__package__} has no attribute {name}") |
| 74 | |
| 75 | |
| 76 | __all__ = [ |
| 77 | "__version__", |
| 78 | "__version_tuple__", |
| 79 | "LLM", |
| 80 | "ModelRegistry", |
| 81 | "PromptType", |
| 82 | "TextPrompt", |
| 83 | "TokensPrompt", |
| 84 | "SamplingParams", |
| 85 | "RequestOutput", |
| 86 | "CompletionOutput", |
| 87 | "PoolingOutput", |
| 88 | "PoolingRequestOutput", |
| 89 | "EmbeddingOutput", |
| 90 | "EmbeddingRequestOutput", |
| 91 | "ClassificationOutput", |
| 92 | "ClassificationRequestOutput", |
| 93 | "ScoringOutput", |
| 94 | "ScoringRequestOutput", |
| 95 | "LLMEngine", |
| 96 | "EngineArgs", |
| 97 | "AsyncLLMEngine", |
| 98 | "AsyncEngineArgs", |
| 99 | "initialize_ray_cluster", |
| 100 | "PoolingParams", |
| 101 | ] |