@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ebce310b74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Model] Snowflake arctic model implementation (#4652)

Co-authored-by: Dash Desai <1723932+iamontheinet@users.noreply.github.com> Co-authored-by: Aurick Qiao <qiao@aurick.net> Co-authored-by: Aurick Qiao <aurick.qiao@snowflake.com> Co-authored-by: Aurick Qiao <aurickq@users.noreply.github.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Hao Zhang committed May 9, 2024 at 15:37 UTC ebce310b7433e050086f52ca48571807df467f50
8 files changed +1042 -48
examples/offline_inference_arctic.py new
+26
@@ -0,0 +1,26 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +# Sample prompts.
4 +prompts = [
5 + "Hello, my name is",
6 + "The president of the United States is",
7 + "The capital of France is",
8 + "The future of AI is",
9 +]
10 +# Create a sampling params object.
11 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12 +
13 +# Create an LLM.
14 +llm = LLM(model="snowflake/snowflake-arctic-instruct",
15 + quantization="deepspeedfp",
16 + tensor_parallel_size=8,
17 + trust_remote_code=True)
18 +# Generate texts from the prompts. The output is a list of RequestOutput objects
19 +# that contain the prompt, generated text, and other information.
20 +
21 +outputs = llm.generate(prompts, sampling_params)
22 +# Print the outputs.
23 +for output in outputs:
24 + prompt = output.prompt
25 + generated_text = output.outputs[0].text
26 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/model_executor/layers/fused_moe/__init__.py
+3 -1
@@ -1,7 +1,9 @@
1 from vllm.model_executor.layers.fused_moe.fused_moe import (
2 - fused_moe, get_config_file_name)
2 + fused_experts, fused_moe, fused_topk, get_config_file_name)
3
4 __all__ = [
5 "fused_moe",
6 + "fused_topk",
7 + "fused_experts",
8 "get_config_file_name",
9 ]
vllm/model_executor/layers/fused_moe/fused_moe.py
+90 -47
@@ -308,60 +308,16 @@ def get_moe_configs(E: int, N: int,
308 return None
309
310
311 -def fused_moe(
311 +def fused_topk(
312 hidden_states: torch.Tensor,
313 - w1: torch.Tensor,
314 - w2: torch.Tensor,
313 gating_output: torch.Tensor,
314 topk: int,
315 renormalize: bool,
318 - inplace: bool = False,
319 - override_config: Optional[Dict[str, Any]] = None,
320 - use_fp8: bool = False,
321 - w1_scale: Optional[torch.Tensor] = None,
322 - w2_scale: Optional[torch.Tensor] = None,
323 - a1_scale: Optional[torch.Tensor] = None,
324 - a2_scale: Optional[torch.Tensor] = None,
325 -) -> torch.Tensor:
326 - """
327 - This function computes a Mixture of Experts (MoE) layer using two sets of
328 - weights, w1 and w2, and top-k gating mechanism.
329 -
330 - Parameters:
331 - - hidden_states (torch.Tensor): The input tensor to the MoE layer.
332 - - w1 (torch.Tensor): The first set of expert weights.
333 - - w2 (torch.Tensor): The second set of expert weights.
334 - - gating_output (torch.Tensor): The output of the gating operation
335 - (before softmax).
336 - - topk (int): The number of top-k experts to select.
337 - - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
338 - - inplace (bool): If True, perform the operation in-place.
339 - Defaults to False.
340 - - override_config (Optional[Dict[str, Any]]): Optional override
341 - for the kernel configuration.
342 - - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
343 - products for w1 and w2. Defaults to False.
344 - - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
345 - w1.
346 - - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
347 - w2.
348 -
349 - Returns:
350 - - torch.Tensor: The output tensor after applying the MoE layer.
351 - """
352 - # Check constraints.
316 +):
317 assert hidden_states.shape[0] == gating_output.shape[0], (
318 "Number of tokens mismatch")
355 - assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
356 - assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
357 - assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
358 - assert w1.is_contiguous(), "Expert weights1 must be contiguous"
359 - assert w2.is_contiguous(), "Expert weights2 must be contiguous"
360 - assert hidden_states.dtype in [
361 - torch.float32, torch.float16, torch.bfloat16
362 - ]
319 +
320 M, _ = hidden_states.shape
364 - E, N, _ = w1.shape
321
322 if is_hip():
323 # The MoE kernels are not yet supported on ROCm.
@@ -393,6 +349,33 @@ def fused_moe(
349 del token_expert_indicies # Not used. Will be used in the future.
350 if renormalize:
351 topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
352 + return topk_weights, topk_ids
353 +
354 +
355 +def fused_experts(hidden_states: torch.Tensor,
356 + w1: torch.Tensor,
357 + w2: torch.Tensor,
358 + topk_weights: torch.Tensor,
359 + topk_ids: torch.Tensor,
360 + inplace: bool = False,
361 + override_config: Optional[Dict[str, Any]] = None,
362 + use_fp8: bool = False,
363 + w1_scale: Optional[torch.Tensor] = None,
364 + w2_scale: Optional[torch.Tensor] = None,
365 + a1_scale: Optional[torch.Tensor] = None,
366 + a2_scale: Optional[torch.Tensor] = None):
367 + # Check constraints.
368 + assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
369 + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
370 + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
371 + assert w1.is_contiguous(), "Expert weights1 must be contiguous"
372 + assert w2.is_contiguous(), "Expert weights2 must be contiguous"
373 + assert hidden_states.dtype in [
374 + torch.float32, torch.float16, torch.bfloat16
375 + ]
376 +
377 + M, _ = hidden_states.shape
378 + E, N, _ = w1.shape
379
380 if override_config:
381 config = override_config
@@ -477,3 +460,63 @@ def fused_moe(
460 out=hidden_states)
461 return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
462 dim=1)
463 +
464 +
465 +def fused_moe(
466 + hidden_states: torch.Tensor,
467 + w1: torch.Tensor,
468 + w2: torch.Tensor,
469 + gating_output: torch.Tensor,
470 + topk: int,
471 + renormalize: bool,
472 + inplace: bool = False,
473 + override_config: Optional[Dict[str, Any]] = None,
474 + use_fp8: bool = False,
475 + w1_scale: Optional[torch.Tensor] = None,
476 + w2_scale: Optional[torch.Tensor] = None,
477 + a1_scale: Optional[torch.Tensor] = None,
478 + a2_scale: Optional[torch.Tensor] = None,
479 +) -> torch.Tensor:
480 + """
481 + This function computes a Mixture of Experts (MoE) layer using two sets of
482 + weights, w1 and w2, and top-k gating mechanism.
483 +
484 + Parameters:
485 + - hidden_states (torch.Tensor): The input tensor to the MoE layer.
486 + - w1 (torch.Tensor): The first set of expert weights.
487 + - w2 (torch.Tensor): The second set of expert weights.
488 + - gating_output (torch.Tensor): The output of the gating operation
489 + (before softmax).
490 + - topk (int): The number of top-k experts to select.
491 + - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
492 + - inplace (bool): If True, perform the operation in-place.
493 + Defaults to False.
494 + - override_config (Optional[Dict[str, Any]]): Optional override
495 + for the kernel configuration.
496 + - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
497 + products for w1 and w2. Defaults to False.
498 + - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
499 + w1.
500 + - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
501 + w2.
502 +
503 + Returns:
504 + - torch.Tensor: The output tensor after applying the MoE layer.
505 + """
506 + # Check constraints.
507 + assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
508 +
509 + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
510 + renormalize)
511 + return fused_experts(hidden_states,
512 + w1,
513 + w2,
514 + topk_weights,
515 + topk_ids,
516 + inplace=inplace,
517 + override_config=override_config,
518 + use_fp8=use_fp8,
519 + w1_scale=w1_scale,
520 + w2_scale=w2_scale,
521 + a1_scale=a1_scale,
522 + a2_scale=a2_scale)
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -4,6 +4,8 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 from vllm.model_executor.layers.quantization.base_config import (
6 QuantizationConfig)
7 +from vllm.model_executor.layers.quantization.deepspeedfp import (
8 + DeepSpeedFPConfig)
9 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
10 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
11 from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -19,6 +21,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
21 "squeezellm": SqueezeLLMConfig,
22 "gptq_marlin": GPTQMarlinConfig,
23 "marlin": MarlinConfig,
24 + "deepspeedfp": DeepSpeedFPConfig
25 }
26
27
vllm/model_executor/layers/quantization/deepspeedfp.py new
+194
@@ -0,0 +1,194 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +import torch.nn as nn
5 +import torch.nn.functional as F
6 +
7 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 +from vllm.model_executor.layers.quantization.base_config import (
9 + QuantizationConfig)
10 +from vllm.model_executor.utils import set_weight_attrs
11 +
12 +
13 +class DeepSpeedFPConfig(QuantizationConfig):
14 + """Config for DeepSpeed FP quantizer. It supports fp6 and fp8.
15 +
16 + Args:
17 + weight_bits: the target quantization bits, 6 or 8.
18 + group_size: group size for quantizaiton, default to 128.
19 + """
20 +
21 + def __init__(
22 + self,
23 + weight_bits: int = 8,
24 + group_size: int = 512,
25 + ) -> None:
26 + self.weight_bits = weight_bits
27 + self.group_size = group_size
28 + self.valid_types = [torch.bfloat16, torch.float16]
29 +
30 + if self.weight_bits not in (6, 8):
31 + raise ValueError(
32 + "Currently, only 6-bit or 8-bit weight quantization are "
33 + f"supported for DeepSpeed FP quantizaiton, but got "
34 + f"{self.weight_bits} bits.")
35 +
36 + def __repr__(self) -> str:
37 + return (f"DeepSpeedFPConfig(weight_bits={self.weight_bits}), "
38 + f"group_size={self.group_size}")
39 +
40 + @classmethod
41 + def get_name(cls) -> str:
42 + return "DeepSpeedFP"
43 +
44 + @classmethod
45 + def from_config(cls, config: Dict[str, Any]) -> "DeepSpeedFPConfig":
46 + weight_bits = cls.get_from_keys(config, ["bits"])
47 + group_size = cls.get_from_keys(config, ["group_size"])
48 + return cls(weight_bits=weight_bits, group_size=group_size)
49 +
50 + def get_linear_method(self) -> "DeepSpeedFPLinearMethod":
51 + return DeepSpeedFPLinearMethod(self)
52 +
53 + def get_scaled_act_names(self) -> List[str]:
54 + return []
55 +
56 + @classmethod
57 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
58 + return [torch.half, torch.bfloat16]
59 +
60 + @classmethod
61 + # Need to figure it out
62 + def get_min_capability(cls) -> int:
63 + return 60
64 +
65 + @staticmethod
66 + def get_config_filenames() -> List[str]:
67 + return [
68 + "quant_config.json",
69 + "quantize_config.json",
70 + ]
71 +
72 + def get_quant_method(
73 + self,
74 + layer: torch.nn.Module) -> Optional["DeepSpeedFPLinearMethod"]:
75 + if isinstance(layer, LinearBase):
76 + return DeepSpeedFPLinearMethod(self)
77 + return None
78 +
79 +
80 +class DeepSpeedFPLinearMethod(LinearMethodBase):
81 + """Linear method for DeepSpeedFP quantizer.
82 +
83 + Args:
84 + quant_config: the DeepSpeedFP quantization config.
85 + """
86 +
87 + def __init__(self, quant_config: DeepSpeedFPConfig):
88 + self.quant_config = quant_config
89 + self.weight = None
90 +
91 + def create_weights(self,
92 + layer: torch.nn.Module,
93 + input_size_per_partition: int,
94 + output_partition_sizes: List[int],
95 + input_size: int,
96 + output_size: int,
97 + params_dtype: torch.dtype,
98 + weight_loader=None,
99 + **extra_weight_attrs):
100 + del output_size
101 + del input_size
102 + output_size_per_partition = sum(output_partition_sizes)
103 + weight = DeepSpeedFPParameter(
104 + torch.Size((output_size_per_partition, input_size_per_partition)),
105 + params_dtype=params_dtype,
106 + quant_config=self.quant_config,
107 + )
108 + set_weight_attrs(weight, {
109 + "input_dim": 1,
110 + "output_dim": 0,
111 + })
112 + layer.register_parameter("weight", weight)
113 +
114 + def quant_weight_loader(param, loaded_weight, *args, **kwargs):
115 + # Calls the original weight loader (if any), quantizes the result,
116 + # and then loads the quantized parameter.
117 + if weight_loader is not None:
118 + orig_param_data = param.data
119 + param.data = param.ds_dequantize()
120 + weight_loader(param, loaded_weight, *args, **kwargs)
121 + param.data, loaded_weight = orig_param_data, param.data
122 + param.ds_quantize_(loaded_weight.cuda())
123 +
124 + extra_weight_attrs["weight_loader"] = quant_weight_loader
125 + set_weight_attrs(weight, extra_weight_attrs)
126 +
127 + def apply(self,
128 + layer: torch.nn.Module,
129 + x: torch.Tensor,
130 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
131 + weight = layer.weight
132 + y = weight.ds_dequantize()
133 + return F.linear(x, y, bias)
134 +
135 +
136 +class DeepSpeedFPParameter(nn.Parameter):
137 + """
138 + DeepSpeedFP quantized parameter class that implements fp8/fp6
139 + quantization deepspeed. Weights are stored in quantized form on
140 + GPUs, and can be dequantized on-the-fly when needed by the model.
141 + """
142 +
143 + def __new__(cls, orig_shape: torch.Size, params_dtype: torch.dtype,
144 + quant_config: DeepSpeedFPConfig):
145 + try:
146 + import deepspeed
147 + if deepspeed.__version__ < "0.14.2":
148 + raise ImportError("deepspeed version is wrong. Please "
149 + "install deepspeed>=0.14.2.")
150 + from deepspeed.ops.fp_quantizer import FP_Quantize
151 + except ImportError as err:
152 + raise ImportError("Please install deepspeed>=0.14.2 via "
153 + "`pip install deepspeed>=0.14.2` to use "
154 + "deepspeedfp quantizer.") from err
155 + data = torch.empty((
156 + orig_shape.numel() // quant_config.group_size,
157 + quant_config.group_size * quant_config.weight_bits // 8 + 4,
158 + ),
159 + dtype=torch.int8)
160 + self = torch.Tensor._make_subclass(cls, data, data.requires_grad)
161 + self.orig_shape = orig_shape
162 + self.quant_config = quant_config
163 + self.fp_quantizer = FP_Quantize(group_size=quant_config.group_size)
164 + self.fp_quantizer.orig_shape = orig_shape
165 + self.fp_quantizer.orig_dtype = params_dtype
166 + return self
167 +
168 + def ds_quantize_(self, tensor: torch.Tensor):
169 + assert tensor.device.type == "cuda" and tensor.dtype != torch.int8
170 + return self.data.copy_(
171 + self.fp_quantizer.quantize(
172 + tensor.data,
173 + q_bits=self.quant_config.weight_bits,
174 + ))
175 +
176 + def ds_dequantize(self, fp_out=None) -> torch.Tensor:
177 + """
178 + Return a tensor containing the dequantized weights of this parameter.
179 + """
180 + assert self.data.device.type == "cuda" and self.data.dtype == torch.int8
181 + return self.fp_quantizer.dequantize(
182 + self.data, fp_out=fp_out, q_bits=self.quant_config.weight_bits)
183 +
184 + def ds_selective_dequantize(self, indices, fp_out=None) -> torch.Tensor:
185 + """
186 + Return a tensor where only the weights at `indices` are dequantized
187 + (to save HBM -> SRAM bandwidth).
188 + """
189 + assert self.data.device.type == "cuda" and self.data.dtype == torch.int8
190 + return self.fp_quantizer.selective_dequantize(
191 + self.data,
192 + indices,
193 + fp_out=fp_out,
194 + q_bits=self.quant_config.weight_bits)
vllm/model_executor/models/__init__.py
+1
@@ -54,6 +54,7 @@ _MODELS = {
54 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
55 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
56 "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
57 + "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
58 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
59 }
60
vllm/model_executor/models/arctic.py new
+521
@@ -0,0 +1,521 @@
1 +"""Inference-only Snowflake Arctic model."""
2 +from typing import Iterable, List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +
7 +from vllm.attention import Attention, AttentionMetadata
8 +from vllm.distributed import (get_tensor_model_parallel_rank,
9 + get_tensor_model_parallel_world_size,
10 + tensor_model_parallel_all_reduce)
11 +from vllm.logger import init_logger
12 +from vllm.model_executor.layers.activation import SiluAndMul
13 +from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
14 +from vllm.model_executor.layers.layernorm import RMSNorm
15 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
16 + QKVParallelLinear,
17 + ReplicatedLinear,
18 + RowParallelLinear)
19 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 +from vllm.model_executor.layers.quantization.base_config import (
21 + QuantizationConfig)
22 +from vllm.model_executor.layers.quantization.deepspeedfp import (
23 + DeepSpeedFPConfig, DeepSpeedFPParameter)
24 +from vllm.model_executor.layers.rotary_embedding import get_rope
25 +from vllm.model_executor.layers.sampler import Sampler
26 +from vllm.model_executor.layers.vocab_parallel_embedding import (
27 + ParallelLMHead, VocabParallelEmbedding)
28 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 +from vllm.model_executor.sampling_metadata import SamplingMetadata
30 +from vllm.model_executor.utils import set_weight_attrs
31 +from vllm.sequence import SamplerOutput
32 +from vllm.transformers_utils.configs.arctic import ArcticConfig
33 +
34 +logger = init_logger(__name__)
35 +
36 +
37 +class ArcticMLP(nn.Module):
38 +
39 + def __init__(self,
40 + config: ArcticConfig,
41 + layer_id: int,
42 + expert_id: int = -1,
43 + is_residual_mlp: bool = False,
44 + quant_config: Optional[QuantizationConfig] = None,
45 + reduce_results: bool = True):
46 + super(ArcticMLP, self).__init__()
47 + self.hidden_size = config.hidden_size
48 + self.expert_id = expert_id
49 + self.layer_id = layer_id
50 +
51 + self.ffn_dim = config.intermediate_size if not is_residual_mlp \
52 + else self.hidden_size
53 +
54 + self.w13 = MergedColumnParallelLinear(self.hidden_size,
55 + [self.ffn_dim] * 2,
56 + bias=False,
57 + quant_config=quant_config)
58 + self.w2 = RowParallelLinear(self.ffn_dim,
59 + self.hidden_size,
60 + bias=False,
61 + reduce_results=reduce_results,
62 + quant_config=quant_config)
63 + if config.hidden_act != "silu":
64 + raise ValueError(f"Unsupported activation: {config.hidden_act}. "
65 + "Only silu is supported for now.")
66 + self.act_fn = SiluAndMul()
67 +
68 + def forward(self, hidden_states):
69 + gate_up, _ = self.w13(hidden_states)
70 + hidden_states = self.act_fn(gate_up)
71 + hidden_states, _ = self.w2(hidden_states)
72 + return hidden_states
73 +
74 +
75 +class ArcticMoE(nn.Module):
76 + """
77 + Model-parallel implementation of Arctic MoE Layer.
78 + """
79 +
80 + def __init__(self,
81 + config: ArcticConfig,
82 + layer_id: int,
83 + tp_size: Optional[int] = None,
84 + params_dtype: Optional[torch.dtype] = None,
85 + quant_config: Optional[QuantizationConfig] = None,
86 + reduce_results: bool = True):
87 + super(ArcticMoE, self).__init__()
88 +
89 + self.tp_size = tp_size or get_tensor_model_parallel_world_size()
90 + self.hidden_size = config.hidden_size
91 + self.num_experts = config.num_local_experts
92 + self.layer_id = layer_id
93 + self.top_k = config.num_experts_per_tok
94 + self.intermediate_size = config.intermediate_size // self.tp_size
95 +
96 + self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0
97 + self.is_quant = isinstance(quant_config, DeepSpeedFPConfig)
98 + self.reduce_results = reduce_results
99 + # Some other parameters
100 + if params_dtype is None:
101 + params_dtype = torch.get_default_dtype()
102 + self.params_dtype = params_dtype
103 +
104 + if not self.is_moe_layer:
105 + self.mlp = ArcticMLP(config,
106 + layer_id=layer_id,
107 + quant_config=quant_config,
108 + reduce_results=reduce_results)
109 + else:
110 + self.gate = ReplicatedLinear(self.hidden_size,
111 + self.num_experts,
112 + bias=False,
113 + params_dtype=self.params_dtype,
114 + quant_config=quant_config)
115 + if self.is_quant:
116 + self.ws = DeepSpeedFPParameter(
117 + torch.Size((self.num_experts, 2 * self.intermediate_size,
118 + self.hidden_size)),
119 + params_dtype=params_dtype,
120 + quant_config=quant_config,
121 + )
122 + self.w2s = DeepSpeedFPParameter(
123 + torch.Size((self.num_experts, self.hidden_size,
124 + self.intermediate_size)),
125 + params_dtype=params_dtype,
126 + quant_config=quant_config,
127 + )
128 + else:
129 + self.ws = nn.Parameter(
130 + torch.empty(self.num_experts,
131 + 2 * self.intermediate_size,
132 + self.hidden_size,
133 + device="cuda",
134 + dtype=self.params_dtype))
135 + self.w2s = nn.Parameter(
136 + torch.empty(self.num_experts,
137 + self.hidden_size,
138 + self.intermediate_size,
139 + device="cuda",
140 + dtype=self.params_dtype))
141 + set_weight_attrs(self.ws, {
142 + "weight_loader": self.weight_loader,
143 + })
144 + set_weight_attrs(self.w2s, {
145 + "weight_loader": self.weight_loader,
146 + })
147 +
148 + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
149 + weight_name: str, expert_id: int):
150 + tp_rank = get_tensor_model_parallel_rank()
151 + param_data = param.ds_dequantize() if self.is_quant else param.data
152 + shard_size = self.intermediate_size
153 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
154 + if weight_name.endswith("w1.weight"):
155 + param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
156 + if weight_name.endswith("w3.weight"):
157 + param_data[expert_id,
158 + shard_size:2 * shard_size, :] = loaded_weight[shard, :]
159 + if weight_name.endswith("w2.weight"):
160 + param_data[expert_id, :, :] = loaded_weight[:, shard]
161 + if self.is_quant:
162 + param.ds_quantize_(param_data)
163 +
164 + def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor:
165 + num_tokens, hidden_size = hidden_states.shape
166 + hidden_states = hidden_states.view(-1, self.hidden_size)
167 + # router_logits: (num_tokens, n_experts)
168 + router_logits, _ = self.gate(hidden_states)
169 + do_normalize = self.top_k > 1
170 + topk_weights, topk_ids = fused_topk(hidden_states,
171 + router_logits,
172 + self.top_k,
173 + renormalize=do_normalize)
174 + # topk_ids: (num_tokens, k)
175 + if self.is_quant:
176 + if 2 * num_tokens <= self.num_experts:
177 + # If much fewer tokens than experts, use selective dequantize.
178 + ws_dequantized = self.ws.ds_selective_dequantize(
179 + topk_ids.flatten())
180 + w2s_dequantized = self.w2s.ds_selective_dequantize(
181 + topk_ids.flatten())
182 + # We gathered the experts to the tokens so update the mapping.
183 + topk_ids = torch.arange(
184 + 0,
185 + topk_ids.numel(),
186 + device=topk_ids.device,
187 + ).reshape(topk_ids.shape)
188 + else:
189 + ws_dequantized = self.ws.ds_dequantize()
190 + w2s_dequantized = self.w2s.ds_dequantize()
191 +
192 + final_hidden_states = fused_experts(
193 + hidden_states,
194 + ws_dequantized if self.is_quant else self.ws,
195 + w2s_dequantized if self.is_quant else self.w2s,
196 + topk_weights,
197 + topk_ids,
198 + inplace=True)
199 + if self.reduce_results and self.tp_size > 1:
200 + final_hidden_states = tensor_model_parallel_all_reduce(
201 + final_hidden_states)
202 + return final_hidden_states.view(num_tokens, hidden_size)
203 +
204 + def forward(self, hidden_states: torch.Tensor):
205 + if self.is_moe_layer:
206 + final_hidden_states = self.local_moe_fused(hidden_states)
207 + else:
208 + final_hidden_states = self.mlp(hidden_states)
209 + return final_hidden_states
210 +
211 +
212 +class ArcticAttention(nn.Module):
213 +
214 + def __init__(
215 + self,
216 + config: ArcticConfig,
217 + layer_idx: Optional[int] = None,
218 + quant_config: Optional[QuantizationConfig] = None,
219 + ):
220 + super().__init__()
221 + self.config = config
222 + self.layer_idx = layer_idx
223 + self.hidden_size = config.hidden_size
224 +
225 + tp_size = get_tensor_model_parallel_world_size()
226 + self.total_num_heads = config.num_attention_heads
227 + assert self.total_num_heads % tp_size == 0
228 + self.num_heads = self.total_num_heads // tp_size
229 + self.total_num_kv_heads = config.num_key_value_heads
230 + if self.total_num_kv_heads >= tp_size:
231 + assert self.total_num_kv_heads % tp_size == 0
232 + else:
233 + assert tp_size % self.total_num_kv_heads == 0
234 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
235 + self.head_dim = self.hidden_size // self.total_num_heads
236 + self.q_size = self.num_heads * self.head_dim
237 + self.kv_size = self.num_kv_heads * self.head_dim
238 +
239 + self.max_position_embeddings = config.max_position_embeddings
240 + self.rope_theta = config.rope_theta
241 + self.scaling = self.head_dim**-0.5
242 +
243 + self.qkv_proj = QKVParallelLinear(self.hidden_size,
244 + self.head_dim,
245 + self.total_num_heads,
246 + self.total_num_kv_heads,
247 + bias=False,
248 + quant_config=quant_config)
249 + self.o_proj = RowParallelLinear(
250 + self.total_num_heads * self.head_dim,
251 + self.hidden_size,
252 + bias=False,
253 + reduce_results=True,
254 + quant_config=quant_config,
255 + )
256 +
257 + self.rotary_emb = get_rope(
258 + self.head_dim,
259 + rotary_dim=self.head_dim,
260 + max_position=self.max_position_embeddings,
261 + base=int(self.rope_theta),
262 + is_neox_style=True,
263 + )
264 +
265 + self.attn = Attention(self.num_heads,
266 + self.head_dim,
267 + self.scaling,
268 + num_kv_heads=self.num_kv_heads)
269 +
270 + def forward(
271 + self,
272 + positions: torch.Tensor,
273 + hidden_states: torch.Tensor,
274 + kv_cache: torch.Tensor,
275 + attn_metadata: AttentionMetadata,
276 + ) -> torch.Tensor:
277 + qkv, _ = self.qkv_proj(hidden_states)
278 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
279 + q, k = self.rotary_emb(positions, q, k)
280 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
281 + output, _ = self.o_proj(attn_output)
282 + return output
283 +
284 +
285 +class ArcticDecoderLayer(nn.Module):
286 +
287 + def __init__(
288 + self,
289 + config: ArcticConfig,
290 + layer_idx: int,
291 + quant_config: Optional[QuantizationConfig] = None,
292 + ) -> None:
293 + super().__init__()
294 + self.layer_idx = layer_idx
295 + self.hidden_size = config.hidden_size
296 + is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
297 + self.use_residual = config.use_residual and is_moe_layer
298 + self.self_attn = ArcticAttention(config,
299 + layer_idx,
300 + quant_config=quant_config)
301 + self.block_sparse_moe = ArcticMoE(
302 + config,
303 + layer_id=layer_idx,
304 + quant_config=quant_config,
305 + reduce_results=(not self.use_residual))
306 +
307 + self.input_layernorm = RMSNorm(config.hidden_size,
308 + eps=config.rms_norm_eps)
309 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
310 + eps=config.rms_norm_eps)
311 +
312 + if self.use_residual:
313 + self.residual_layernorm = RMSNorm(config.hidden_size,
314 + eps=config.rms_norm_eps)
315 + self.residual_mlp = ArcticMLP(config,
316 + layer_id=layer_idx,
317 + is_residual_mlp=True,
318 + reduce_results=False)
319 +
320 + def forward(
321 + self,
322 + positions: torch.Tensor,
323 + hidden_states: torch.Tensor,
324 + kv_cache: torch.Tensor,
325 + attn_metadata: AttentionMetadata,
326 + ) -> torch.Tensor:
327 + residual_input = hidden_states
328 + hidden_states = self.input_layernorm(hidden_states)
329 + hidden_states = self.self_attn(
330 + positions=positions,
331 + hidden_states=hidden_states,
332 + kv_cache=kv_cache,
333 + attn_metadata=attn_metadata,
334 + )
335 + hidden_states = residual_input + hidden_states
336 +
337 + residual_attn = hidden_states
338 + if self.use_residual:
339 + hidden_states = self.residual_layernorm(hidden_states)
340 + hidden_states = self.residual_mlp(hidden_states)
341 + residual_mlp = hidden_states
342 + hidden_states = self.post_attention_layernorm(residual_input)
343 + hidden_states = self.block_sparse_moe(hidden_states)
344 + hidden_states = residual_mlp + hidden_states
345 + hidden_states = tensor_model_parallel_all_reduce(hidden_states)
346 + hidden_states = residual_attn + hidden_states
347 + else:
348 + hidden_states = self.post_attention_layernorm(hidden_states)
349 + hidden_states = self.block_sparse_moe(hidden_states)
350 + hidden_states = residual_attn + hidden_states
351 + return hidden_states
352 +
353 +
354 +class ArcticModel(nn.Module):
355 +
356 + def __init__(
357 + self,
358 + config: ArcticConfig,
359 + quant_config: Optional[QuantizationConfig] = None,
360 + ) -> None:
361 + super().__init__()
362 + self.padding_idx = config.pad_token_id
363 + self.vocab_size = config.vocab_size
364 + self.embed_tokens = VocabParallelEmbedding(
365 + self.vocab_size,
366 + config.hidden_size,
367 + org_num_embeddings=self.vocab_size)
368 + self.layers = nn.ModuleList([
369 + ArcticDecoderLayer(config, layer_idx, quant_config=quant_config)
370 + for layer_idx in range(config.num_hidden_layers)
371 + ])
372 + self._attn_implementation = config._attn_implementation
373 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
374 +
375 + def forward(
376 + self,
377 + input_ids: torch.Tensor,
378 + positions: torch.Tensor,
379 + kv_caches: List[torch.Tensor],
380 + attn_metadata: AttentionMetadata,
381 + ) -> torch.Tensor:
382 + hidden_states = self.embed_tokens(input_ids)
383 + for i in range(len(self.layers)):
384 + layer = self.layers[i]
385 + hidden_states = layer(positions, hidden_states, kv_caches[i],
386 + attn_metadata)
387 + hidden_states = self.norm(hidden_states)
388 + return hidden_states
389 +
390 +
391 +class ArcticForCausalLM(nn.Module):
392 +
393 + def __init__(self,
394 + config: ArcticConfig,
395 + quant_config: Optional[QuantizationConfig] = None,
396 + **kwargs) -> None:
397 + super().__init__()
398 + self.config = config
399 + self.model = ArcticModel(config, quant_config)
400 + self.vocab_size = config.vocab_size
401 + self.lm_head = ParallelLMHead(
402 + self.vocab_size,
403 + config.hidden_size,
404 + )
405 + self.num_experts = config.num_local_experts
406 + self.num_experts_per_tok = config.num_experts_per_tok
407 + self.unpadded_vocab_size = config.vocab_size
408 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
409 + config.vocab_size)
410 + self.sampler = Sampler()
411 +
412 + def forward(
413 + self,
414 + input_ids: torch.Tensor,
415 + positions: torch.Tensor,
416 + kv_caches: List[torch.Tensor],
417 + attn_metadata: AttentionMetadata,
418 + ) -> torch.Tensor:
419 + hidden_states = self.model(input_ids, positions, kv_caches,
420 + attn_metadata)
421 + return hidden_states
422 +
423 + def compute_logits(self, hidden_states: torch.Tensor,
424 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
425 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
426 + sampling_metadata)
427 + return logits
428 +
429 + def sample(
430 + self,
431 + logits: Optional[torch.Tensor],
432 + sampling_metadata: SamplingMetadata,
433 + ) -> Optional[SamplerOutput]:
434 + next_tokens = self.sampler(logits, sampling_metadata)
435 + return next_tokens
436 +
437 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
438 + stacked_params_mapping = [
439 + # (param_name, shard_name, shard_id)
440 + ("qkv_proj", "q_proj", "q"),
441 + ("qkv_proj", "k_proj", "k"),
442 + ("qkv_proj", "v_proj", "v"),
443 + ]
444 +
445 + mlp_params_mapping = []
446 + expert_params_mapping = []
447 + num_layers = self.config.num_hidden_layers
448 +
449 + for layer in range(num_layers):
450 + mlp_params_mapping.append(
451 + (f"layers.{layer}.residual_mlp.w13.weight",
452 + f"layers.{layer}.residual_mlp.w1.weight", 0))
453 + mlp_params_mapping.append(
454 + (f"layers.{layer}.residual_mlp.w13.weight",
455 + f"layers.{layer}.residual_mlp.w3.weight", 1))
456 + if layer % 2 == 0:
457 + # MLP layers
458 + mlp_params_mapping.append(
459 + (f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
460 + f"layers.{layer}.block_sparse_moe.mlp.w1.weight", 0))
461 + mlp_params_mapping.append(
462 + (f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
463 + f"layers.{layer}.block_sparse_moe.mlp.w3.weight", 1))
464 + else:
465 + # MoE layers
466 + for expert_id in range(self.config.num_local_experts):
467 + expert_params_mapping.append(
468 + ("ws", f"experts.{expert_id}.w1.weight", expert_id))
469 + expert_params_mapping.append(
470 + ("w2s", f"experts.{expert_id}.w2.weight", expert_id))
471 + expert_params_mapping.append(
472 + ("ws", f"experts.{expert_id}.w3.weight", expert_id))
473 +
474 + params_dict = dict(self.named_parameters())
475 +
476 + logger.info(
477 + "It will take ~10 minutes loading from the 16-bit weights. "
478 + "Alternatively, use the prequantized 8-bit weights of arctic "
479 + "and set load-format to `sharded_state` will accelerate loading.")
480 + for name, loaded_weight in weights:
481 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
482 + if weight_name not in name:
483 + continue
484 + name = name.replace(weight_name, param_name)
485 + # Skip loading extra bias for GPTQ models.
486 + if name.endswith(".bias") and name not in params_dict:
487 + continue
488 + param = params_dict[name]
489 + weight_loader = param.weight_loader
490 + weight_loader(param, loaded_weight, shard_id)
491 + break
492 + else:
493 + for param_name, weight_name, shard_id in mlp_params_mapping:
494 + if weight_name not in name:
495 + continue
496 + name = name.replace(weight_name, param_name)
497 + param = params_dict[name]
498 + weight_loader = param.weight_loader
499 + weight_loader(param, loaded_weight, shard_id)
500 + break
501 + else:
502 + for param_name, weight_name, shard_id \
503 + in expert_params_mapping:
504 + if weight_name not in name:
505 + continue
506 + name = name.replace(weight_name, param_name)
507 + param = params_dict[name]
508 + weight_loader = param.weight_loader
509 + weight_loader(param,
510 + loaded_weight,
511 + weight_name,
512 + expert_id=shard_id)
513 + break
514 + else:
515 + if name.endswith(".bias") and name not in params_dict:
516 + continue
517 + param = params_dict[name]
518 +
519 + weight_loader = getattr(param, "weight_loader",
520 + default_weight_loader)
521 + weight_loader(param, loaded_weight)
vllm/transformers_utils/configs/arctic.py new
+204
@@ -0,0 +1,204 @@
1 +# yapf: disable
2 +# ruff: noqa: E501
3 +# coding=utf-8
4 +# Copied from
5 +# https://huggingface.co/Snowflake/snowflake-arctic-instruct/blob/main/configuration_arctic.py
6 +""" Arctic model configuration"""
7 +
8 +from dataclasses import asdict, dataclass
9 +from typing import Any, Dict
10 +
11 +from transformers.configuration_utils import PretrainedConfig
12 +from transformers.utils import logging
13 +
14 +logger = logging.get_logger(__name__)
15 +
16 +ARCTIC_PRETRAINED_CONFIG_ARCHIVE_MAP = {
17 + "arctic": "https://huggingface.co/Snowflake/snowflake-arctic-instruct/tree/main/config.json",
18 +}
19 +
20 +
21 +@dataclass
22 +class ArcticLoraConfig:
23 + lora_r: int = 64
24 + lora_alpha: float = 16
25 + shard_base_weights: bool = False
26 +
27 +
28 +@dataclass
29 +class ArcticQuantizationConfig:
30 + q_bits: int = 8
31 + rounding: str = "nearest"
32 + mantissa_bits: int = 3
33 + group_size: int = 128
34 +
35 +
36 +class ArcticConfig(PretrainedConfig):
37 + r"""
38 + This is the configuration class to store the configuration of a [`ArcticModel`]. It is used to instantiate an
39 + Arctic model according to the specified arguments, defining the model architecture. Instantiating a configuration
40 + with the defaults will yield a similar configuration to that of the #TODO(rsamdani): add what model has the default config..
41 +
42 +
43 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
44 + documentation from [`PretrainedConfig`] for more information.
45 +
46 +
47 + Args:
48 + vocab_size (`int`, *optional*, defaults to 32000):
49 + Vocabulary size of the Arctic model. Defines the number of different tokens that can be represented by the
50 + `inputs_ids` passed when calling [`ArcticModel`]
51 + hidden_size (`int`, *optional*, defaults to 4096):
52 + Dimension of the hidden representations.
53 + intermediate_size (`int`, *optional*, defaults to 14336):
54 + Dimension of the MLP representations.
55 + num_hidden_layers (`int`, *optional*, defaults to 32):
56 + Number of hidden layers in the Transformer encoder.
57 + num_attention_heads (`int`, *optional*, defaults to 32):
58 + Number of attention heads for each attention layer in the Transformer encoder.
59 + num_key_value_heads (`int`, *optional*, defaults to 8):
60 + This is the number of key_value heads that should be used to implement Grouped Query Attention. If
61 + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
62 + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
63 + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
64 + by meanpooling all the original heads within that group. For more details checkout [this
65 + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
66 + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
67 + The non-linear activation function (function or string) in the decoder.
68 + max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
69 + The maximum sequence length that this model might ever be used with. Arctic's sliding window attention
70 + allows sequence of up to 4096*32 tokens.
71 + initializer_range (`float`, *optional*, defaults to 0.02):
72 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
73 + rms_norm_eps (`float`, *optional*, defaults to 1e-05):
74 + The epsilon used by the rms normalization layers.
75 + use_cache (`bool`, *optional*, defaults to `True`):
76 + Whether or not the model should return the last key/values attentions (not used by all models). Only
77 + relevant if `config.is_decoder=True`.
78 + pad_token_id (`int`, *optional*):
79 + The id of the padding token.
80 + bos_token_id (`int`, *optional*, defaults to 1):
81 + The id of the "beginning-of-sequence" token.
82 + eos_token_id (`int`, *optional*, defaults to 2):
83 + The id of the "end-of-sequence" token.
84 + tie_word_embeddings (`bool`, *optional*, defaults to `False`):
85 + Whether the model's input and output word embeddings should be tied.
86 + rope_theta (`float`, *optional*, defaults to 1000000.0):
87 + The base period of the RoPE embeddings.
88 + sliding_window (`int`, *optional*):
89 + Sliding window attention window size. If not specified, will default to `4096`.
90 + attention_dropout (`float`, *optional*, defaults to 0.0):
91 + The dropout ratio for the attention probabilities.
92 + num_experts_per_tok (`int`, *optional*, defaults to 2):
93 + The number of experts to root per-token, can be also interpreted as the `top-p` routing
94 + parameter
95 + num_local_experts (`int`, *optional*, defaults to 8):
96 + Number of experts per Sparse MLP layer.
97 + router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
98 + The aux loss factor for the total loss.
99 +
100 + ```python
101 + >>> from transformers import ArcticModel, ArcticConfig
102 +
103 + >>> # Initializing a Arctic 7B style configuration TODO(rsamdani): verify which model does the default configuration correspond to.
104 + >>> configuration = ArcticConfig()
105 +
106 + >>> # Initializing a model from the Arctic 7B style configuration
107 + >>> model = ArcticModel(configuration)
108 +
109 + >>> # Accessing the model configuration
110 + >>> configuration = model.config
111 + ```"""
112 +
113 + model_type = "arctic"
114 + keys_to_ignore_at_inference = ["past_key_values"]
115 +
116 + def __init__(
117 + self,
118 + vocab_size=32000,
119 + hidden_size=4096,
120 + intermediate_size=14336,
121 + num_hidden_layers=32,
122 + num_attention_heads=32,
123 + num_key_value_heads=None,
124 + hidden_act="silu",
125 + max_position_embeddings=4096,
126 + initializer_range=0.02,
127 + rms_norm_eps=1e-5,
128 + use_cache=True,
129 + pad_token_id=None,
130 + bos_token_id=1,
131 + eos_token_id=2,
132 + tie_word_embeddings=False,
133 + rope_theta=1e6,
134 + sliding_window=None,
135 + attention_dropout=0.0,
136 + num_experts_per_tok=1,
137 + num_local_experts=8,
138 + router_aux_loss_coef=0.001,
139 + moe_layer_frequency=2,
140 + parallel_attn_mlp_res=False,
141 + moe_train_capacity_factor=1,
142 + moe_eval_capacity_factor=1,
143 + enable_expert_tensor_parallelism=False,
144 + moe_min_capacity=0,
145 + moe_token_dropping=True,
146 + quantization=None,
147 + **kwargs,
148 + ):
149 + self.vocab_size = vocab_size
150 + self.max_position_embeddings = max_position_embeddings
151 + self.hidden_size = hidden_size
152 + self.intermediate_size = intermediate_size
153 + self.num_hidden_layers = num_hidden_layers
154 + self.num_attention_heads = num_attention_heads
155 + self.sliding_window = sliding_window
156 +
157 + # for backward compatibility
158 + if num_key_value_heads is None:
159 + num_key_value_heads = num_attention_heads
160 +
161 + self.num_key_value_heads = num_key_value_heads
162 + self.hidden_act = hidden_act
163 + self.initializer_range = initializer_range
164 + self.rms_norm_eps = rms_norm_eps
165 + self.use_cache = use_cache
166 + self.rope_theta = rope_theta
167 + self.attention_dropout = attention_dropout
168 +
169 + self.num_experts_per_tok = num_experts_per_tok
170 + self.num_local_experts = num_local_experts
171 + self.router_aux_loss_coef = router_aux_loss_coef
172 + self.moe_layer_frequency = moe_layer_frequency
173 + self.moe_train_capacity_factor = moe_train_capacity_factor
174 + self.moe_eval_capacity_factor = moe_eval_capacity_factor
175 + self.enable_expert_tensor_parallelism = enable_expert_tensor_parallelism
176 + self.moe_min_capacity = moe_min_capacity
177 + self.moe_token_dropping = moe_token_dropping
178 + self.parallel_attn_mlp_res = parallel_attn_mlp_res
179 + if isinstance(quantization, dict):
180 + self.quantization = ArcticQuantizationConfig(**quantization)
181 + else:
182 + self.quantization = quantization
183 +
184 + super().__init__(
185 + pad_token_id=pad_token_id,
186 + bos_token_id=bos_token_id,
187 + eos_token_id=eos_token_id,
188 + tie_word_embeddings=tie_word_embeddings,
189 + **kwargs,
190 + )
191 +
192 + @classmethod
193 + def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "ArcticConfig":
194 + result = super().from_dict(config_dict, **kwargs)
195 + config = result[0] if isinstance(result, tuple) else result
196 + if isinstance(config.quantization, dict):
197 + config.quantization = ArcticQuantizationConfig(**config.quantization)
198 + return result
199 +
200 + def to_dict(self) -> Dict[str, Any]:
201 + ret = super().to_dict()
202 + if isinstance(ret["quantization"], ArcticQuantizationConfig):
203 + ret["quantization"] = asdict(ret["quantization"])
204 + return ret