import{s as kn,o as An,n as Cn}from"../chunks/scheduler.78382b47.js";import{S as jn,i as qn,e as s,s as n,c as p,h as Rn,a as l,d as a,b as o,f as w,g as c,j as d,k as x,l as i,m as r,n as m,t as g,o as f,p as h}from"../chunks/index.6dd35eb6.js";import{C as Un,H as M,E as Jn}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fed9dfe4.js";import{D as L}from"../chunks/Docstring.32e826ec.js";import{C as Ln}from"../chunks/CodeBlock.af1ec02e.js";import{E as $n}from"../chunks/ExampleCodeBlock.7f184f2e.js";function Dn(Je){let u,j="Example:",b,C,q;return C=new Ln({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUwQWZyb20lMjBwZWZ0JTIwaW1wb3J0JTIwTG9yYU1vZGVsJTJDJTIwTG9yYUNvbmZpZyUwQSUwQWNvbmZpZyUyMCUzRCUyMExvcmFDb25maWcoJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyU0VRXzJfU0VRX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwciUzRDglMkMlMEElMjAlMjAlMjAlMjBsb3JhX2FscGhhJTNEMzIlMkMlMEElMjAlMjAlMjAlMjB0YXJnZXRfbW9kdWxlcyUzRCU1QiUyMnElMjIlMkMlMjAlMjJ2JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9kcm9wb3V0JTNEMC4wMSUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTS5mcm9tX3ByZXRyYWluZWQoJTIydDUtYmFzZSUyMiklMEFsb3JhX21vZGVsJTIwJTNEJTIwTG9yYU1vZGVsKG1vZGVsJTJDJTIwY29uZmlnJTJDJTIwJTIyZGVmYXVsdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraModel, LoraConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = LoraConfig(
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;SEQ_2_SEQ_LM&quot;</span>,
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    lora_alpha=<span class="hljs-number">32</span>,
<span class="hljs-meta">... </span>    target_modules=[<span class="hljs-string">&quot;q&quot;</span>, <span class="hljs-string">&quot;v&quot;</span>],
<span class="hljs-meta">... </span>    lora_dropout=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;t5-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>lora_model = LoraModel(model, config, <span class="hljs-string">&quot;default&quot;</span>)`,wrap:!1}}),{c(){u=s("p"),u.textContent=j,b=n(),p(C.$$.fragment)},l(v){u=l(v,"P",{"data-svelte-h":!0}),d(u)!=="svelte-11lpom8"&&(u.textContent=j),b=o(v),c(C.$$.fragment,v)},m(v,R){r(v,u,R),r(v,b,R),m(C,v,R),q=!0},p:Cn,i(v){q||(g(C.$$.fragment,v),q=!0)},o(v){f(C.$$.fragment,v),q=!1},d(v){v&&(a(u),a(b)),h(C,v)}}}function En(Je){let u,j;return u=new Ln({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0cmFuc2Zvcm1lcnMlMEFmcm9tJTIwcGVmdCUyMGltcG9ydCUyMExvcmFDb25maWclMkMlMjBQZWZ0TW9kZWwlMkMlMjBnZXRfcGVmdF9tb2RlbCUyQyUyMHByZXBhcmVfbW9kZWxfZm9yX2tiaXRfdHJhaW5pbmclMEElMEFyYW5rJTIwJTNEJTIwLi4uJTBBdGFyZ2V0X21vZHVsZXMlMjAlM0QlMjAlNUIlMjJxX3Byb2olMjIlMkMlMjAlMjJrX3Byb2olMjIlMkMlMjAlMjJ2X3Byb2olMjIlMkMlMjAlMjJvdXRfcHJvaiUyMiUyQyUyMCUyMmZjX2luJTIyJTJDJTIwJTIyZmNfb3V0JTIyJTJDJTIwJTIyd3RlJTIyJTVEJTBBY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNENCUyQyUyMGxvcmFfYWxwaGElM0QxNiUyQyUyMHRhcmdldF9tb2R1bGVzJTNEdGFyZ2V0X21vZHVsZXMlMkMlMjBsb3JhX2Ryb3BvdXQlM0QwLjElMkMlMjBiaWFzJTNEJTIybm9uZSUyMiUyQyUyMHRhc2tfdHlwZSUzRCUyMkNBVVNBTF9MTSUyMiUwQSklMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwdHJhbnNmb3JtZXJzLkJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMHRyYW5zZm9ybWVycy5BdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJrYWthb2JyYWluJTJGa29ncHQlMjIlMkMlMEElMjAlMjAlMjAlMjByZXZpc2lvbiUzRCUyMktvR1BUNkItcnlhbjEuNWItZmxvYXQxNiUyMiUyQyUyMCUyMCUyMyUyMG9yJTIwZmxvYXQzMiUyMHZlcnNpb24lM0ElMjByZXZpc2lvbiUzREtvR1BUNkItcnlhbjEuNWIlMEElMjAlMjAlMjAlMjBib3NfdG9rZW4lM0QlMjIlNUJCT1MlNUQlMjIlMkMlMEElMjAlMjAlMjAlMjBlb3NfdG9rZW4lM0QlMjIlNUJFT1MlNUQlMjIlMkMlMEElMjAlMjAlMjAlMjB1bmtfdG9rZW4lM0QlMjIlNUJVTkslNUQlMjIlMkMlMEElMjAlMjAlMjAlMjBwYWRfdG9rZW4lM0QlMjIlNUJQQUQlNUQlMjIlMkMlMEElMjAlMjAlMjAlMjBtYXNrX3Rva2VuJTNEJTIyJTVCTUFTSyU1RCUyMiUyQyUwQSklMEFtb2RlbCUyMCUzRCUyMHRyYW5zZm9ybWVycy5HUFRKRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmtha2FvYnJhaW4lMkZrb2dwdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHJldmlzaW9uJTNEJTIyS29HUFQ2Qi1yeWFuMS41Yi1mbG9hdDE2JTIyJTJDJTIwJTIwJTIzJTIwb3IlMjBmbG9hdDMyJTIwdmVyc2lvbiUzQSUyMHJldmlzaW9uJTNES29HUFQ2Qi1yeWFuMS41YiUwQSUyMCUyMCUyMCUyMHBhZF90b2tlbl9pZCUzRHRva2VuaXplci5lb3NfdG9rZW5faWQlMkMlMEElMjAlMjAlMjAlMjB1c2VfY2FjaGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlN0IlMjIlMjIlM0ElMjByYW5rJTdEJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEpJTBBbW9kZWwlMjAlM0QlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nKG1vZGVsKSUwQWxvcmFfbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChtb2RlbCUyQyUyMGNvbmZpZyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> transformers
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training

<span class="hljs-meta">&gt;&gt;&gt; </span>rank = ...
<span class="hljs-meta">&gt;&gt;&gt; </span>target_modules = [<span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;out_proj&quot;</span>, <span class="hljs-string">&quot;fc_in&quot;</span>, <span class="hljs-string">&quot;fc_out&quot;</span>, <span class="hljs-string">&quot;wte&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>config = LoraConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">4</span>, lora_alpha=<span class="hljs-number">16</span>, target_modules=target_modules, lora_dropout=<span class="hljs-number">0.1</span>, bias=<span class="hljs-string">&quot;none&quot;</span>, task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>quantization_config = transformers.BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = transformers.AutoTokenizer.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;kakaobrain/kogpt&quot;</span>,
<span class="hljs-meta">... </span>    revision=<span class="hljs-string">&quot;KoGPT6B-ryan1.5b-float16&quot;</span>,  <span class="hljs-comment"># or float32 version: revision=KoGPT6B-ryan1.5b</span>
<span class="hljs-meta">... </span>    bos_token=<span class="hljs-string">&quot;[BOS]&quot;</span>,
<span class="hljs-meta">... </span>    eos_token=<span class="hljs-string">&quot;[EOS]&quot;</span>,
<span class="hljs-meta">... </span>    unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>,
<span class="hljs-meta">... </span>    pad_token=<span class="hljs-string">&quot;[PAD]&quot;</span>,
<span class="hljs-meta">... </span>    mask_token=<span class="hljs-string">&quot;[MASK]&quot;</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = transformers.GPTJForCausalLM.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;kakaobrain/kogpt&quot;</span>,
<span class="hljs-meta">... </span>    revision=<span class="hljs-string">&quot;KoGPT6B-ryan1.5b-float16&quot;</span>,  <span class="hljs-comment"># or float32 version: revision=KoGPT6B-ryan1.5b</span>
<span class="hljs-meta">... </span>    pad_token_id=tokenizer.eos_token_id,
<span class="hljs-meta">... </span>    use_cache=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    device_map={<span class="hljs-string">&quot;&quot;</span>: rank},
<span class="hljs-meta">... </span>    torch_dtype=torch.float16,
<span class="hljs-meta">... </span>    quantization_config=quantization_config,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = prepare_model_for_kbit_training(model)
<span class="hljs-meta">&gt;&gt;&gt; </span>lora_model = get_peft_model(model, config)`,wrap:!1}}),{c(){p(u.$$.fragment)},l(b){c(u.$$.fragment,b)},m(b,C){m(u,b,C),j=!0},p:Cn,i(b){j||(g(u.$$.fragment,b),j=!0)},o(b){f(u.$$.fragment,b),j=!1},d(b){h(u,b)}}}function In(Je){let u,j,b,C,q,v,R,_t,H,Sa='Low-Rank Adaptation (<a href="https://huggingface.co/papers/2309.15223" rel="nofollow">LoRA</a>) is a PEFT method that decomposes a large matrix into two smaller low-rank matrices in the attention layers. This drastically reduces the number of parameters that need to be fine-tuned.',bt,Q,Na="The abstract from the paper is:",vt,X,Va="<em>We propose a neural language modeling system based on low-rank adaptation (LoRA) for speech recognition output rescoring. Although pretrained language models (LMs) like BERT have shown superior performance in second-pass rescoring, the high computational cost of scaling up the pretraining stage and adapting the pretrained models to specific domains limit their practical use in rescoring. Here we present a method based on low-rank decomposition to train a rescoring BERT model and adapt it to new domains using only a fraction (0.08%) of the pretrained parameters. These inserted matrices are optimized through a discriminative training objective along with a correlation-based regularization loss. The proposed low-rank adaptation Rescore-BERT (LoRB) architecture is evaluated on LibriSpeech and internal datasets with decreased training times by factors between 5.4 and 3.6.</em>.",yt,Y,wt,U,K,na,De,Ba='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/lora#peft.LoraModel">LoraModel</a>.',oa,P,ee,ia,Ee,Pa="Returns the configuration for your adapter model as a dictionary. Removes runtime configurations.",xt,te,Mt,_,ae,ra,Ie,Fa="Creates Low Rank Adapter (LoRA) model from a pretrained transformers model.",sa,ze,Ga='The method is described in detail in <a href="https://huggingface.co/papers/2106.09685" rel="nofollow">https://huggingface.co/papers/2106.09685</a>.',la,F,da,G,pa,Se,Za="<strong>Attributes</strong>:",ca,Ne,Oa='<li><strong>model</strong> (<a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel" rel="nofollow">PreTrainedModel</a>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/main/en/package_reference/lora#peft.LoraConfig">LoraConfig</a>): The configuration of the Lora model.</li>',ma,I,ne,ga,Ve,Wa="This method adds a new adapter by merging the given adapters with the given weights.",fa,Be,Ha=`When using the <code>cat</code> combination_type you should be aware that rank of the resulting adapter will be equal to
the sum of all adapters ranks. So it’s possible that the mixed adapter may become too big and result in OOM
errors.`,ha,Z,oe,ua,Pe,Qa=`This function can calculate the updates of the PiSSA/CorDA/OLoRA by comparing the parameters of the
PiSSA/CorDA/OLoRA adapter in <code>output_state_dict</code> with the initial values of PiSSA/CorDA/OLoRA in
<code>adapter_name</code>, thus converting PiSSA/CorDA/OLoRA to LoRA.`,Tt,ie,$t,re,Ct,N,se,_a,Fe,Xa=`This is the sub-configuration class to store the configuration for Arrow and GenKnowSub algorithm. Arrow is a
routing algorithm to combine the trained LoRA modules to solve new tasks, proposed in
’<a href="https://huggingface.co/papers/2405.11157&#39;" rel="nofollow">https://huggingface.co/papers/2405.11157’</a>. GenKnowSub is a refinement on the trained modules before being combined
via Arrow, introduced in ’<a href="https://aclanthology.org/2025.acl-short.54/&#39;" rel="nofollow">https://aclanthology.org/2025.acl-short.54/’</a>`,Lt,le,kt,T,de,ba,Ge,Ya="Replace the LoRA weights of a model quantized with bitsandbytes, using the LoftQ technique.",va,Ze,Ka=`The replacement is done on the fly by loading in the non-quantized weights from a locally stored safetensors model
file and initializing the LoRA weights such that the quantization error between the original and quantized weights
is minimized.`,ya,Oe,en="As lazy loading is not possible with pickle, normal PyTorch checkpoint files cannot be supported.",wa,We,tn="Depending on the model size, calling this function may take some time to finish.",At,pe,jt,ce,qt,V,me,xa,He,an=`This is the sub-configuration class to store the configuration for a data-driven initialization via EVA. EVA was
introduced in <a href="https://huggingface.co/papers/2410.07170">Explained Variance Adaptation</a>.`,Rt,ge,Ut,J,fe,Ma,Qe,nn="Initialize the weights of the LoRA layers using the EVA method.",Ta,Xe,on=`This function initializes the weights of the LoRA layers using the EVA method. It computes the SVD for each adapter
layer and updates the weights accordingly.`,Jt,he,Dt,D,ue,$a,Ye,rn="Compute the SVD for each layer in the model.",Ca,Ke,sn=`This function computes the Singular Value Decomposition (SVD) for each layer in the model. It uses the incremental
PCA method to compute the SVD components. The function also checks for convergence of the computed components using
cosine similarity. The rank distribution for each layer is determined based on the explained variance ratio.`,Et,_e,It,be,zt,ve,ln='<a href="https://hf.co/papers/2407.05000" rel="nofollow">LoRA-GA</a> (Low-Rank Adaptation with Gradient Approximation) improves upon standard LoRA by using gradient information during initialization to achieve faster convergence. Instead of random initialization, LoRA-GA performs SVD on estimated gradients to initialize adapter weights in a direction that aligns with full fine-tuning, resulting in 2-4x faster convergence with the same final performance.',St,ye,dn="The abstract from the paper is:",Nt,we,pn="<em>Low-rank adaptation (LoRA) is a popular technique for parameter-efficient fine-tuning of large language models. However, LoRA’s random initialization of adapter weights leads to slow convergence during the initial training phase. In this paper, we propose LoRA-GA (Low-Rank Adaptation with Gradient Approximation), a novel initialization method that leverages gradient information to initialize LoRA adapters. Specifically, we estimate gradients on a small set of training samples and perform singular value decomposition (SVD) to extract principal components. These components are used to initialize the adapter matrices, aligning the initial update direction with that of full fine-tuning. Our experiments across various tasks and model scales demonstrate that LoRA-GA achieves 2-4x faster convergence compared to standard LoRA while maintaining the same final performance. The method is orthogonal to existing LoRA variants and can be easily integrated with techniques like DoRA and LoRA+.</em>",Vt,xe,Bt,Me,cn="<li><strong>Gradient Estimation</strong>: LoRA-GA requires a gradient estimation phase before model initialization. Use <code>preprocess_loraga()</code> with a <code>train_step</code> callback to compute gradients over a small number of training batches (typically 64-128 batches).</li>",Pt,Te,mn="<li><p><strong>Initialization Strategies</strong>: LoRA-GA supports four direction strategies (<code>direction</code>): <code>&quot;ArBr&quot;</code>, <code>&quot;A2rBr&quot;</code>, <code>&quot;ArB2r&quot;</code> (default), and <code>&quot;random&quot;</code>, and four scaling strategies (<code>scale</code>): <code>&quot;stable&quot;</code> (default), <code>&quot;weight_svd&quot;</code>, <code>&quot;gd_scale&quot;</code>, and <code>&quot;unit&quot;</code>. The default combination provides the best balance of convergence speed and stability.</p></li> <li><p><strong>Base Weight Modification</strong>: Unlike standard LoRA, LoRA-GA modifies the base model weights during initialization by subtracting a scaled version of the low-rank approximation. This enables better alignment with full fine-tuning gradients. Since base weights are modified, use <code>save_pretrained()</code> with the <code>save_embedding_layers</code> argument or <code>save_mutated_as_lora</code> pattern to properly save the adapter.</p></li> <li><p><strong>Computational Overhead</strong>: The gradient estimation adds a small overhead during initialization (typically 1-2 minutes for 64 batches), but this is quickly amortized by faster convergence during training.</p></li> <li><p><strong>Compatibility</strong>: LoRA-GA requires full-precision weights and does not support quantized models. Can be combined with other LoRA variants like DoRA.</p></li>",Ft,$e,Gt,k,Ce,La,et,gn="This is the sub-configuration class to store the configuration for LoRA-GA initialization.",ka,tt,fn=`LoRA-GA (Low-Rank Adaptation with Gradient Approximation) uses gradient information during initialization to
achieve faster convergence (2-4x speedup) by aligning the initial adapter weights with the direction of full
fine-tuning gradients.`,Aa,at,hn='Reference: <a href="https://arxiv.org/abs/2407.05000" rel="nofollow">https://arxiv.org/abs/2407.05000</a>',Zt,Le,Ot,E,ke,ja,nt,un="Estimate gradients for LoRA-GA initialization.",qa,ot,_n=`This function enables gradient computation ONLY on target module weights and runs the train_step callback. This is
more memory-efficient than enabling gradients globally.`,Wt,A,Ae,Ra,it,bn="Build necessary LoRA-GA fields for a model by estimating gradients.",Ua,rt,vn=`For each linear layer, gradients will be estimated by running the provided train_step callback. These gradients are
then attached to the modules and used during initialization.`,Ja,st,yn=`Upon completion, the following fields are set for each target module:
_peft_loraga_grad (<code>torch.Tensor</code>):
Accumulated gradient for the weight matrix.`,Ht,je,Qt,$,qe,Da,lt,wn=`Intruder dimension mitigation based on <a href="https://huggingface.co/papers/2410.21228" rel="nofollow">https://huggingface.co/papers/2410.21228</a> (“LoRA vs Full Fine-tuning: An
Illusion of Equivalence”).`,Ea,dt,xn=`This method can recover previous knowledge (i.e. mitigate forgetting) by post-processing already trained low-rank
adapters. This comes at a cost of task accuracy - tuning the <code>migration_lambda</code> value can be used to trade between
these two factors.`,Ia,pt,Mn=`After mitigation is done there will be a new adapter with the name set in <code>new_adapter_name</code> which is also set to
be the currently active adapter. Inference on the mitigated model will therefore use the modified adapter. To
switch back to the original adapter you can use <code>peft_model.set_adapter(&lt;old_adapter_name&gt;)</code>.`,za,ct,Tn="Currently only LoRA is supported as it is not clear whether this method generalizes to other delta-weight methods.",Xt,Re,Yt,ut,Kt;return q=new Un({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),R=new M({props:{title:"LoRA",local:"lora",headingTag:"h1"}}),Y=new M({props:{title:"LoraConfig",local:"peft.LoraConfig",headingTag:"h2"}}),K=new L({props:{name:"class peft.LoraConfig",anchor:"peft.LoraConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"r",val:": int = 8"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"lora_alpha",val:": int = 8"},{name:"lora_dropout",val:": float = 0.0"},{name:"fan_in_fan_out",val:": bool = False"},{name:"bias",val:": Literal['none', 'all', 'lora_only'] = 'none'"},{name:"use_rslora",val:": bool = False"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"init_lora_weights",val:": bool | Literal['gaussian', 'eva', 'olora', 'pissa', 'pissa_niter_[number of iters]', 'corda', 'loftq', 'orthogonal'] = True"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"rank_pattern",val:": Optional[dict] = <factory>"},{name:"alpha_pattern",val:": Optional[dict] = <factory>"},{name:"megatron_config",val:": Optional[dict] = None"},{name:"megatron_core",val:": Optional[str] = 'megatron.core'"},{name:"trainable_token_indices",val:": Optional[Union[list[int], dict[str, list[int]]]] = None"},{name:"loftq_config",val:": Union[LoftQConfig, dict] = <factory>"},{name:"eva_config",val:": Optional[EvaConfig] = None"},{name:"corda_config",val:": Optional[CordaConfig] = None"},{name:"lora_ga_config",val:": Optional[LoraGAConfig] = None"},{name:"use_dora",val:": bool = False"},{name:"alora_invocation_tokens",val:": Optional[list[int]] = None"},{name:"use_qalora",val:": bool = False"},{name:"qalora_group_size",val:": int = 16"},{name:"layer_replication",val:": Optional[list[tuple[int, int]]] = None"},{name:"runtime_config",val:": LoraRuntimeConfig = <factory>"},{name:"lora_bias",val:": bool = False"},{name:"target_parameters",val:": Optional[list[str]] = None"},{name:"use_bdlora",val:": Optional[BdLoraConfig] = None"},{name:"arrow_config",val:": Optional[ArrowConfig] = None"},{name:"ensure_weight_tying",val:": bool = False"}],parametersDescription:[{anchor:"peft.LoraConfig.r",description:`<strong>r</strong> (<code>int</code>) &#x2014;
Lora attention dimension (the &#x201C;rank&#x201D;).`,name:"r"},{anchor:"peft.LoraConfig.target_modules",description:`<strong>target_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to apply the adapter to. If this is specified, only the modules with the specified
names will be replaced. When passing a string, a regex match will be performed. When passing a list of
strings, either an exact match will be performed or it is checked if the name of the module ends with any
of the passed strings. If this is specified as &#x2018;all-linear&#x2019;, then all linear/Conv1D modules are chosen (if
the model is a PreTrainedModel, the output layer excluded). If this is not specified, modules will be
chosen according to the model architecture. If the architecture is not known, an error will be raised &#x2014; in
this case, you should specify the target modules manually. To avoid targeting any modules (because you want
to apply <code>target_parameters</code>), set <code>target_modules=[]</code>.`,name:"target_modules"},{anchor:"peft.LoraConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
When passing a list of strings, either an exact match will be performed or it is checked if the name of the
module ends with any of the passed strings.`,name:"exclude_modules"},{anchor:"peft.LoraConfig.lora_alpha",description:`<strong>lora_alpha</strong> (<code>int</code>) &#x2014;
The alpha parameter for Lora scaling.`,name:"lora_alpha"},{anchor:"peft.LoraConfig.lora_dropout",description:`<strong>lora_dropout</strong> (<code>float</code>) &#x2014;
The dropout probability for Lora layers.`,name:"lora_dropout"},{anchor:"peft.LoraConfig.fan_in_fan_out",description:`<strong>fan_in_fan_out</strong> (<code>bool</code>) &#x2014;
Set this to True if the layer to replace stores weight like (fan_in, fan_out). For example, gpt-2 uses
<code>Conv1D</code> which stores weights like (fan_in, fan_out) and hence this should be set to <code>True</code>.`,name:"fan_in_fan_out"},{anchor:"peft.LoraConfig.bias",description:`<strong>bias</strong> (<code>str</code>) &#x2014;
Bias type for LoRA. Can be &#x2018;none&#x2019;, &#x2018;all&#x2019; or &#x2018;lora_only&#x2019;. If &#x2018;all&#x2019; or &#x2018;lora_only&#x2019;, the corresponding biases
will be updated during training. Be aware that this means that, even when disabling the adapters, the model
will not produce the same output as the base model would have without adaptation.`,name:"bias"},{anchor:"peft.LoraConfig.use_rslora",description:`<strong>use_rslora</strong> (<code>bool</code>) &#x2014;
When set to True, uses <a href="https://huggingface.co/papers/2312.03732" rel="nofollow">Rank-Stabilized LoRA</a> which sets the
adapter scaling factor to <code>lora_alpha/math.sqrt(r)</code>, since it was proven to work better. Otherwise, it will
use the original default value of <code>lora_alpha/r</code>.`,name:"use_rslora"},{anchor:"peft.LoraConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>List[str]</code>) &#x2014;
List of modules apart from adapter layers to be set as trainable and saved in the final checkpoint.`,name:"modules_to_save"},{anchor:"peft.LoraConfig.init_lora_weights",description:`<strong>init_lora_weights</strong> (<code>bool</code> | <code>Literal[&quot;gaussian&quot;, &quot;eva&quot;, &quot;olora&quot;, &quot;pissa&quot;, &quot;pissa_niter_[number of iters]&quot;, &quot;corda&quot;, &quot;loftq&quot;, &quot;orthogonal&quot;]</code>) &#x2014;
How to initialize the weights of the adapter layers. Passing True (default) results in the default
initialization from the reference implementation from Microsoft, with the LoRA B weight being set to 0.
This means that without further training, the LoRA adapter will be a no-op. Setting the initialization to
False leads to random initialization of LoRA A and B, meaning that LoRA is not a no-op before training;
this setting is intended for debugging purposes. Passing &#x2018;gaussian&#x2019; results in Gaussian initialization
scaled by the LoRA rank for linear and layers. Pass <code>&apos;loftq&apos;</code> to use LoftQ initialization. Passing <code>&apos;eva&apos;</code>
results in a data-driven initialization of <a href="https://huggingface.co/papers/2410.07170">Explained
Variance Adaptation</a>. EVA initializes LoRA based on the SVD of layer input activations and achieves SOTA
performance due to its ability to adapt to the finetuning data. Pass <code>&apos;olora&apos;</code> to use OLoRA initialization.
Passing <code>&apos;pissa&apos;</code> results in the initialization of <a href="&#x2019;&lt;a" rel="nofollow">https://huggingface.co/papers/2404.02948&#x2019;</a>
<blockquote>
<p>Principal Singular values and Singular vectors Adaptation (PiSSA), which converges more rapidly than
LoRA and ultimately achieves superior performance. Moreover, PiSSA reduces the quantization error compared
to QLoRA, leading to further enhancements. Passing <code>&apos;pissa_niter_[number of iters]&apos;</code> initiates
Fast-SVD-based PiSSA initialization, where <code>[number of iters]</code> indicates the number of subspace iterations
to perform FSVD, and must be a nonnegative integer. When <code>[number of iters]</code> is set to 16, it can complete
the initialization of a 7B model within seconds, and the training effect is approximately equivalent to
using SVD. Passing <code>&apos;corda&apos;</code> results in the initialization of <a href="https://huggingface.co/papers/2406.05223">Context-Oriented Decomposition Adaptation</a>, which
converges even more rapidly than PiSSA in Instruction-Previewed Mode, and preserves world knowledge better
than LoRA in Knowledge-Preserved Mode. Passing <code>&quot;orthogonal&quot;</code> results in LoRA A and B being intialized
orthogonally; in this, it resembles <code>&quot;olora&quot;</code>, but the base weights are left untouched (requires <code>r</code> to be
even, only supported for linear layers for now).</p>
</blockquote>`,name:"init_lora_weights"},{anchor:"peft.LoraConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[List[int], int]</code>) &#x2014;
The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices
that are specified in this list. If a single integer is passed, it will apply the transformations on the
layer at this index.`,name:"layers_to_transform"},{anchor:"peft.LoraConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The layer pattern name, used only if <code>layers_to_transform</code> is different from <code>None</code>. This should target the
<code>nn.ModuleList</code> of the model, which is often called <code>&apos;layers&apos;</code> or <code>&apos;h&apos;</code>.`,name:"layers_pattern"},{anchor:"peft.LoraConfig.rank_pattern",description:`<strong>rank_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to ranks which are different from the default rank
specified by <code>r</code>. For example, <code>{&apos;^model.decoder.layers.0.encoder_attn.k_proj&apos;: 16}</code>.`,name:"rank_pattern"},{anchor:"peft.LoraConfig.alpha_pattern",description:`<strong>alpha_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to alphas which are different from the default alpha
specified by <code>lora_alpha</code>. For example, <code>{&apos;^model.decoder.layers.0.encoder_attn.k_proj&apos;: 16}</code>.`,name:"alpha_pattern"},{anchor:"peft.LoraConfig.megatron_config",description:`<strong>megatron_config</strong> (<code>Optional[dict]</code>) &#x2014;
The TransformerConfig arguments for Megatron. It is used to create LoRA&#x2019;s parallel linear layer. You can
get it like this, <code>core_transformer_config_from_args(get_args())</code>, these two functions being from Megatron.
The arguments will be used to initialize the TransformerConfig of Megatron. You need to specify this
parameter when you want to apply LoRA to the ColumnParallelLinear and RowParallelLinear layers of megatron.`,name:"megatron_config"},{anchor:"peft.LoraConfig.megatron_core",description:`<strong>megatron_core</strong> (<code>Optional[str]</code>) &#x2014;
The core module from Megatron to use, defaults to <code>&quot;megatron.core&quot;</code>.`,name:"megatron_core"},{anchor:"peft.LoraConfig.trainable_token_indices",description:`<strong>trainable_token_indices</strong> (<code>Optional[Union[List[int], dict[str, List[int]]]]</code>) &#x2014;
Lets you specify which token indices to selectively fine-tune without requiring to re-train the whole
embedding matrix using the <code>peft.TrainableTokensModel</code> method. You can specify token indices in two ways.
Either you specify a list of indices which will then target the model&#x2019;s input embedding layer (or, if not
found, <code>embed_tokens</code>). Alternatively, you can specify a dictionary where the key is the name of the
embedding module and the values are the list of token indices, e.g. <code>{&apos;embed_tokens&apos;: [0, 1, ...]}</code>. Note
that training with FSDP requires <code>use_orig_params=True</code> to avoid issues with non-uniform <code>requires_grad</code>.`,name:"trainable_token_indices"},{anchor:"peft.LoraConfig.loftq_config",description:`<strong>loftq_config</strong> (<code>Optional[LoftQConfig]</code>) &#x2014;
The configuration of LoftQ. If this is not None, then LoftQ will be used to quantize the backbone weights
and initialize Lora layers. Also pass <code>init_lora_weights=&apos;loftq&apos;</code>. Note that you should not pass a
quantized model in this case, as LoftQ will quantize the model itself.`,name:"loftq_config"},{anchor:"peft.LoraConfig.eva_config",description:`<strong>eva_config</strong> (<code>Optional[EvaConfig]</code>) &#x2014;
The configuration of EVA. At a minimum the dataset argument needs to be set (use the same dataset as for
finetuning).`,name:"eva_config"},{anchor:"peft.LoraConfig.corda_config",description:`<strong>corda_config</strong> (<code>Optional[CordaConfig]</code>) &#x2014;
The configuration of CorDA. If this is not None, then CorDA will be used to build the adapter layers. Also
pass <code>init_lora_weights=&apos;corda&apos;</code>.`,name:"corda_config"},{anchor:"peft.LoraConfig.use_dora",description:`<strong>use_dora</strong> (<code>bool</code>) &#x2014;
Enable &#x2018;Weight-Decomposed Low-Rank Adaptation&#x2019; (DoRA). This technique decomposes the updates of the weights
into two parts, magnitude and direction. Direction is handled by normal LoRA, whereas the magnitude is
handled by a separate learnable parameter. This can improve the performance of LoRA especially at low
ranks. Right now, DoRA only supports linear and Conv2D layers. DoRA introduces a bigger overhead than pure
LoRA, so it is recommended to merge weights for inference. For more information, see
<a href="https://huggingface.co/papers/2402.09353" rel="nofollow">https://huggingface.co/papers/2402.09353</a>.`,name:"use_dora"},{anchor:"peft.LoraConfig.alora_invocation_tokens",description:`<strong>alora_invocation_tokens</strong> (<code>List[int]</code>) &#x2014;
If not None, enable <a href="https://huggingface.co/papers/2504.12397">&#x2018;Activated LoRA&#x2019; (aLoRA)</a>, with
alora_invocation_tokens being the tokenized invocation string for the adapter (must be present in all model
input strings). This technique selectively activates the adapter weights only on tokens during and after
the alora_invocation_tokens. When used in a CausalLM, this means that the KV cache prior to invocation is
interchangeable with that of the base model (and other aLoRA adapters operating this way). As a result, in
inference pipelines involving switching between base model inference and adapter inference (e.g. agentic
pipelines, see paper for examples), significant savings are realized (relative to LoRA) by saving prefill
operations. Overall adapter inference speedups of an order of magnitude or more can occur on vLLM,
depending on the length of the shared context. Note that merging is not possible due to the selective
application of the weights.`,name:"alora_invocation_tokens"},{anchor:"peft.LoraConfig.layer_replication",description:`<strong>layer_replication</strong> (<code>List[Tuple[int, int]]</code>) &#x2014;
Build a new stack of layers by stacking the original model layers according to the ranges specified. This
allows expanding (or shrinking) the model without duplicating the base model weights. The new layers will
all have separate LoRA adapters attached to them.`,name:"layer_replication"},{anchor:"peft.LoraConfig.runtime_config",description:`<strong>runtime_config</strong> (<code>LoraRuntimeConfig</code>) &#x2014;
Runtime configurations (which are not saved or restored).`,name:"runtime_config"},{anchor:"peft.LoraConfig.lora_bias",description:`<strong>lora_bias</strong> (<code>bool</code>) &#x2014;
Defaults to <code>False</code>. Whether to enable the bias term for the LoRA B parameter. Typically, this should be
disabled. The main use case for this is when the LoRA weights were extracted from fully fine-tuned
parameters so the bias of those parameters can be taken into account.`,name:"lora_bias"},{anchor:"peft.LoraConfig.target_parameters",description:`<strong>target_parameters</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of parameter names or regex expression of the parameter names to replace with LoRA. This argument
behaves similarly to <code>target_modules</code>, except that the parameter name should be passed. Generally, you
should use <code>target_modules</code> to target the module (e.g. <code>nn.Linear</code>). However, in some circumstances, this
is not possible. E.g., in many mixture of expert (MoE) layers in HF Transformers, instead of using
<code>nn.Linear</code>, an <code>nn.Parameter</code> is used. PEFT normally overwrites the <code>forward</code> method for LoRA, but for
<code>nn.Parameter</code>, there is none. Therefore, to apply LoRA to that parameter, it needs to be targeted with
<code>target_parameters</code>. As an example, for Llama4, you can pass:
<code>target_parameters=[&apos;feed_forward.experts.gate_up_proj&apos;, &apos;feed_forward.experts.down_proj]</code>. Passing a
string for regex matching is not implemented yet.`,name:"target_parameters"},{anchor:"peft.LoraConfig.ensure_weight_tying",description:`<strong>ensure_weight_tying</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to tie weights or not after peft initialization. This will ensure that the adapters added to the
tied layers are also tied. This is only applicable for layers passed via <code>modules_to_save</code> and
<code>target_modules</code>.`,name:"ensure_weight_tying"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L323"}}),ee=new L({props:{name:"to_dict",anchor:"peft.LoraConfig.to_dict",parameters:[],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L774"}}),te=new M({props:{title:"LoraModel",local:"peft.LoraModel",headingTag:"h2"}}),ae=new L({props:{name:"class peft.LoraModel",anchor:"peft.LoraModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],parametersDescription:[{anchor:"peft.LoraModel.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to be adapted.",name:"model"},{anchor:"peft.LoraModel.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/lora#peft.LoraConfig">LoraConfig</a>) &#x2014; The configuration of the Lora model.',name:"config"},{anchor:"peft.LoraModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.LoraModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/model.py#L88",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The Lora model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}}),F=new $n({props:{anchor:"peft.LoraModel.example",$$slots:{default:[Dn]},$$scope:{ctx:Je}}}),G=new $n({props:{anchor:"peft.LoraModel.example-2",$$slots:{default:[En]},$$scope:{ctx:Je}}}),ne=new L({props:{name:"add_weighted_adapter",anchor:"peft.LoraModel.add_weighted_adapter",parameters:[{name:"adapters",val:": list[str]"},{name:"weights",val:": list[float]"},{name:"adapter_name",val:": str"},{name:"combination_type",val:": str = 'svd'"},{name:"svd_rank",val:": int | None = None"},{name:"svd_clamp",val:": int | None = None"},{name:"svd_full_matrices",val:": bool = True"},{name:"svd_driver",val:": str | None = None"},{name:"density",val:": float | None = None"},{name:"majority_sign_method",val:": Literal['total', 'frequency'] = 'total'"}],parametersDescription:[{anchor:"peft.LoraModel.add_weighted_adapter.adapters",description:`<strong>adapters</strong> (<code>list</code>) &#x2014;
List of adapter names to be merged.`,name:"adapters"},{anchor:"peft.LoraModel.add_weighted_adapter.weights",description:`<strong>weights</strong> (<code>list</code>) &#x2014;
List of weights for each adapter. Weights can be positive or negative, allowing for both addition and
subtraction of adapter effects.`,name:"weights"},{anchor:"peft.LoraModel.add_weighted_adapter.adapter_name",description:`<strong>adapter_name</strong> (<code>str</code>) &#x2014;
Name of the new adapter.`,name:"adapter_name"},{anchor:"peft.LoraModel.add_weighted_adapter.combination_type",description:`<strong>combination_type</strong> (<code>str</code>) &#x2014;
The merging type can be one of [<code>svd</code>, <code>linear</code>, <code>cat</code>, <code>ties</code>, <code>ties_svd</code>, <code>dare_ties</code>, <code>dare_linear</code>,
<code>dare_ties_svd</code>, <code>dare_linear_svd</code>, <code>magnitude_prune</code>, <code>magnitude_prune_svd</code>]. When using the <code>cat</code>
combination_type, the rank of the resulting adapter is equal to the sum of all adapters ranks (the
mixed adapter may be too big and result in OOM errors).`,name:"combination_type"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_rank",description:`<strong>svd_rank</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Rank of output adapter for svd. If None provided, will use max rank of merging adapters.`,name:"svd_rank"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_clamp",description:`<strong>svd_clamp</strong> (<code>float</code>, <em>optional</em>) &#x2014;
A quantile threshold for clamping SVD decomposition output. If None is provided, do not perform
clamping. Defaults to None.`,name:"svd_clamp"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_full_matrices",description:`<strong>svd_full_matrices</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Controls whether to compute the full or reduced SVD, and consequently, the shape of the returned
tensors U and Vh. Defaults to True.`,name:"svd_full_matrices"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_driver",description:`<strong>svd_driver</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Name of the cuSOLVER method to be used. This keyword argument only works when merging on CUDA. Can be
one of [None, <code>gesvd</code>, <code>gesvdj</code>, <code>gesvda</code>]. For more info please refer to <code>torch.linalg.svd</code>
documentation. Defaults to None.`,name:"svd_driver"},{anchor:"peft.LoraModel.add_weighted_adapter.density",description:`<strong>density</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Value between 0 and 1. 0 means all values are pruned and 1 means no values are pruned. Should be used
with [<code>ties</code>, <code>ties_svd</code>, <code>dare_ties</code>, <code>dare_linear</code>, <code>dare_ties_svd</code>, <code>dare_linear_svd</code>,
<code>magnintude_prune</code>, <code>magnitude_prune_svd</code>]`,name:"density"},{anchor:"peft.LoraModel.add_weighted_adapter.majority_sign_method",description:`<strong>majority_sign_method</strong> (<code>str</code>) &#x2014;
The method, should be one of [&#x201C;total&#x201D;, &#x201C;frequency&#x201D;], to use to get the magnitude of the sign values.
Should be used with [<code>ties</code>, <code>ties_svd</code>, <code>dare_ties</code>, <code>dare_ties_svd</code>]`,name:"majority_sign_method"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/model.py#L652"}}),oe=new L({props:{name:"subtract_mutated_init",anchor:"peft.LoraModel.subtract_mutated_init",parameters:[{name:"output_state_dict",val:": dict[str, torch.Tensor]"},{name:"adapter_name",val:": str"},{name:"kwargs",val:" = None"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/model.py#L909"}}),ie=new M({props:{title:"Utility",local:"utility",headingTag:"h2"}}),re=new M({props:{title:"ArrowConfig",local:"peft.ArrowConfig",headingTag:"h3"}}),se=new L({props:{name:"class peft.ArrowConfig",anchor:"peft.ArrowConfig",parameters:[{name:"top_k",val:": int = 3"},{name:"router_temperature",val:": float = 1.0"},{name:"use_gks",val:": bool = False"},{name:"rng_seed",val:": Optional[int] = None"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L74"}}),le=new M({props:{title:"LoftQ",local:"peft.replace_lora_weights_loftq",headingTag:"h3"}}),de=new L({props:{name:"peft.replace_lora_weights_loftq",anchor:"peft.replace_lora_weights_loftq",parameters:[{name:"peft_model",val:""},{name:"model_path",val:": Optional[str] = None"},{name:"adapter_name",val:": str = 'default'"},{name:"callback",val:": Optional[Callable[[torch.nn.Module, str], bool]] = None"}],parametersDescription:[{anchor:"peft.replace_lora_weights_loftq.peft_model",description:`<strong>peft_model</strong> (<code>PeftModel</code>) &#x2014;
The model to replace the weights of. Must be a quantized PEFT model with LoRA layers.`,name:"peft_model"},{anchor:"peft.replace_lora_weights_loftq.model_path",description:`<strong>model_path</strong> (<code>Optional[str]</code>) &#x2014;
The path to the model safetensors file. If the model is a Hugging Face model, this will be inferred from
the model&#x2019;s config. Otherwise, it must be provided.`,name:"model_path"},{anchor:"peft.replace_lora_weights_loftq.adapter_name",description:`<strong>adapter_name</strong> (<code>str</code>) &#x2014;
The name of the adapter to replace the weights of. The default adapter name is &#x201C;default&#x201D;.`,name:"adapter_name"},{anchor:"peft.replace_lora_weights_loftq.callback",description:`<strong>callback</strong> (<code>Optional[Callable[[PeftModel, str], bool]]</code>) &#x2014;
A callback function that will be called after each module is replaced. The callback function should take
the model and the name of the current module as input and return a boolean indicating whether the
replacement should be kept. If the callback returns False, the replacement will be rolled back. This can be
very useful to confirm that the LoftQ initialization actually decreases the quantization error of the
model. As an example, this callback could generate logits for given input and compare it with the logits
from the original, non-quanitzed model with the same input, and only return <code>True</code> if there is an
improvement. As this is a greedy optimization, it&#x2019;s possible that calling this function multiple times
yields incremental improvements.`,name:"callback"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/utils/loftq_utils.py#L353"}}),pe=new M({props:{title:"Eva",local:"eva",headingTag:"h3"}}),ce=new M({props:{title:"EvaConfig",local:"peft.EvaConfig",headingTag:"h4"}}),me=new L({props:{name:"class peft.EvaConfig",anchor:"peft.EvaConfig",parameters:[{name:"rho",val:": float = 2.0"},{name:"tau",val:": float = 0.99"},{name:"use_label_mask",val:": bool = True"},{name:"label_mask_value",val:": int = -100"},{name:"whiten",val:": bool = False"},{name:"adjust_scaling_factors",val:": bool = True"}],parametersDescription:[{anchor:"peft.EvaConfig.rho",description:`<strong>rho</strong> (<code>float</code>) &#x2014;
Rho value for EVA redistribution (&gt;= 1.0). The maximum rank for a layer is lora_r * rho. Default is 2.0,
meaning the maximum rank allowed for a layer is 2r. Increasing rho will allow for a higher degree of
redistribution of ranks across layers. Some pre-trained models might be more sensitive to a rank
redistribution. It can therefore be beneficial to try rho=1.0 (no redistribution) if the performance is
lower than expected.`,name:"rho"},{anchor:"peft.EvaConfig.tau",description:`<strong>tau</strong> (<code>float</code>) &#x2014;
Cosine similarity threshold for early stopping. Compares the cosine similarity of right-singular vectors
between two consecutive SVD steps. If the cosine similarity is above this threshold, the SVD iteration is
stopped. Default is 0.99.`,name:"tau"},{anchor:"peft.EvaConfig.use_label_mask",description:`<strong>use_label_mask</strong> (<code>bool</code>) &#x2014;
Use label mask for EVA initialization. This means that positions where labels=label_mask_value are ignored
for the SVD computation. Setting use_label_mask=True is preferred in most cases and can be especially
beneficial for multi-turn conversations. The default value is True. Filtering out items based on the label
mask can sometimes lead to a small batch size and as a result instabilities in the SVD computation. For
cases where a large share of batch items would be filtered out, set use_label_mask=False.`,name:"use_label_mask"},{anchor:"peft.EvaConfig.label_mask_value",description:`<strong>label_mask_value</strong> (<code>int</code>) &#x2014;
If use_label_mask=True the value to look for to mask out ignored tokens. Default is -100.`,name:"label_mask_value"},{anchor:"peft.EvaConfig.whiten",description:`<strong>whiten</strong> (<code>bool</code>) &#x2014; Apply whitening to singular vectors. Default is False.
Whitening has been shown to be beneficial for EVA in the vision domain.`,name:"whiten"},{anchor:"peft.EvaConfig.adjust_scaling_factors",description:`<strong>adjust_scaling_factors</strong> (<code>bool</code>) &#x2014;
Adjust LoRA scaling factors after the rank redistribution. Setting this to True means the scaling factors
are adjusted so that all LoRA gradients have the same scale regardless of their rank. Default is True.`,name:"adjust_scaling_factors"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L196"}}),ge=new M({props:{title:"initialize_lora_eva_weights",local:"peft.initialize_lora_eva_weights",headingTag:"h4"}}),fe=new L({props:{name:"peft.initialize_lora_eva_weights",anchor:"peft.initialize_lora_eva_weights",parameters:[{name:"model",val:": Module"},{name:"dataloader",val:": typing.Optional[collections.abc.Iterable] = None"},{name:"eva_state_dict",val:": typing.Optional[dict] = None"},{name:"forward_fn",val:": typing.Optional[collections.abc.Callable] = <function forward_fn_dict at 0x7eff8db54550>"},{name:"prepare_model_inputs_fn",val:": typing.Optional[collections.abc.Callable] = <function prepare_model_inputs_fn_language_modeling at 0x7eff8db54430>"},{name:"prepare_layer_inputs_fn",val:": typing.Union[collections.abc.Callable, dict[str, collections.abc.Callable], NoneType] = <function prepare_layer_inputs_fn_language_modeling at 0x7eff8db544c0>"},{name:"adapter_name",val:": str = 'default'"},{name:"gather_distributed_inputs",val:": bool = True"},{name:"show_progress_bar",val:": bool = True"}],parametersDescription:[{anchor:"peft.initialize_lora_eva_weights.model",description:"<strong>model</strong> (PeftModel) &#x2014; The peft model to compute the SVD for.",name:"model"},{anchor:"peft.initialize_lora_eva_weights.dataloader",description:`<strong>dataloader</strong> (Optional[Iterable]) &#x2014;
The dataloader to use for the forward pass. If None, eva_state_dict needs to be provided.`,name:"dataloader"},{anchor:"peft.initialize_lora_eva_weights.eva_state_dict",description:`<strong>eva_state_dict</strong> (Optional[dict]) &#x2014;
The state_dict to load into the model. If None, a dataloader needs to be provided and the state_dict will
be computed using <code>get_eva_state_dict</code>.`,name:"eva_state_dict"},{anchor:"peft.initialize_lora_eva_weights.forward_fn",description:`<strong>forward_fn</strong> (Callable) &#x2014;
The forward function to use for the forward pass. Takes two arguments: <code>model</code> and <code>inputs</code>. Default
behavior is <code>return model(**inputs)</code>`,name:"forward_fn"},{anchor:"peft.initialize_lora_eva_weights.prepare_model_inputs_fn",description:`<strong>prepare_model_inputs_fn</strong> (Optional[Callable]) &#x2014;
This function receives the model inputs and the peft_config and passes the output to
<code>prepare_layer_inputs_fn</code>. Can be used to modify the input to the SVD computation based on the original
model inputs. For example for language modeling the attention mask is used to determine which indices are
padding tokens and should not be used for SVD. Any function defined here expects two arguments:
<code>model_input</code> and <code>peft_config</code>. <code>peft.tuners.lora.eva.prepare_model_inputs_fn_language_modeling</code> is used
by default.`,name:"prepare_model_inputs_fn"},{anchor:"peft.initialize_lora_eva_weights.prepare_layer_inputs_fn",description:`<strong>prepare_layer_inputs_fn</strong> (Union[Callable, Dict[str, Callable], None]) &#x2014;
This function receives the layer inputs, the model inputs (potentially modified by
<code>prepare_model_inputs_fn</code>) and the name of the layer and returns the inputs that should be used for SVD for
that particular layer. Any custom function defined here expects three arguments: <code>layer_input</code>,
<code>model_input</code>, and <code>layer_name</code> and should return a 2d tensor. The default logic can be found in
peft.tuners.lora.eva.prepare_layer_inputs_fn_language_modeling and works for language modeling. In this
case model_inputs is the mask used to determine which indices should be used for SVD (created by
<code>prepare_model_inputs_fn_language_modeling</code>).`,name:"prepare_layer_inputs_fn"},{anchor:"peft.initialize_lora_eva_weights.adapter_name",description:"<strong>adapter_name</strong> (str) &#x2014; The name of the adapter to initialize the weights for.",name:"adapter_name"},{anchor:"peft.initialize_lora_eva_weights.gather_distributed_inputs",description:`<strong>gather_distributed_inputs</strong> (bool) &#x2014;
Whether to gather the layer inputs from all ranks. Default is True meaning in a distributed setting the
layer inputs will be gathered from all ranks for the SVD computation. For non-distributed settings this
argument is ignored. Set to False if you are using a non-distributed dataloader in a distributed setting.`,name:"gather_distributed_inputs"},{anchor:"peft.initialize_lora_eva_weights.show_progress_bar",description:"<strong>show_progress_bar</strong> (bool) &#x2014; Whether to show a progress bar. Default is True.",name:"show_progress_bar"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/eva.py#L654",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The model with the initialized LoRA weights.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>model (torch.nn.Module)</p>
`}}),he=new M({props:{title:"get_eva_state_dict",local:"peft.get_eva_state_dict",headingTag:"h4"}}),ue=new L({props:{name:"peft.get_eva_state_dict",anchor:"peft.get_eva_state_dict",parameters:[{name:"model",val:": Module"},{name:"dataloader",val:": Iterable"},{name:"peft_config",val:": typing.Optional[peft.tuners.lora.config.LoraConfig] = None"},{name:"forward_fn",val:": typing.Optional[collections.abc.Callable] = <function forward_fn_dict at 0x7eff8db54550>"},{name:"prepare_model_inputs_fn",val:": typing.Optional[collections.abc.Callable] = <function prepare_model_inputs_fn_language_modeling at 0x7eff8db54430>"},{name:"prepare_layer_inputs_fn",val:": typing.Union[collections.abc.Callable, dict[str, collections.abc.Callable], NoneType] = <function prepare_layer_inputs_fn_language_modeling at 0x7eff8db544c0>"},{name:"adapter_name",val:": str = 'default'"},{name:"gather_distributed_inputs",val:": bool = True"},{name:"show_progress_bar",val:": bool = True"}],parametersDescription:[{anchor:"peft.get_eva_state_dict.model",description:"<strong>model</strong> (torch.nn.Module) &#x2014; The model to compute the SVD for. Does not need to be a PeftModel.",name:"model"},{anchor:"peft.get_eva_state_dict.dataloader",description:"<strong>dataloader</strong> (Iterable) &#x2014; The dataloader to use for the forward pass.",name:"dataloader"},{anchor:"peft.get_eva_state_dict.peft_config",description:`<strong>peft_config</strong> (Optional[LoraConfig]) &#x2014;
The configuration for the LoRA layers. Only required if <code>model</code> is not a PeftModel.`,name:"peft_config"},{anchor:"peft.get_eva_state_dict.forward_fn",description:`<strong>forward_fn</strong> (Callable) &#x2014;
The forward function to use for the forward pass. Takes two arguments: <code>model</code> and <code>inputs</code>. Default
behavior is <code>return model(**inputs)</code>`,name:"forward_fn"},{anchor:"peft.get_eva_state_dict.prepare_model_inputs_fn",description:`<strong>prepare_model_inputs_fn</strong> (Optional[Callable]) &#x2014;
This function receives the model inputs and the peft_config and passes the output to
<code>prepare_layer_inputs_fn</code>. Can be used to modify the input to the SVD computation based on the original
model inputs. For example for language modeling the attention mask is used to determine which indices are
padding tokens and should not be used for SVD. Any function defined here expects two arguments:
<code>model_input</code> and <code>peft_config</code>. <code>peft.tuners.lora.eva.prepare_model_inputs_fn_language_modeling</code> is used
by default.`,name:"prepare_model_inputs_fn"},{anchor:"peft.get_eva_state_dict.prepare_layer_inputs_fn",description:`<strong>prepare_layer_inputs_fn</strong> (Union[Callable, Dict[str, Callable], None]) &#x2014;
This function receives the layer inputs, the model inputs (potentially modified by
<code>prepare_model_inputs_fn</code>) and the name of the layer and returns the inputs that should be used for SVD for
that particular layer. Any custom function defined here expects three arguments: <code>layer_input</code>,
<code>model_input</code>, and <code>layer_name</code> and should return a 2d tensor. The default logic can be found in
peft.tuners.lora.eva.prepare_layer_inputs_fn_language_modeling and works for language modeling. In this
case model_inputs is the mask used to determine which indices should be used for SVD (created by
<code>prepare_model_inputs_fn_language_modeling</code>).`,name:"prepare_layer_inputs_fn"},{anchor:"peft.get_eva_state_dict.adapter_name",description:"<strong>adapter_name</strong> (str) &#x2014; The name of the adapter to compute the SVD for.",name:"adapter_name"},{anchor:"peft.get_eva_state_dict.gather_distributed_inputs",description:`<strong>gather_distributed_inputs</strong> (bool) &#x2014;
Whether to gather the layer inputs from all ranks. Default is True meaning in a distributed setting the
layer inputs will be gathered from all ranks for the SVD computation. For non-distributed settings this
argument is ignored. Set to False if you are using a non-distributed dataloader in a distributed setting.`,name:"gather_distributed_inputs"},{anchor:"peft.get_eva_state_dict.show_progress_bar",description:"<strong>show_progress_bar</strong> (bool) &#x2014; Whether to show a progress bar. Default is True.",name:"show_progress_bar"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/eva.py#L556",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The state dictionary containing the SVD components for each layer.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>eva_state_dict (dict)</p>
`}}),_e=new M({props:{title:"Variants",local:"variants",headingTag:"h2"}}),be=new M({props:{title:"LoRA-GA",local:"lora-ga",headingTag:"h3"}}),xe=new M({props:{title:"Usage Tips",local:"usage-tips",headingTag:"h4"}}),$e=new M({props:{title:"LoraGAConfig",local:"peft.LoraGAConfig",headingTag:"h4"}}),Ce=new L({props:{name:"class peft.LoraGAConfig",anchor:"peft.LoraGAConfig",parameters:[{name:"direction",val:": Literal['ArBr', 'A2rBr', 'ArB2r', 'random'] = 'ArB2r'"},{name:"scale",val:": Literal['stable', 'weight_svd', 'gd_scale', 'unit'] = 'stable'"},{name:"stable_gamma",val:": int = 16"}],parametersDescription:[{anchor:"peft.LoraGAConfig.direction",description:`<strong>direction</strong> (<code>Literal[&quot;ArBr&quot;, &quot;A2rBr&quot;, &quot;ArB2r&quot;, &quot;random&quot;]</code>) &#x2014;
Strategy for distributing gradient SVD components to lora_A and lora_B matrices.
<ul>
<li>&#x201C;ArBr&#x201D;: Alternating indices (A takes odd, B takes even)</li>
<li>&#x201C;A2rBr&#x201D;: A takes indices [r:2r], B takes indices [:r]</li>
<li>&#x201C;ArB2r&#x201D;: A takes indices [:r], B takes indices [r:2r] (recommended)</li>
<li>&#x201C;random&#x201D;: Random selection of indices
Default: &#x201C;ArB2r&#x201D;</li>
</ul>`,name:"direction"},{anchor:"peft.LoraGAConfig.scale",description:`<strong>scale</strong> (<code>Literal[&quot;stable&quot;, &quot;weight_svd&quot;, &quot;gd_scale&quot;, &quot;unit&quot;]</code>) &#x2014;
Scaling strategy for adapter initialization.
<ul>
<li>&#x201C;stable&#x201D;: Stable scaling with gamma parameter</li>
<li>&#x201C;weight_svd&#x201D;: Scale based on weight matrix singular values</li>
<li>&#x201C;gd_scale&#x201D;: Gradient descent based scaling</li>
<li>&#x201C;unit&#x201D;: No additional scaling
Default: &#x201C;stable&#x201D;</li>
</ul>`,name:"scale"},{anchor:"peft.LoraGAConfig.stable_gamma",description:`<strong>stable_gamma</strong> (<code>int</code>) &#x2014;
Gamma parameter for stable scaling method. Default: 16`,name:"stable_gamma"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L898"}}),Le=new M({props:{title:"Utilities",local:"peft.tuners.lora.loraga.estimate_gradients",headingTag:"h4"}}),ke=new L({props:{name:"peft.tuners.lora.loraga.estimate_gradients",anchor:"peft.tuners.lora.loraga.estimate_gradients",parameters:[{name:"model",val:": Module"},{name:"lora_config",val:": LoraConfig"},{name:"train_step",val:": Callable"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/loraga.py#L118"}}),Ae=new L({props:{name:"peft.preprocess_loraga",anchor:"peft.preprocess_loraga",parameters:[{name:"model",val:": Module"},{name:"lora_config",val:": LoraConfig"},{name:"train_step",val:": Callable"},{name:"cache_file",val:": typing.Optional[str] = None"}],parametersDescription:[{anchor:"peft.preprocess_loraga.model",description:`<strong>model</strong> (<code>nn.Module</code>) &#x2014;
Model to preprocess.`,name:"model"},{anchor:"peft.preprocess_loraga.lora_config",description:`<strong>lora_config</strong> (<code>LoraConfig</code>) &#x2014;
Lora configuration of the model. <code>lora_config.lora_ga_config</code> should be set.`,name:"lora_config"},{anchor:"peft.preprocess_loraga.train_step",description:`<strong>train_step</strong> (<code>Callable[[], None]</code>) &#x2014;
Callback to run gradient estimation. Typically you should run model forward and backward passes in this
callback. The gradients will be accumulated across all calls within this callback.`,name:"train_step"},{anchor:"peft.preprocess_loraga.cache_file",description:`<strong>cache_file</strong> (<code>Optional[str]</code>) &#x2014;
Optional path to cache file for saving/loading gradients. If provided and the file exists, gradients will
be loaded from cache. Otherwise, gradients will be estimated and saved to this path.`,name:"cache_file"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/loraga.py#L46"}}),je=new M({props:{title:"Intruder Dimension Reduction",local:"peft.tuners.lora.intruders.reduce_intruder_dimension",headingTag:"h2"}}),qe=new L({props:{name:"peft.tuners.lora.intruders.reduce_intruder_dimension",anchor:"peft.tuners.lora.intruders.reduce_intruder_dimension",parameters:[{name:"peft_model",val:""},{name:"old_adapter_name",val:" = 'default'"},{name:"new_adapter_name",val:" = 'intruder_reduced'"},{name:"top_k",val:" = 10"},{name:"threshold_epsilon",val:" = 0.5"},{name:"mitigation_lambda",val:" = 0.75"},{name:"logging_sink",val:" = <built-in function print>"}],parametersDescription:[{anchor:"peft.tuners.lora.intruders.reduce_intruder_dimension.peft_model",description:`<strong>peft_model</strong> &#x2014;
The PEFT model with a loaded LoRA adapter with the name provided in <code>old_adapter_name</code>. Currently mixed
models are not supported.`,name:"peft_model"},{anchor:"peft.tuners.lora.intruders.reduce_intruder_dimension.top_k",description:`<strong>top_k</strong> (default &#x2014; 10)
Consider the top-k dimensions for intruder detection. The larger the value, the more dimensions will be
considered for intruder detection analysis (and the more false-postiives there can be). Operates on the
cosine similarity between base weights and adapter weights roughly sorted by influence of dimension
(determined by singular value decomposition), so a top-k of 10 will look at the 10 most &#x2018;important&#x2019;
dimensions.`,name:"top_k"},{anchor:"peft.tuners.lora.intruders.reduce_intruder_dimension.threshold_epsilon",description:`<strong>threshold_epsilon</strong> (default &#x2014; 0.5)
Threshold value when to consider a cosine similarity between base weight and adapter weight as intruder.
According to the paper, intruder dimensions show near-zero absolute cosine similarity with pre-trained
singular vectors. The lower this value, the less potential intruder dimensions are identified. The higher
the value, the more potential false-positives are considered as intruders.`,name:"threshold_epsilon"},{anchor:"peft.tuners.lora.intruders.reduce_intruder_dimension.mitigation_lambda",description:`<strong>mitigation_lambda</strong> (default &#x2014; 0.75)
The relative portion of the intruder dimensions that is subtracted from the adapter&#x2019;s delta weight. The
higher the value the more of the intruder dimension is subtracted but the more information is lost. Refer
to Figure 8 in the paper for a trade-off analysis.`,name:"mitigation_lambda"},{anchor:"peft.tuners.lora.intruders.reduce_intruder_dimension.logging_sink",description:`<strong>logging_sink</strong> (default &#x2014; print)
Function that prints information about the mitigation process. Set to None if you don&#x2019;t want any output.`,name:"logging_sink"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/intruders.py#L20"}}),Re=new Jn({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/lora.md"}}),{c(){u=s("meta"),j=n(),b=s("p"),C=n(),p(q.$$.fragment),v=n(),p(R.$$.fragment),_t=n(),H=s("p"),H.innerHTML=Sa,bt=n(),Q=s("p"),Q.textContent=Na,vt=n(),X=s("p"),X.innerHTML=Va,yt=n(),p(Y.$$.fragment),wt=n(),U=s("div"),p(K.$$.fragment),na=n(),De=s("p"),De.innerHTML=Ba,oa=n(),P=s("div"),p(ee.$$.fragment),ia=n(),Ee=s("p"),Ee.textContent=Pa,xt=n(),p(te.$$.fragment),Mt=n(),_=s("div"),p(ae.$$.fragment),ra=n(),Ie=s("p"),Ie.textContent=Fa,sa=n(),ze=s("p"),ze.innerHTML=Ga,la=n(),p(F.$$.fragment),da=n(),p(G.$$.fragment),pa=n(),Se=s("p"),Se.innerHTML=Za,ca=n(),Ne=s("ul"),Ne.innerHTML=Oa,ma=n(),I=s("div"),p(ne.$$.fragment),ga=n(),Ve=s("p"),Ve.textContent=Wa,fa=n(),Be=s("p"),Be.innerHTML=Ha,ha=n(),Z=s("div"),p(oe.$$.fragment),ua=n(),Pe=s("p"),Pe.innerHTML=Qa,Tt=n(),p(ie.$$.fragment),$t=n(),p(re.$$.fragment),Ct=n(),N=s("div"),p(se.$$.fragment),_a=n(),Fe=s("p"),Fe.innerHTML=Xa,Lt=n(),p(le.$$.fragment),kt=n(),T=s("div"),p(de.$$.fragment),ba=n(),Ge=s("p"),Ge.textContent=Ya,va=n(),Ze=s("p"),Ze.textContent=Ka,ya=n(),Oe=s("p"),Oe.textContent=en,wa=n(),We=s("p"),We.textContent=tn,At=n(),p(pe.$$.fragment),jt=n(),p(ce.$$.fragment),qt=n(),V=s("div"),p(me.$$.fragment),xa=n(),He=s("p"),He.innerHTML=an,Rt=n(),p(ge.$$.fragment),Ut=n(),J=s("div"),p(fe.$$.fragment),Ma=n(),Qe=s("p"),Qe.textContent=nn,Ta=n(),Xe=s("p"),Xe.textContent=on,Jt=n(),p(he.$$.fragment),Dt=n(),D=s("div"),p(ue.$$.fragment),$a=n(),Ye=s("p"),Ye.textContent=rn,Ca=n(),Ke=s("p"),Ke.textContent=sn,Et=n(),p(_e.$$.fragment),It=n(),p(be.$$.fragment),zt=n(),ve=s("p"),ve.innerHTML=ln,St=n(),ye=s("p"),ye.textContent=dn,Nt=n(),we=s("p"),we.innerHTML=pn,Vt=n(),p(xe.$$.fragment),Bt=n(),Me=s("ul"),Me.innerHTML=cn,Pt=n(),Te=s("ul"),Te.innerHTML=mn,Ft=n(),p($e.$$.fragment),Gt=n(),k=s("div"),p(Ce.$$.fragment),La=n(),et=s("p"),et.textContent=gn,ka=n(),tt=s("p"),tt.textContent=fn,Aa=n(),at=s("p"),at.innerHTML=hn,Zt=n(),p(Le.$$.fragment),Ot=n(),E=s("div"),p(ke.$$.fragment),ja=n(),nt=s("p"),nt.textContent=un,qa=n(),ot=s("p"),ot.textContent=_n,Wt=n(),A=s("div"),p(Ae.$$.fragment),Ra=n(),it=s("p"),it.textContent=bn,Ua=n(),rt=s("p"),rt.textContent=vn,Ja=n(),st=s("p"),st.innerHTML=yn,Ht=n(),p(je.$$.fragment),Qt=n(),$=s("div"),p(qe.$$.fragment),Da=n(),lt=s("p"),lt.innerHTML=wn,Ea=n(),dt=s("p"),dt.innerHTML=xn,Ia=n(),pt=s("p"),pt.innerHTML=Mn,za=n(),ct=s("p"),ct.textContent=Tn,Xt=n(),p(Re.$$.fragment),Yt=n(),ut=s("p"),this.h()},l(e){const t=Rn("svelte-u9bgzb",document.head);u=l(t,"META",{name:!0,content:!0}),t.forEach(a),j=o(e),b=l(e,"P",{}),w(b).forEach(a),C=o(e),c(q.$$.fragment,e),v=o(e),c(R.$$.fragment,e),_t=o(e),H=l(e,"P",{"data-svelte-h":!0}),d(H)!=="svelte-e6qy8f"&&(H.innerHTML=Sa),bt=o(e),Q=l(e,"P",{"data-svelte-h":!0}),d(Q)!=="svelte-1cwsb16"&&(Q.textContent=Na),vt=o(e),X=l(e,"P",{"data-svelte-h":!0}),d(X)!=="svelte-p2y1b1"&&(X.innerHTML=Va),yt=o(e),c(Y.$$.fragment,e),wt=o(e),U=l(e,"DIV",{class:!0});var B=w(U);c(K.$$.fragment,B),na=o(B),De=l(B,"P",{"data-svelte-h":!0}),d(De)!=="svelte-o3o25r"&&(De.innerHTML=Ba),oa=o(B),P=l(B,"DIV",{class:!0});var Ue=w(P);c(ee.$$.fragment,Ue),ia=o(Ue),Ee=l(Ue,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-1nggz95"&&(Ee.textContent=Pa),Ue.forEach(a),B.forEach(a),xt=o(e),c(te.$$.fragment,e),Mt=o(e),_=l(e,"DIV",{class:!0});var y=w(_);c(ae.$$.fragment,y),ra=o(y),Ie=l(y,"P",{"data-svelte-h":!0}),d(Ie)!=="svelte-kipsu2"&&(Ie.textContent=Fa),sa=o(y),ze=l(y,"P",{"data-svelte-h":!0}),d(ze)!=="svelte-wrg681"&&(ze.innerHTML=Ga),la=o(y),c(F.$$.fragment,y),da=o(y),c(G.$$.fragment,y),pa=o(y),Se=l(y,"P",{"data-svelte-h":!0}),d(Se)!=="svelte-1xx6nm4"&&(Se.innerHTML=Za),ca=o(y),Ne=l(y,"UL",{"data-svelte-h":!0}),d(Ne)!=="svelte-3vno46"&&(Ne.innerHTML=Oa),ma=o(y),I=l(y,"DIV",{class:!0});var mt=w(I);c(ne.$$.fragment,mt),ga=o(mt),Ve=l(mt,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-lg9b7q"&&(Ve.textContent=Wa),fa=o(mt),Be=l(mt,"P",{"data-svelte-h":!0}),d(Be)!=="svelte-gmuli4"&&(Be.innerHTML=Ha),mt.forEach(a),ha=o(y),Z=l(y,"DIV",{class:!0});var ea=w(Z);c(oe.$$.fragment,ea),ua=o(ea),Pe=l(ea,"P",{"data-svelte-h":!0}),d(Pe)!=="svelte-1rwd503"&&(Pe.innerHTML=Qa),ea.forEach(a),y.forEach(a),Tt=o(e),c(ie.$$.fragment,e),$t=o(e),c(re.$$.fragment,e),Ct=o(e),N=l(e,"DIV",{class:!0});var ta=w(N);c(se.$$.fragment,ta),_a=o(ta),Fe=l(ta,"P",{"data-svelte-h":!0}),d(Fe)!=="svelte-n43q2i"&&(Fe.innerHTML=Xa),ta.forEach(a),Lt=o(e),c(le.$$.fragment,e),kt=o(e),T=l(e,"DIV",{class:!0});var z=w(T);c(de.$$.fragment,z),ba=o(z),Ge=l(z,"P",{"data-svelte-h":!0}),d(Ge)!=="svelte-3ipm59"&&(Ge.textContent=Ya),va=o(z),Ze=l(z,"P",{"data-svelte-h":!0}),d(Ze)!=="svelte-1c9djg"&&(Ze.textContent=Ka),ya=o(z),Oe=l(z,"P",{"data-svelte-h":!0}),d(Oe)!=="svelte-p9czkf"&&(Oe.textContent=en),wa=o(z),We=l(z,"P",{"data-svelte-h":!0}),d(We)!=="svelte-1d7vg8p"&&(We.textContent=tn),z.forEach(a),At=o(e),c(pe.$$.fragment,e),jt=o(e),c(ce.$$.fragment,e),qt=o(e),V=l(e,"DIV",{class:!0});var aa=w(V);c(me.$$.fragment,aa),xa=o(aa),He=l(aa,"P",{"data-svelte-h":!0}),d(He)!=="svelte-98ps9a"&&(He.innerHTML=an),aa.forEach(a),Rt=o(e),c(ge.$$.fragment,e),Ut=o(e),J=l(e,"DIV",{class:!0});var gt=w(J);c(fe.$$.fragment,gt),Ma=o(gt),Qe=l(gt,"P",{"data-svelte-h":!0}),d(Qe)!=="svelte-1lcy19c"&&(Qe.textContent=nn),Ta=o(gt),Xe=l(gt,"P",{"data-svelte-h":!0}),d(Xe)!=="svelte-18makdi"&&(Xe.textContent=on),gt.forEach(a),Jt=o(e),c(he.$$.fragment,e),Dt=o(e),D=l(e,"DIV",{class:!0});var ft=w(D);c(ue.$$.fragment,ft),$a=o(ft),Ye=l(ft,"P",{"data-svelte-h":!0}),d(Ye)!=="svelte-tzfhs9"&&(Ye.textContent=rn),Ca=o(ft),Ke=l(ft,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-1txgnvp"&&(Ke.textContent=sn),ft.forEach(a),Et=o(e),c(_e.$$.fragment,e),It=o(e),c(be.$$.fragment,e),zt=o(e),ve=l(e,"P",{"data-svelte-h":!0}),d(ve)!=="svelte-4gozko"&&(ve.innerHTML=ln),St=o(e),ye=l(e,"P",{"data-svelte-h":!0}),d(ye)!=="svelte-1cwsb16"&&(ye.textContent=dn),Nt=o(e),we=l(e,"P",{"data-svelte-h":!0}),d(we)!=="svelte-61kkev"&&(we.innerHTML=pn),Vt=o(e),c(xe.$$.fragment,e),Bt=o(e),Me=l(e,"UL",{"data-svelte-h":!0}),d(Me)!=="svelte-1xyepfa"&&(Me.innerHTML=cn),Pt=o(e),Te=l(e,"UL",{"data-svelte-h":!0}),d(Te)!=="svelte-15ewtlw"&&(Te.innerHTML=mn),Ft=o(e),c($e.$$.fragment,e),Gt=o(e),k=l(e,"DIV",{class:!0});var O=w(k);c(Ce.$$.fragment,O),La=o(O),et=l(O,"P",{"data-svelte-h":!0}),d(et)!=="svelte-f3tcbz"&&(et.textContent=gn),ka=o(O),tt=l(O,"P",{"data-svelte-h":!0}),d(tt)!=="svelte-1m0xozu"&&(tt.textContent=fn),Aa=o(O),at=l(O,"P",{"data-svelte-h":!0}),d(at)!=="svelte-1snkeam"&&(at.innerHTML=hn),O.forEach(a),Zt=o(e),c(Le.$$.fragment,e),Ot=o(e),E=l(e,"DIV",{class:!0});var ht=w(E);c(ke.$$.fragment,ht),ja=o(ht),nt=l(ht,"P",{"data-svelte-h":!0}),d(nt)!=="svelte-1vwmdin"&&(nt.textContent=un),qa=o(ht),ot=l(ht,"P",{"data-svelte-h":!0}),d(ot)!=="svelte-1u52z7c"&&(ot.textContent=_n),ht.forEach(a),Wt=o(e),A=l(e,"DIV",{class:!0});var W=w(A);c(Ae.$$.fragment,W),Ra=o(W),it=l(W,"P",{"data-svelte-h":!0}),d(it)!=="svelte-1olizyr"&&(it.textContent=bn),Ua=o(W),rt=l(W,"P",{"data-svelte-h":!0}),d(rt)!=="svelte-xytmil"&&(rt.textContent=vn),Ja=o(W),st=l(W,"P",{"data-svelte-h":!0}),d(st)!=="svelte-itlk2i"&&(st.innerHTML=yn),W.forEach(a),Ht=o(e),c(je.$$.fragment,e),Qt=o(e),$=l(e,"DIV",{class:!0});var S=w($);c(qe.$$.fragment,S),Da=o(S),lt=l(S,"P",{"data-svelte-h":!0}),d(lt)!=="svelte-1p6q28m"&&(lt.innerHTML=wn),Ea=o(S),dt=l(S,"P",{"data-svelte-h":!0}),d(dt)!=="svelte-1kvdxh9"&&(dt.innerHTML=xn),Ia=o(S),pt=l(S,"P",{"data-svelte-h":!0}),d(pt)!=="svelte-1443q3v"&&(pt.innerHTML=Mn),za=o(S),ct=l(S,"P",{"data-svelte-h":!0}),d(ct)!=="svelte-k6g2gm"&&(ct.textContent=Tn),S.forEach(a),Xt=o(e),c(Re.$$.fragment,e),Yt=o(e),ut=l(e,"P",{}),w(ut).forEach(a),this.h()},h(){x(u,"name","hf:doc:metadata"),x(u,"content",zn),x(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),x($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){i(document.head,u),r(e,j,t),r(e,b,t),r(e,C,t),m(q,e,t),r(e,v,t),m(R,e,t),r(e,_t,t),r(e,H,t),r(e,bt,t),r(e,Q,t),r(e,vt,t),r(e,X,t),r(e,yt,t),m(Y,e,t),r(e,wt,t),r(e,U,t),m(K,U,null),i(U,na),i(U,De),i(U,oa),i(U,P),m(ee,P,null),i(P,ia),i(P,Ee),r(e,xt,t),m(te,e,t),r(e,Mt,t),r(e,_,t),m(ae,_,null),i(_,ra),i(_,Ie),i(_,sa),i(_,ze),i(_,la),m(F,_,null),i(_,da),m(G,_,null),i(_,pa),i(_,Se),i(_,ca),i(_,Ne),i(_,ma),i(_,I),m(ne,I,null),i(I,ga),i(I,Ve),i(I,fa),i(I,Be),i(_,ha),i(_,Z),m(oe,Z,null),i(Z,ua),i(Z,Pe),r(e,Tt,t),m(ie,e,t),r(e,$t,t),m(re,e,t),r(e,Ct,t),r(e,N,t),m(se,N,null),i(N,_a),i(N,Fe),r(e,Lt,t),m(le,e,t),r(e,kt,t),r(e,T,t),m(de,T,null),i(T,ba),i(T,Ge),i(T,va),i(T,Ze),i(T,ya),i(T,Oe),i(T,wa),i(T,We),r(e,At,t),m(pe,e,t),r(e,jt,t),m(ce,e,t),r(e,qt,t),r(e,V,t),m(me,V,null),i(V,xa),i(V,He),r(e,Rt,t),m(ge,e,t),r(e,Ut,t),r(e,J,t),m(fe,J,null),i(J,Ma),i(J,Qe),i(J,Ta),i(J,Xe),r(e,Jt,t),m(he,e,t),r(e,Dt,t),r(e,D,t),m(ue,D,null),i(D,$a),i(D,Ye),i(D,Ca),i(D,Ke),r(e,Et,t),m(_e,e,t),r(e,It,t),m(be,e,t),r(e,zt,t),r(e,ve,t),r(e,St,t),r(e,ye,t),r(e,Nt,t),r(e,we,t),r(e,Vt,t),m(xe,e,t),r(e,Bt,t),r(e,Me,t),r(e,Pt,t),r(e,Te,t),r(e,Ft,t),m($e,e,t),r(e,Gt,t),r(e,k,t),m(Ce,k,null),i(k,La),i(k,et),i(k,ka),i(k,tt),i(k,Aa),i(k,at),r(e,Zt,t),m(Le,e,t),r(e,Ot,t),r(e,E,t),m(ke,E,null),i(E,ja),i(E,nt),i(E,qa),i(E,ot),r(e,Wt,t),r(e,A,t),m(Ae,A,null),i(A,Ra),i(A,it),i(A,Ua),i(A,rt),i(A,Ja),i(A,st),r(e,Ht,t),m(je,e,t),r(e,Qt,t),r(e,$,t),m(qe,$,null),i($,Da),i($,lt),i($,Ea),i($,dt),i($,Ia),i($,pt),i($,za),i($,ct),r(e,Xt,t),m(Re,e,t),r(e,Yt,t),r(e,ut,t),Kt=!0},p(e,[t]){const B={};t&2&&(B.$$scope={dirty:t,ctx:e}),F.$set(B);const Ue={};t&2&&(Ue.$$scope={dirty:t,ctx:e}),G.$set(Ue)},i(e){Kt||(g(q.$$.fragment,e),g(R.$$.fragment,e),g(Y.$$.fragment,e),g(K.$$.fragment,e),g(ee.$$.fragment,e),g(te.$$.fragment,e),g(ae.$$.fragment,e),g(F.$$.fragment,e),g(G.$$.fragment,e),g(ne.$$.fragment,e),g(oe.$$.fragment,e),g(ie.$$.fragment,e),g(re.$$.fragment,e),g(se.$$.fragment,e),g(le.$$.fragment,e),g(de.$$.fragment,e),g(pe.$$.fragment,e),g(ce.$$.fragment,e),g(me.$$.fragment,e),g(ge.$$.fragment,e),g(fe.$$.fragment,e),g(he.$$.fragment,e),g(ue.$$.fragment,e),g(_e.$$.fragment,e),g(be.$$.fragment,e),g(xe.$$.fragment,e),g($e.$$.fragment,e),g(Ce.$$.fragment,e),g(Le.$$.fragment,e),g(ke.$$.fragment,e),g(Ae.$$.fragment,e),g(je.$$.fragment,e),g(qe.$$.fragment,e),g(Re.$$.fragment,e),Kt=!0)},o(e){f(q.$$.fragment,e),f(R.$$.fragment,e),f(Y.$$.fragment,e),f(K.$$.fragment,e),f(ee.$$.fragment,e),f(te.$$.fragment,e),f(ae.$$.fragment,e),f(F.$$.fragment,e),f(G.$$.fragment,e),f(ne.$$.fragment,e),f(oe.$$.fragment,e),f(ie.$$.fragment,e),f(re.$$.fragment,e),f(se.$$.fragment,e),f(le.$$.fragment,e),f(de.$$.fragment,e),f(pe.$$.fragment,e),f(ce.$$.fragment,e),f(me.$$.fragment,e),f(ge.$$.fragment,e),f(fe.$$.fragment,e),f(he.$$.fragment,e),f(ue.$$.fragment,e),f(_e.$$.fragment,e),f(be.$$.fragment,e),f(xe.$$.fragment,e),f($e.$$.fragment,e),f(Ce.$$.fragment,e),f(Le.$$.fragment,e),f(ke.$$.fragment,e),f(Ae.$$.fragment,e),f(je.$$.fragment,e),f(qe.$$.fragment,e),f(Re.$$.fragment,e),Kt=!1},d(e){e&&(a(j),a(b),a(C),a(v),a(_t),a(H),a(bt),a(Q),a(vt),a(X),a(yt),a(wt),a(U),a(xt),a(Mt),a(_),a(Tt),a($t),a(Ct),a(N),a(Lt),a(kt),a(T),a(At),a(jt),a(qt),a(V),a(Rt),a(Ut),a(J),a(Jt),a(Dt),a(D),a(Et),a(It),a(zt),a(ve),a(St),a(ye),a(Nt),a(we),a(Vt),a(Bt),a(Me),a(Pt),a(Te),a(Ft),a(Gt),a(k),a(Zt),a(Ot),a(E),a(Wt),a(A),a(Ht),a(Qt),a($),a(Xt),a(Yt),a(ut)),a(u),h(q,e),h(R,e),h(Y,e),h(K),h(ee),h(te,e),h(ae),h(F),h(G),h(ne),h(oe),h(ie,e),h(re,e),h(se),h(le,e),h(de),h(pe,e),h(ce,e),h(me),h(ge,e),h(fe),h(he,e),h(ue),h(_e,e),h(be,e),h(xe,e),h($e,e),h(Ce),h(Le,e),h(ke),h(Ae),h(je,e),h(qe),h(Re,e)}}}const zn='{"title":"LoRA","local":"lora","sections":[{"title":"LoraConfig","local":"peft.LoraConfig","sections":[],"depth":2},{"title":"LoraModel","local":"peft.LoraModel","sections":[],"depth":2},{"title":"Utility","local":"utility","sections":[{"title":"ArrowConfig","local":"peft.ArrowConfig","sections":[],"depth":3},{"title":"LoftQ","local":"peft.replace_lora_weights_loftq","sections":[],"depth":3},{"title":"Eva","local":"eva","sections":[{"title":"EvaConfig","local":"peft.EvaConfig","sections":[],"depth":4},{"title":"initialize_lora_eva_weights","local":"peft.initialize_lora_eva_weights","sections":[],"depth":4},{"title":"get_eva_state_dict","local":"peft.get_eva_state_dict","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Variants","local":"variants","sections":[{"title":"LoRA-GA","local":"lora-ga","sections":[{"title":"Usage Tips","local":"usage-tips","sections":[],"depth":4},{"title":"LoraGAConfig","local":"peft.LoraGAConfig","sections":[],"depth":4},{"title":"Utilities","local":"peft.tuners.lora.loraga.estimate_gradients","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Intruder Dimension Reduction","local":"peft.tuners.lora.intruders.reduce_intruder_dimension","sections":[],"depth":2}],"depth":1}';function Sn(Je){return An(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Zn extends jn{constructor(u){super(),qn(this,u,Sn,In,kn,{})}}export{Zn as component};
