import{s as we,n as $e,o as Ce}from"../chunks/scheduler.78382b47.js";import{S as Le,i as Me,e as i,s as o,c as L,h as ke,a as s,d as n,b as a,f as re,g as M,j as f,k as oe,l as c,m as r,n as k,t as q,o as P,p as A}from"../chunks/index.6dd35eb6.js";import{C as qe,H as ce,E as Pe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fed9dfe4.js";import{D as ve}from"../chunks/Docstring.32e826ec.js";function Ae(me){let d,S,D,z,u,W,h,B,g,de='<a href="https://huggingface.co/papers/2405.03003" rel="nofollow">FourierFT</a> is a parameter-efficient fine-tuning technique that leverages Discrete Fourier Transform to compress the model’s tunable weights. This method outperforms LoRA in the GLUE benchmark and common ViT classification tasks using much less parameters.',j,_,pe="FourierFT currently has the following constraints:",I,F,ue="<li>Only <code>nn.Linear</code> layers are supported.</li> <li>Quantized layers are not supported.</li>",V,T,he="If these constraints don’t work for your use case, consider other methods instead.",G,x,ge="The abstract from the paper is:",Q,y,_e="<p>Low-rank adaptation (LoRA) has recently gained much interest in fine-tuning foundation models. It effectively reduces the number of trainable parameters by incorporating low-rank matrices A and B to represent the weight change, i.e., Delta W=BA. Despite LoRA’s progress, it faces storage challenges when handling extensive customization adaptations or larger base models. In this work, we aim to further compress trainable parameters by enjoying the powerful expressiveness of the Fourier transform. Specifically, we introduce FourierFT, which treats Delta W as a matrix in the spatial domain and learns only a small fraction of its spectral coefficients. With the trained spectral coefficients, we implement the inverse discrete Fourier transform to recover Delta W. Empirically, our FourierFT method shows comparable or better performance with fewer parameters than LoRA on various tasks, including natural language understanding, natural language generation, instruction tuning, and image classification. For example, when performing instruction tuning on the LLaMA2-7B model, FourierFT surpasses LoRA with only 0.064M trainable parameters, compared to LoRA’s 33.5M.</p>",K,b,J,p,v,ae,O,Fe='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/fourierft#peft.FourierFTModel">FourierFTModel</a>.',X,w,Y,l,$,ie,U,Te="Creates FourierFT model from a pretrained transformers model.",se,E,xe='The method is described in detail in <a href="https://huggingface.co/papers/2405.03003" rel="nofollow">https://huggingface.co/papers/2405.03003</a>.',le,R,ye="<strong>Attributes</strong>:",fe,H,be='<li><strong>model</strong> (<a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel" rel="nofollow">PreTrainedModel</a>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/main/en/package_reference/fourierft#peft.FourierFTConfig">FourierFTConfig</a>): The configuration of the Fourier model.</li>',Z,C,ee,N,te;return u=new qe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new ce({props:{title:"FourierFT: Discrete Fourier Transformation Fine-Tuning",local:"fourierft-discrete-fourier-transformation-fine-tuning",headingTag:"h1"}}),b=new ce({props:{title:"FourierFTConfig",local:"peft.FourierFTConfig",headingTag:"h2"}}),v=new ve({props:{name:"class peft.FourierFTConfig",anchor:"peft.FourierFTConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"n_frequency",val:": int = 1000"},{name:"scaling",val:": float = 150.0"},{name:"random_loc_seed",val:": Optional[int] = 777"},{name:"fan_in_fan_out",val:": bool = False"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"bias",val:": str = 'none'"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"n_frequency_pattern",val:": Optional[dict] = <factory>"},{name:"init_weights",val:": bool = False"}],parametersDescription:[{anchor:"peft.FourierFTConfig.n_frequency",description:`<strong>n_frequency</strong> (<code>int</code>) &#x2014;
Num of learnable frequencies for the Discrete Fourier Transform. &#x2018;n_frequency&#x2019; is an integer that is
greater than 0 and less than or equal to d^2 (assuming the weight W has dimensions of d by d).
Additionally, it is the number of trainable parameters required to update each delta W weight.
&#x2018;n_frequency&#x2019; will affect the performance and efficiency for PEFT. Specifically, it has little impact on
training speed, but higher values of it (typically) result in larger GPU memory costs and better accuracy.
With the same <code>target_modules</code>, the number of parameters of LoRA is (2<em>d</em>r/n_frequency) times that of
FourierFT. The following examples of settings regarding &#x2018;n_frequency&#x2019; can be used as reference for users.
For NLU tasks with the RoBERTa-large model, adopting &#x2018;n_frequency&#x2019;: 1000 can almost achieve similar results
as &#x2018;r&#x2019;: 8 in LoRA. At this time, the number of parameters of LoRA is about 16 times that of FourierFT. For
image classification tasks with Vit-large models, adopting &#x2018;n_frequency&#x2019;: 3000 can almost achieve similar
results as &#x2018;r&#x2019;: 16 in LoRA, where the number of parameters of LoRA is about 11 times that of FourierFT.`,name:"n_frequency"},{anchor:"peft.FourierFTConfig.scaling",description:`<strong>scaling</strong> (<code>float</code>) &#x2014;
The scaling value for the delta W matrix. This is an important hyperparameter used for scaling, similar to
the &#x2018;lora_alpha&#x2019; parameter in the LoRA method. &#x2018;scaling&#x2019; can be determined during the hyperparameter search
process. However, if users want to skip this process, one can refer to the settings in the following
scenarios. This parameter can be set to 100.0 or 150.0 for both RoBERTa-base and RoBERTa-large models
across all NLU (GLUE) tasks. This parameter can be set to 300.0 for both LLaMA family models for all
instruction tuning. This parameter can be set to 300.0 for both ViT-base and ViT-large models across all
image classification tasks.`,name:"scaling"},{anchor:"peft.FourierFTConfig.random_loc_seed",description:`<strong>random_loc_seed</strong> (<code>int</code>) &#x2014;
Seed for the random location of the frequencies, i.e., the spectral entry matrix.`,name:"random_loc_seed"},{anchor:"peft.FourierFTConfig.target_modules",description:`<strong>target_modules</strong> (<code>Union[list[str],str]</code>) &#x2014;
List of module names or regex expression of the module names to replace with FourierFT. For example, [&#x2018;q&#x2019;,
&#x2018;v&#x2019;] or &#x2018;.<em>decoder.</em>(SelfAttention|EncDecAttention).*(q|v)$&#x2018;. Only linear layers are supported.`,name:"target_modules"},{anchor:"peft.FourierFTConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
When passing a list of strings, either an exact match will be performed or it is checked if the name of the
module ends with any of the passed strings.`,name:"exclude_modules"},{anchor:"peft.FourierFTConfig.fan_in_fan_out",description:`<strong>fan_in_fan_out</strong> (<code>bool</code>) &#x2014;
Set this to True if the layer to replace stores weight like (fan_in, fan_out).`,name:"fan_in_fan_out"},{anchor:"peft.FourierFTConfig.bias",description:`<strong>bias</strong> (<code>str</code>) &#x2014;
Bias type for FourierFT. Can be &#x2018;none&#x2019;, &#x2018;all&#x2019; or &#x2018;fourier_only&#x2019;.`,name:"bias"},{anchor:"peft.FourierFTConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>list[str]</code>) &#x2014;
List of modules apart from FourierFT layers to be set as trainable and saved in the final checkpoint. For
example, in Sequence Classification or Token Classification tasks, the final layer <code>classifier/score</code> are
randomly initialized and as such need to be trainable and saved.`,name:"modules_to_save"},{anchor:"peft.FourierFTConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[list[int],int]</code>) &#x2014;
The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes
that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at
this index.`,name:"layers_to_transform"},{anchor:"peft.FourierFTConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The layer pattern name, used only if <code>layers_to_transform</code> is different to None and if the layer pattern is
not in the common layers pattern. This should target the <code>nn.ModuleList</code> of the model, which is often
called <code>&apos;layers&apos;</code> or <code>&apos;h&apos;</code>.`,name:"layers_pattern"},{anchor:"peft.FourierFTConfig.n_frequency_pattern",description:`<strong>n_frequency_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to n_frequency which are different from the default
specified. For example, <code>{model.decoder.layers.0.encoder_attn.k_proj: 1000</code>}.`,name:"n_frequency_pattern"},{anchor:"peft.FourierFTConfig.init_weights",description:`<strong>init_weights</strong> (<code>bool</code>) &#x2014;
The initialization of the Fourier weights. Set this to False (the default) if the spectrum are initialized
to a standard normal distribution. Set this to True if the spectrum are initialized to zeros.`,name:"init_weights"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/fourierft/config.py#L25"}}),w=new ce({props:{title:"FourierFTModel",local:"peft.FourierFTModel",headingTag:"h2"}}),$=new ve({props:{name:"class peft.FourierFTModel",anchor:"peft.FourierFTModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],parametersDescription:[{anchor:"peft.FourierFTModel.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to be adapted.",name:"model"},{anchor:"peft.FourierFTModel.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/fourierft#peft.FourierFTConfig">FourierFTConfig</a>) &#x2014; The configuration of the FourierFT model.',name:"config"},{anchor:"peft.FourierFTModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.FourierFTModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/fourierft/model.py#L31",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The FourierFT model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}}),C=new Pe({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/fourierft.md"}}),{c(){d=i("meta"),S=o(),D=i("p"),z=o(),L(u.$$.fragment),W=o(),L(h.$$.fragment),B=o(),g=i("p"),g.innerHTML=de,j=o(),_=i("p"),_.textContent=pe,I=o(),F=i("ul"),F.innerHTML=ue,V=o(),T=i("p"),T.textContent=he,G=o(),x=i("p"),x.textContent=ge,Q=o(),y=i("blockquote"),y.innerHTML=_e,K=o(),L(b.$$.fragment),J=o(),p=i("div"),L(v.$$.fragment),ae=o(),O=i("p"),O.innerHTML=Fe,X=o(),L(w.$$.fragment),Y=o(),l=i("div"),L($.$$.fragment),ie=o(),U=i("p"),U.textContent=Te,se=o(),E=i("p"),E.innerHTML=xe,le=o(),R=i("p"),R.innerHTML=ye,fe=o(),H=i("ul"),H.innerHTML=be,Z=o(),L(C.$$.fragment),ee=o(),N=i("p"),this.h()},l(e){const t=ke("svelte-u9bgzb",document.head);d=s(t,"META",{name:!0,content:!0}),t.forEach(n),S=a(e),D=s(e,"P",{}),re(D).forEach(n),z=a(e),M(u.$$.fragment,e),W=a(e),M(h.$$.fragment,e),B=a(e),g=s(e,"P",{"data-svelte-h":!0}),f(g)!=="svelte-1our0gb"&&(g.innerHTML=de),j=a(e),_=s(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-7ls0zq"&&(_.textContent=pe),I=a(e),F=s(e,"UL",{"data-svelte-h":!0}),f(F)!=="svelte-8km6hk"&&(F.innerHTML=ue),V=a(e),T=s(e,"P",{"data-svelte-h":!0}),f(T)!=="svelte-hr8j55"&&(T.textContent=he),G=a(e),x=s(e,"P",{"data-svelte-h":!0}),f(x)!=="svelte-1cwsb16"&&(x.textContent=ge),Q=a(e),y=s(e,"BLOCKQUOTE",{"data-svelte-h":!0}),f(y)!=="svelte-1lxn511"&&(y.innerHTML=_e),K=a(e),M(b.$$.fragment,e),J=a(e),p=s(e,"DIV",{class:!0});var ne=re(p);M(v.$$.fragment,ne),ae=a(ne),O=s(ne,"P",{"data-svelte-h":!0}),f(O)!=="svelte-1j2s7ln"&&(O.innerHTML=Fe),ne.forEach(n),X=a(e),M(w.$$.fragment,e),Y=a(e),l=s(e,"DIV",{class:!0});var m=re(l);M($.$$.fragment,m),ie=a(m),U=s(m,"P",{"data-svelte-h":!0}),f(U)!=="svelte-16dbami"&&(U.textContent=Te),se=a(m),E=s(m,"P",{"data-svelte-h":!0}),f(E)!=="svelte-15mietp"&&(E.innerHTML=xe),le=a(m),R=s(m,"P",{"data-svelte-h":!0}),f(R)!=="svelte-1xx6nm4"&&(R.innerHTML=ye),fe=a(m),H=s(m,"UL",{"data-svelte-h":!0}),f(H)!=="svelte-1ffj8s8"&&(H.innerHTML=be),m.forEach(n),Z=a(e),M(C.$$.fragment,e),ee=a(e),N=s(e,"P",{}),re(N).forEach(n),this.h()},h(){oe(d,"name","hf:doc:metadata"),oe(d,"content",Oe),oe(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),oe(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){c(document.head,d),r(e,S,t),r(e,D,t),r(e,z,t),k(u,e,t),r(e,W,t),k(h,e,t),r(e,B,t),r(e,g,t),r(e,j,t),r(e,_,t),r(e,I,t),r(e,F,t),r(e,V,t),r(e,T,t),r(e,G,t),r(e,x,t),r(e,Q,t),r(e,y,t),r(e,K,t),k(b,e,t),r(e,J,t),r(e,p,t),k(v,p,null),c(p,ae),c(p,O),r(e,X,t),k(w,e,t),r(e,Y,t),r(e,l,t),k($,l,null),c(l,ie),c(l,U),c(l,se),c(l,E),c(l,le),c(l,R),c(l,fe),c(l,H),r(e,Z,t),k(C,e,t),r(e,ee,t),r(e,N,t),te=!0},p:$e,i(e){te||(q(u.$$.fragment,e),q(h.$$.fragment,e),q(b.$$.fragment,e),q(v.$$.fragment,e),q(w.$$.fragment,e),q($.$$.fragment,e),q(C.$$.fragment,e),te=!0)},o(e){P(u.$$.fragment,e),P(h.$$.fragment,e),P(b.$$.fragment,e),P(v.$$.fragment,e),P(w.$$.fragment,e),P($.$$.fragment,e),P(C.$$.fragment,e),te=!1},d(e){e&&(n(S),n(D),n(z),n(W),n(B),n(g),n(j),n(_),n(I),n(F),n(V),n(T),n(G),n(x),n(Q),n(y),n(K),n(J),n(p),n(X),n(Y),n(l),n(Z),n(ee),n(N)),n(d),A(u,e),A(h,e),A(b,e),A(v),A(w,e),A($),A(C,e)}}}const Oe='{"title":"FourierFT: Discrete Fourier Transformation Fine-Tuning","local":"fourierft-discrete-fourier-transformation-fine-tuning","sections":[{"title":"FourierFTConfig","local":"peft.FourierFTConfig","sections":[],"depth":2},{"title":"FourierFTModel","local":"peft.FourierFTModel","sections":[],"depth":2}],"depth":1}';function Ue(me){return Ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ne extends Le{constructor(d){super(),Me(this,d,Ue,Ae,we,{})}}export{Ne as component};
