import{s as lt,n as pt,o as mt}from"../chunks/scheduler.78382b47.js";import{S as ct,i as ht,e as r,s as n,c as B,q as O,H as st,h as dt,a as l,d as a,b as i,f as W,g as D,j as L,r as J,u as nt,k as it,l as I,m as s,n as K,t as N,o as Q,p as V}from"../chunks/index.6dd35eb6.js";import{C as ut,H as ft,E as gt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fed9dfe4.js";function wt(X){let p,_,b,k,m,y,c,A,h,Y='<a href="https://huggingface.co/papers/2505.20355" rel="nofollow"><strong>Granular Low-Rank Adaptation (GraLoRA)</strong></a> is a PEFT method designed to enhance the <strong>expressivity</strong> of low-rank adaptation while improving <strong>robustness to outlier</strong> activations, based on insights from well-known issues in quantization.',R,d,Z='<img src="https://github.com/SqueezeBits/GraLoRA/raw/main/figure/gralora_overview.png" alt="GraLoRA Overview"/>',$,o,U,T,ot='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msup><mi>𝑘</mi><mn>2</mn></msup></mrow><annotation encoding="application/x-tex">𝑘^2</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8141em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.8141em;"><span style="top:-3.063em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span></span></span></span></span></span></span></span>',M,P,rt='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span></span></span></span>',G,H,u,tt="This granular decomposition enables spatially localized and context-aware updates, effectively increasing representational capacity without additional parameters or computational cost. By isolating the influence of extreme activations within smaller subspaces, GraLoRA mitigates gradient distortion and preserves inter-channel balance during adaptation.",z,C,E,f,et="The abstract from the paper is:",F,g,at=`<em>Low-Rank Adaptation (LoRA) is a popular method for parameter-efficient fine-
tuning (PEFT) of generative models, valued for its simplicity and effectiveness.
Despite recent enhancements, LoRA still suffers from a fundamental limitation:
overfitting when the bottleneck is widened. It performs best at ranks 32–64, yet its
accuracy stagnates or declines at higher ranks, still falling short of full fine-tuning
(FFT) performance. We identify the root cause as LoRA’s structural bottleneck,
which introduces gradient entanglement to the unrelated input channels and distorts
gradient propagation. To address this, we introduce a novel structure, Granular
Low-Rank Adaptation (GraLoRA) that partitions weight matrices into sub-blocks,
each with its own low-rank adapter. With negligible computational or storage cost,
GraLoRA overcomes LoRA’s limitations, effectively increases the representational
capacity, and more closely approximates FFT behavior. Experiments on code
generation, commonsense reasoning, mathematical reasoning, general language
understanding, and image generation benchmarks show that GraLoRA consistently
outperforms LoRA and other baselines, achieving up to +8.5% absolute gain in
Pass@1 on HumanEval+. These improvements hold across model sizes and rank
settings, making GraLoRA a scalable and robust solution for PEFT.</em>`,q,w,S,x,j;return m=new ut({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),c=new ft({props:{title:"GraLoRA",local:"gralora",headingTag:"h1"}}),w=new gt({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/gralora.md"}}),{c(){p=r("meta"),_=n(),b=r("p"),k=n(),B(m.$$.fragment),y=n(),B(c.$$.fragment),A=n(),h=r("p"),h.innerHTML=Y,R=n(),d=r("p"),d.innerHTML=Z,$=n(),o=r("p"),U=O("Unlike standard LoRA, which applies a single low-rank adapter across the entire feature space, GraLoRA introduces a structured and fine-grained adaptation scheme. It divides the adaptation space into a grid of "),T=new st(!1),M=O(" smaller, independent adapter pairs, each responsible for a localized subset of the input and output dimensions. As a result, each adapter operates on a subspace that is "),P=new st(!1),G=O(" times smaller in both dimensions than the original LoRA adapter."),H=n(),u=r("p"),u.textContent=tt,z=n(),C=r("hr"),E=n(),f=r("p"),f.textContent=et,F=n(),g=r("p"),g.innerHTML=at,q=n(),B(w.$$.fragment),S=n(),x=r("p"),this.h()},l(t){const e=dt("svelte-u9bgzb",document.head);p=l(e,"META",{name:!0,content:!0}),e.forEach(a),_=i(t),b=l(t,"P",{}),W(b).forEach(a),k=i(t),D(m.$$.fragment,t),y=i(t),D(c.$$.fragment,t),A=i(t),h=l(t,"P",{"data-svelte-h":!0}),L(h)!=="svelte-1ry3fev"&&(h.innerHTML=Y),R=i(t),d=l(t,"P",{"data-svelte-h":!0}),L(d)!=="svelte-7jtsq0"&&(d.innerHTML=Z),$=i(t),o=l(t,"P",{});var v=W(o);U=J(v,"Unlike standard LoRA, which applies a single low-rank adapter across the entire feature space, GraLoRA introduces a structured and fine-grained adaptation scheme. It divides the adaptation space into a grid of "),T=nt(v,!1),M=J(v," smaller, independent adapter pairs, each responsible for a localized subset of the input and output dimensions. As a result, each adapter operates on a subspace that is "),P=nt(v,!1),G=J(v," times smaller in both dimensions than the original LoRA adapter."),v.forEach(a),H=i(t),u=l(t,"P",{"data-svelte-h":!0}),L(u)!=="svelte-zemtjn"&&(u.textContent=tt),z=i(t),C=l(t,"HR",{}),E=i(t),f=l(t,"P",{"data-svelte-h":!0}),L(f)!=="svelte-1cwsb16"&&(f.textContent=et),F=i(t),g=l(t,"P",{"data-svelte-h":!0}),L(g)!=="svelte-1uqtl41"&&(g.innerHTML=at),q=i(t),D(w.$$.fragment,t),S=i(t),x=l(t,"P",{}),W(x).forEach(a),this.h()},h(){it(p,"name","hf:doc:metadata"),it(p,"content",vt),T.a=M,P.a=G},m(t,e){I(document.head,p),s(t,_,e),s(t,b,e),s(t,k,e),K(m,t,e),s(t,y,e),K(c,t,e),s(t,A,e),s(t,h,e),s(t,R,e),s(t,d,e),s(t,$,e),s(t,o,e),I(o,U),T.m(ot,o),I(o,M),P.m(rt,o),I(o,G),s(t,H,e),s(t,u,e),s(t,z,e),s(t,C,e),s(t,E,e),s(t,f,e),s(t,F,e),s(t,g,e),s(t,q,e),K(w,t,e),s(t,S,e),s(t,x,e),j=!0},p:pt,i(t){j||(N(m.$$.fragment,t),N(c.$$.fragment,t),N(w.$$.fragment,t),j=!0)},o(t){Q(m.$$.fragment,t),Q(c.$$.fragment,t),Q(w.$$.fragment,t),j=!1},d(t){t&&(a(_),a(b),a(k),a(y),a(A),a(h),a(R),a(d),a($),a(o),a(H),a(u),a(z),a(C),a(E),a(f),a(F),a(g),a(q),a(S),a(x)),a(p),V(m,t),V(c,t),V(w,t)}}}const vt='{"title":"GraLoRA","local":"gralora","sections":[],"depth":1}';function bt(X){return mt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class kt extends ct{constructor(p){super(),ht(this,p,bt,wt,lt,{})}}export{kt as component};
