import{s as nt,o as it,n as at}from"../chunks/scheduler.78382b47.js";import{S as st,i as lt,e as l,s as a,c as _,h as rt,a as r,d as n,b as s,f as ie,g as x,j as m,k as ae,l as o,m as i,n as M,t as b,o as T,p as y}from"../chunks/index.6dd35eb6.js";import{C as ot,H as we,E as pt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fed9dfe4.js";import{D as ze}from"../chunks/Docstring.32e826ec.js";import{C as tt}from"../chunks/CodeBlock.af1ec02e.js";import{E as ft}from"../chunks/ExampleCodeBlock.7f184f2e.js";function mt(se){let d,j="Example:",w,u,g;return u=new tt({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBQcmVmaXhFbmNvZGVyJTJDJTIwUHJlZml4VHVuaW5nQ29uZmlnJTBBJTBBY29uZmlnJTIwJTNEJTIwUHJlZml4VHVuaW5nQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHBlZnRfdHlwZSUzRCUyMlBSRUZJWF9UVU5JTkclMjIlMkMlMEElMjAlMjAlMjAlMjB0YXNrX3R5cGUlM0QlMjJTRVFfMl9TRVFfTE0lMjIlMkMlMEElMjAlMjAlMjAlMjBudW1fdmlydHVhbF90b2tlbnMlM0QyMCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuX2RpbSUzRDc2OCUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFuc2Zvcm1lcl9zdWJtb2R1bGVzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMG51bV9hdHRlbnRpb25faGVhZHMlM0QxMiUyQyUwQSUyMCUyMCUyMCUyMG51bV9sYXllcnMlM0QxMiUyQyUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3NpemUlM0Q3NjglMkMlMEEpJTBBcHJlZml4X2VuY29kZXIlMjAlM0QlMjBQcmVmaXhFbmNvZGVyKGNvbmZpZyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PrefixEncoder, PrefixTuningConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = PrefixTuningConfig(
<span class="hljs-meta">... </span>    peft_type=<span class="hljs-string">&quot;PREFIX_TUNING&quot;</span>,
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;SEQ_2_SEQ_LM&quot;</span>,
<span class="hljs-meta">... </span>    num_virtual_tokens=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    token_dim=<span class="hljs-number">768</span>,
<span class="hljs-meta">... </span>    num_transformer_submodules=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    num_attention_heads=<span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>    num_layers=<span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>    encoder_hidden_size=<span class="hljs-number">768</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>prefix_encoder = PrefixEncoder(config)`,wrap:!1}}),{c(){d=l("p"),d.textContent=j,w=a(),_(u.$$.fragment)},l(f){d=r(f,"P",{"data-svelte-h":!0}),m(d)!=="svelte-11lpom8"&&(d.textContent=j),w=s(f),x(u.$$.fragment,f)},m(f,v){i(f,d,v),i(f,w,v),M(u,f,v),g=!0},p:at,i(f){g||(b(u.$$.fragment,f),g=!0)},o(f){T(u.$$.fragment,f),g=!1},d(f){f&&(n(d),n(w)),y(u,f)}}}function dt(se){let d,j,w,u,g,f,v,le,k,Le='<a href="https://hf.co/papers/2101.00190" rel="nofollow">Prefix tuning</a> prefixes a series of task-specific vectors to the input sequence that can be learned while keeping the pretrained model frozen. The prefix parameters are inserted in all of the model layers.',re,J,Ie="<strong>Note</strong> For encoder-decoder models (seq2seq), the prefix is only applied to the decoder, which does not correspond to the paper specification (see e.g. Figure 2). Prefix tuning can still be fine-tuned on these model architectures but the performance could be sub-par; consider using other PEFT methods for encoder-decoder models.",oe,P,pe,Z,Be=`By default, prefix tuning is randomly initialized. There’s also the option to initialize the embeddings (or the
projection thereof) to be close to a no-op (initialized to zero, it will still shift the probability mass a bit).
This means that the KV-cache injected prefixes have less impact from the beginning and reduces the variance in training
performance.`,fe,E,Qe=`PEFT also provides utilities to initialize a prefix-tuning adapter from an existing KV cache prefix (for example, from
the first <code>p</code> tokens of a prompt/corpus). This is only supported when <code>prefix_projection=False</code> (the default), because
in that case the learned parameters are the KV prefix itself.`,me,V,de,H,Ne="Make sure the text is long enough to produce at least <code>num_virtual_tokens</code> tokens, otherwise initialization will fail.",ce,z,Fe="As a guideline:",ue,L,Ge=`<li>start with a neutral starting sequence using <code>initialize_kv_prefix_from_text</code>, it can be a very short string like
“Question: ”</li> <li>if that doesn’t help, use a longer sequence with task relevance (i.e. an engineered prompt), giving you more virtual
tokens to fit but also more steering of the model</li> <li>if it is not possible to use an initialization text or you want to quickly check if prefix tuning is viable at all,
use a zero init without projection</li>`,ge,I,Xe="The abstract from the paper is:",he,B,We="<em>Fine-tuning is the de facto way to leverage large pretrained language models to perform downstream tasks. However, it modifies all the language model parameters and therefore necessitates storing a full copy for each task. In this paper, we propose prefix-tuning, a lightweight alternative to fine-tuning for natural language generation tasks, which keeps language model parameters frozen, but optimizes a small continuous task-specific vector (called the prefix). Prefix-tuning draws inspiration from prompting, allowing subsequent tokens to attend to this prefix as if it were “virtual tokens”. We apply prefix-tuning to GPT-2 for table-to-text generation and to BART for summarization. We find that by learning only 0.1\\% of the parameters, prefix-tuning obtains comparable performance in the full data setting, outperforms fine-tuning in low-data settings, and extrapolates better to examples with topics unseen during training</em>.",_e,Q,xe,$,N,$e,q,Re='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/prefix_tuning#peft.PrefixEncoder">PrefixEncoder</a>.',Me,F,be,p,G,Ce,S,qe="The <code>torch.nn</code> model to encode the prefix.",Ue,C,je,A,Se="<strong>Attributes</strong>:",ke,D,Ae=`<li><strong>embedding</strong> (<code>torch.nn.Embedding</code>) — The embedding layer of the prefix encoder.</li> <li><strong>transform</strong> (<code>torch.nn.Sequential</code>) — The two-layer MLP to transform the prefix embeddings if
<code>prefix_projection</code> is <code>True</code>.</li> <li><strong>prefix_projection</strong> (<code>bool</code>) — Whether to project the prefix embeddings.</li>`,Je,O,De="Input shape: (<code>batch_size</code>, <code>num_virtual_tokens</code>)",Pe,Y,Oe="Output shape: (<code>batch_size</code>, <code>num_virtual_tokens</code>, <code>2*layers*hidden</code>)",Ze,h,X,Ee,K,Ye="Load the flattened prompt embeddings saved by PEFT (<code>prompt_embeddings</code>).",Ve,ee,Ke=`For prefix tuning, this is only supported when <code>prefix_projection=False</code>, because in that case the learned
parameters are the KV prefix itself (<code>embedding.weight</code> has shape <code>[num_virtual_tokens, num_layers*2*token_dim]</code>).`,He,te,et=`If <code>prefix_projection=True</code>, the parameters are (virtual token embeddings + an MLP) and there is no general way
to invert the projection to recover those parameters from a flattened KV prefix.`,Te,W,ye,ne,ve;return g=new ot({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),v=new we({props:{title:"Prefix tuning",local:"prefix-tuning",headingTag:"h1"}}),P=new we({props:{title:"Possible Initialization",local:"possible-initialization",headingTag:"h2"}}),V=new tt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQWZyb20lMjBwZWZ0JTIwaW1wb3J0JTIwUHJlZml4VHVuaW5nQ29uZmlnJTJDJTIwZ2V0X3BlZnRfbW9kZWwlMkMlMjBpbml0aWFsaXplX2t2X3ByZWZpeF9mcm9tX3RleHQlMEElMEFiYXNlJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmdwdDIlMjIpJTBBdG9rJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ3B0MiUyMiklMEElMEFwZWZ0X2NmZyUyMCUzRCUyMFByZWZpeFR1bmluZ0NvbmZpZyh0YXNrX3R5cGUlM0QlMjJDQVVTQUxfTE0lMjIlMkMlMjBudW1fdmlydHVhbF90b2tlbnMlM0QyMCUyQyUyMHByZWZpeF9wcm9qZWN0aW9uJTNERmFsc2UpJTBBbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChiYXNlJTJDJTIwcGVmdF9jZmcpJTBBJTBBaW5pdGlhbGl6ZV9rdl9wcmVmaXhfZnJvbV90ZXh0KCUwQSUyMCUyMCUyMCUyMG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwdG9rJTJDJTBBJTIwJTIwJTIwJTIwdGV4dCUzRCUyMi4uLmElMjBsb25nJTIwY29udGV4dCUyMHdpdGglMjBhdCUyMGxlYXN0JTIwbnVtX3ZpcnR1YWxfdG9rZW5zJTIwdG9rZW5zLi4uJTIyJTJDJTBBJTIwJTIwJTIwJTIwdXNlX2NoYXRfdGVtcGxhdGUlM0RGYWxzZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PrefixTuningConfig, get_peft_model, initialize_kv_prefix_from_text

base = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
tok = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)

peft_cfg = PrefixTuningConfig(task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>, num_virtual_tokens=<span class="hljs-number">20</span>, prefix_projection=<span class="hljs-literal">False</span>)
model = get_peft_model(base, peft_cfg)

initialize_kv_prefix_from_text(
    model,
    tok,
    text=<span class="hljs-string">&quot;...a long context with at least num_virtual_tokens tokens...&quot;</span>,
    use_chat_template=<span class="hljs-literal">False</span>,
)`,wrap:!1}}),Q=new we({props:{title:"PrefixTuningConfig",local:"peft.PrefixTuningConfig",headingTag:"h2"}}),N=new ze({props:{name:"class peft.PrefixTuningConfig",anchor:"peft.PrefixTuningConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"num_virtual_tokens",val:": int = None"},{name:"token_dim",val:": int = None"},{name:"num_transformer_submodules",val:": Optional[int] = None"},{name:"num_attention_heads",val:": Optional[int] = None"},{name:"num_layers",val:": Optional[int] = None"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"init_weights",val:": typing.Optional[typing.Literal['zero']] = None"},{name:"encoder_hidden_size",val:": int = None"},{name:"prefix_projection",val:": bool = False"}],parametersDescription:[{anchor:"peft.PrefixTuningConfig.init_weights",description:`<strong>init_weights</strong> (<code>Optional[str]</code>) &#x2014; If not set, weights are initialized at random, if set to &#x201C;zero&#x201D;
the weights are initialized so that the activations will be a no-op (zero).`,name:"init_weights"},{anchor:"peft.PrefixTuningConfig.encoder_hidden_size",description:"<strong>encoder_hidden_size</strong> (<code>int</code>) &#x2014; The hidden size of the prompt encoder.",name:"encoder_hidden_size"},{anchor:"peft.PrefixTuningConfig.prefix_projection",description:"<strong>prefix_projection</strong> (<code>bool</code>) &#x2014; Whether to project the prefix embeddings.",name:"prefix_projection"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/prefix_tuning/config.py#L23"}}),F=new we({props:{title:"PrefixEncoder",local:"peft.PrefixEncoder",headingTag:"h2"}}),G=new ze({props:{name:"class peft.PrefixEncoder",anchor:"peft.PrefixEncoder",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"peft.PrefixEncoder.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/prefix_tuning#peft.PrefixTuningConfig">PrefixTuningConfig</a>) &#x2014; The configuration of the prefix encoder.',name:"config"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/prefix_tuning/model.py#L20"}}),C=new ft({props:{anchor:"peft.PrefixEncoder.example",$$slots:{default:[mt]},$$scope:{ctx:se}}}),X=new ze({props:{name:"load_prompt_embeddings",anchor:"peft.PrefixEncoder.load_prompt_embeddings",parameters:[{name:"prompt_embeddings",val:": Tensor"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/prefix_tuning/model.py#L89"}}),W=new pt({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/prefix_tuning.md"}}),{c(){d=l("meta"),j=a(),w=l("p"),u=a(),_(g.$$.fragment),f=a(),_(v.$$.fragment),le=a(),k=l("p"),k.innerHTML=Le,re=a(),J=l("p"),J.innerHTML=Ie,oe=a(),_(P.$$.fragment),pe=a(),Z=l("p"),Z.textContent=Be,fe=a(),E=l("p"),E.innerHTML=Qe,me=a(),_(V.$$.fragment),de=a(),H=l("p"),H.innerHTML=Ne,ce=a(),z=l("p"),z.textContent=Fe,ue=a(),L=l("ul"),L.innerHTML=Ge,ge=a(),I=l("p"),I.textContent=Xe,he=a(),B=l("p"),B.innerHTML=We,_e=a(),_(Q.$$.fragment),xe=a(),$=l("div"),_(N.$$.fragment),$e=a(),q=l("p"),q.innerHTML=Re,Me=a(),_(F.$$.fragment),be=a(),p=l("div"),_(G.$$.fragment),Ce=a(),S=l("p"),S.innerHTML=qe,Ue=a(),_(C.$$.fragment),je=a(),A=l("p"),A.innerHTML=Se,ke=a(),D=l("ul"),D.innerHTML=Ae,Je=a(),O=l("p"),O.innerHTML=De,Pe=a(),Y=l("p"),Y.innerHTML=Oe,Ze=a(),h=l("div"),_(X.$$.fragment),Ee=a(),K=l("p"),K.innerHTML=Ye,Ve=a(),ee=l("p"),ee.innerHTML=Ke,He=a(),te=l("p"),te.innerHTML=et,Te=a(),_(W.$$.fragment),ye=a(),ne=l("p"),this.h()},l(e){const t=rt("svelte-u9bgzb",document.head);d=r(t,"META",{name:!0,content:!0}),t.forEach(n),j=s(e),w=r(e,"P",{}),ie(w).forEach(n),u=s(e),x(g.$$.fragment,e),f=s(e),x(v.$$.fragment,e),le=s(e),k=r(e,"P",{"data-svelte-h":!0}),m(k)!=="svelte-arymvv"&&(k.innerHTML=Le),re=s(e),J=r(e,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1i1k3ws"&&(J.innerHTML=Ie),oe=s(e),x(P.$$.fragment,e),pe=s(e),Z=r(e,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-qv8apb"&&(Z.textContent=Be),fe=s(e),E=r(e,"P",{"data-svelte-h":!0}),m(E)!=="svelte-15nz2as"&&(E.innerHTML=Qe),me=s(e),x(V.$$.fragment,e),de=s(e),H=r(e,"P",{"data-svelte-h":!0}),m(H)!=="svelte-13cvj6i"&&(H.innerHTML=Ne),ce=s(e),z=r(e,"P",{"data-svelte-h":!0}),m(z)!=="svelte-bpt9nx"&&(z.textContent=Fe),ue=s(e),L=r(e,"UL",{"data-svelte-h":!0}),m(L)!=="svelte-zhndjb"&&(L.innerHTML=Ge),ge=s(e),I=r(e,"P",{"data-svelte-h":!0}),m(I)!=="svelte-1cwsb16"&&(I.textContent=Xe),he=s(e),B=r(e,"P",{"data-svelte-h":!0}),m(B)!=="svelte-pu6a2e"&&(B.innerHTML=We),_e=s(e),x(Q.$$.fragment,e),xe=s(e),$=r(e,"DIV",{class:!0});var R=ie($);x(N.$$.fragment,R),$e=s(R),q=r(R,"P",{"data-svelte-h":!0}),m(q)!=="svelte-1q81emz"&&(q.innerHTML=Re),R.forEach(n),Me=s(e),x(F.$$.fragment,e),be=s(e),p=r(e,"DIV",{class:!0});var c=ie(p);x(G.$$.fragment,c),Ce=s(c),S=r(c,"P",{"data-svelte-h":!0}),m(S)!=="svelte-18w6err"&&(S.innerHTML=qe),Ue=s(c),x(C.$$.fragment,c),je=s(c),A=r(c,"P",{"data-svelte-h":!0}),m(A)!=="svelte-1xx6nm4"&&(A.innerHTML=Se),ke=s(c),D=r(c,"UL",{"data-svelte-h":!0}),m(D)!=="svelte-1fcqv17"&&(D.innerHTML=Ae),Je=s(c),O=r(c,"P",{"data-svelte-h":!0}),m(O)!=="svelte-15aa8e5"&&(O.innerHTML=De),Pe=s(c),Y=r(c,"P",{"data-svelte-h":!0}),m(Y)!=="svelte-1k69wor"&&(Y.innerHTML=Oe),Ze=s(c),h=r(c,"DIV",{class:!0});var U=ie(h);x(X.$$.fragment,U),Ee=s(U),K=r(U,"P",{"data-svelte-h":!0}),m(K)!=="svelte-1v859eu"&&(K.innerHTML=Ye),Ve=s(U),ee=r(U,"P",{"data-svelte-h":!0}),m(ee)!=="svelte-12whc8p"&&(ee.innerHTML=Ke),He=s(U),te=r(U,"P",{"data-svelte-h":!0}),m(te)!=="svelte-1f34ox3"&&(te.innerHTML=et),U.forEach(n),c.forEach(n),Te=s(e),x(W.$$.fragment,e),ye=s(e),ne=r(e,"P",{}),ie(ne).forEach(n),this.h()},h(){ae(d,"name","hf:doc:metadata"),ae(d,"content",ct),ae($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),ae(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),ae(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){o(document.head,d),i(e,j,t),i(e,w,t),i(e,u,t),M(g,e,t),i(e,f,t),M(v,e,t),i(e,le,t),i(e,k,t),i(e,re,t),i(e,J,t),i(e,oe,t),M(P,e,t),i(e,pe,t),i(e,Z,t),i(e,fe,t),i(e,E,t),i(e,me,t),M(V,e,t),i(e,de,t),i(e,H,t),i(e,ce,t),i(e,z,t),i(e,ue,t),i(e,L,t),i(e,ge,t),i(e,I,t),i(e,he,t),i(e,B,t),i(e,_e,t),M(Q,e,t),i(e,xe,t),i(e,$,t),M(N,$,null),o($,$e),o($,q),i(e,Me,t),M(F,e,t),i(e,be,t),i(e,p,t),M(G,p,null),o(p,Ce),o(p,S),o(p,Ue),M(C,p,null),o(p,je),o(p,A),o(p,ke),o(p,D),o(p,Je),o(p,O),o(p,Pe),o(p,Y),o(p,Ze),o(p,h),M(X,h,null),o(h,Ee),o(h,K),o(h,Ve),o(h,ee),o(h,He),o(h,te),i(e,Te,t),M(W,e,t),i(e,ye,t),i(e,ne,t),ve=!0},p(e,[t]){const R={};t&2&&(R.$$scope={dirty:t,ctx:e}),C.$set(R)},i(e){ve||(b(g.$$.fragment,e),b(v.$$.fragment,e),b(P.$$.fragment,e),b(V.$$.fragment,e),b(Q.$$.fragment,e),b(N.$$.fragment,e),b(F.$$.fragment,e),b(G.$$.fragment,e),b(C.$$.fragment,e),b(X.$$.fragment,e),b(W.$$.fragment,e),ve=!0)},o(e){T(g.$$.fragment,e),T(v.$$.fragment,e),T(P.$$.fragment,e),T(V.$$.fragment,e),T(Q.$$.fragment,e),T(N.$$.fragment,e),T(F.$$.fragment,e),T(G.$$.fragment,e),T(C.$$.fragment,e),T(X.$$.fragment,e),T(W.$$.fragment,e),ve=!1},d(e){e&&(n(j),n(w),n(u),n(f),n(le),n(k),n(re),n(J),n(oe),n(pe),n(Z),n(fe),n(E),n(me),n(de),n(H),n(ce),n(z),n(ue),n(L),n(ge),n(I),n(he),n(B),n(_e),n(xe),n($),n(Me),n(be),n(p),n(Te),n(ye),n(ne)),n(d),y(g,e),y(v,e),y(P,e),y(V,e),y(Q,e),y(N),y(F,e),y(G),y(C),y(X),y(W,e)}}}const ct='{"title":"Prefix tuning","local":"prefix-tuning","sections":[{"title":"Possible Initialization","local":"possible-initialization","sections":[],"depth":2},{"title":"PrefixTuningConfig","local":"peft.PrefixTuningConfig","sections":[],"depth":2},{"title":"PrefixEncoder","local":"peft.PrefixEncoder","sections":[],"depth":2}],"depth":1}';function ut(se){return it(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Tt extends st{constructor(d){super(),lt(this,d,ut,dt,nt,{})}}export{Tt as component};
