import{s as ye,o as we,n as ve}from"../chunks/scheduler.78382b47.js";import{S as $e,i as Je,e as l,s as a,c as b,h as Me,a as m,d as n,b as s,f as ee,g as y,j as _,k as te,l as c,m as o,n as w,t as v,o as $,p as J}from"../chunks/index.6dd35eb6.js";import{C as ke,H as le,E as Pe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8d7e60c0.js";import{D as be}from"../chunks/Docstring.57cb1354.js";import{C as xe}from"../chunks/CodeBlock.4d4112a9.js";import{E as Ce}from"../chunks/ExampleCodeBlock.7440be11.js";function Ie(X){let p,k="Example:",h,g,f;return g=new xe({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBQcm9tcHRFbWJlZGRpbmclMkMlMjBQcm9tcHRUdW5pbmdDb25maWclMEElMEFjb25maWclMjAlM0QlMjBQcm9tcHRUdW5pbmdDb25maWcoJTBBJTIwJTIwJTIwJTIwcGVmdF90eXBlJTNEJTIyUFJPTVBUX1RVTklORyUyMiUyQyUwQSUyMCUyMCUyMCUyMHRhc2tfdHlwZSUzRCUyMlNFUV8yX1NFUV9MTSUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV92aXJ0dWFsX3Rva2VucyUzRDIwJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5fZGltJTNENzY4JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYW5zZm9ybWVyX3N1Ym1vZHVsZXMlM0QxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2F0dGVudGlvbl9oZWFkcyUzRDEyJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2xheWVycyUzRDEyJTJDJTBBJTIwJTIwJTIwJTIwcHJvbXB0X3R1bmluZ19pbml0JTNEJTIyVEVYVCUyMiUyQyUwQSUyMCUyMCUyMCUyMHByb21wdF90dW5pbmdfaW5pdF90ZXh0JTNEJTIyUHJlZGljdCUyMGlmJTIwc2VudGltZW50JTIwb2YlMjB0aGlzJTIwcmV2aWV3JTIwaXMlMjBwb3NpdGl2ZSUyQyUyMG5lZ2F0aXZlJTIwb3IlMjBuZXV0cmFsJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyX25hbWVfb3JfcGF0aCUzRCUyMnQ1LWJhc2UlMjIlMkMlMEEpJTBBJTBBJTIzJTIwdDVfbW9kZWwuc2hhcmVkJTIwaXMlMjB0aGUlMjB3b3JkJTIwZW1iZWRkaW5ncyUyMG9mJTIwdGhlJTIwYmFzZSUyMG1vZGVsJTBBcHJvbXB0X2VtYmVkZGluZyUyMCUzRCUyMFByb21wdEVtYmVkZGluZyhjb25maWclMkMlMjB0NV9tb2RlbC5zaGFyZWQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PromptEmbedding, PromptTuningConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = PromptTuningConfig(
<span class="hljs-meta">... </span>    peft_type=<span class="hljs-string">&quot;PROMPT_TUNING&quot;</span>,
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;SEQ_2_SEQ_LM&quot;</span>,
<span class="hljs-meta">... </span>    num_virtual_tokens=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    token_dim=<span class="hljs-number">768</span>,
<span class="hljs-meta">... </span>    num_transformer_submodules=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    num_attention_heads=<span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>    num_layers=<span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>    prompt_tuning_init=<span class="hljs-string">&quot;TEXT&quot;</span>,
<span class="hljs-meta">... </span>    prompt_tuning_init_text=<span class="hljs-string">&quot;Predict if sentiment of this review is positive, negative or neutral&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer_name_or_path=<span class="hljs-string">&quot;t5-base&quot;</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># t5_model.shared is the word embeddings of the base model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt_embedding = PromptEmbedding(config, t5_model.shared)`,wrap:!1}}),{c(){p=l("p"),p.textContent=k,h=a(),b(g.$$.fragment)},l(i){p=m(i,"P",{"data-svelte-h":!0}),_(p)!=="svelte-11lpom8"&&(p.textContent=k),h=s(i),y(g.$$.fragment,i)},m(i,u){o(i,p,u),o(i,h,u),w(g,i,u),f=!0},p:ve,i(i){f||(v(g.$$.fragment,i),f=!0)},o(i){$(g.$$.fragment,i),f=!1},d(i){i&&(n(p),n(h)),J(g,i)}}}function Ue(X){let p,k,h,g,f,i,u,L,P,me='<a href="https://hf.co/papers/2104.08691" rel="nofollow">Prompt tuning</a> adds task-specific prompts to the input, and these prompt parameters are updated independently of the pretrained model parameters which are frozen.',R,x,de="The abstract from the paper is:",D,C,ce="<em>In this work, we explore “prompt tuning”, a simple yet effective mechanism for learning “soft prompts” to condition frozen language models to perform specific downstream tasks. Unlike the discrete text prompts used by GPT-3, soft prompts are learned through backpropagation and can be tuned to incorporate signal from any number of labeled examples. Our end-to-end learned approach outperforms GPT-3’s “few-shot” learning by a large margin. More remarkably, through ablations on model size using T5, we show that prompt tuning becomes more competitive with scale: as models exceed billions of parameters, our method “closes the gap” and matches the strong performance of model tuning (where all model weights are tuned). This finding is especially relevant in that large models are costly to share and serve, and the ability to reuse one frozen model for multiple downstream tasks can ease this burden. Our method can be seen as a simplification of the recently proposed “prefix tuning” of Li and Liang (2021), and we provide a comparison to this and other similar approaches. Finally, we show that conditioning a frozen model with soft prompts confers benefits in robustness to domain transfer, as compared to full model tuning</em>.",F,I,Q,T,U,ne,V,ge='This is the configuration class to store the configuration of a <a href="/docs/peft/v0.18.0.rc0/en/package_reference/prompt_tuning#peft.PromptEmbedding">PromptEmbedding</a>.',S,j,q,r,E,oe,N,fe="The model to encode virtual tokens into prompt embeddings.",ae,Z,ue="<strong>Attributes</strong>:",se,G,he="<li><strong>embedding</strong> (<code>torch.nn.Embedding</code>) — The embedding layer of the prompt embedding.</li>",ie,M,re,H,_e="Input Shape: (<code>batch_size</code>, <code>total_virtual_tokens</code>)",pe,O,Te="Output Shape: (<code>batch_size</code>, <code>total_virtual_tokens</code>, <code>token_dim</code>)",A,z,Y,W,K;return f=new ke({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),u=new le({props:{title:"Prompt tuning",local:"prompt-tuning",headingTag:"h1"}}),I=new le({props:{title:"PromptTuningConfig",local:"peft.PromptTuningConfig",headingTag:"h2"}}),U=new be({props:{name:"class peft.PromptTuningConfig",anchor:"peft.PromptTuningConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"num_virtual_tokens",val:": int = None"},{name:"token_dim",val:": int = None"},{name:"num_transformer_submodules",val:": Optional[int] = None"},{name:"num_attention_heads",val:": Optional[int] = None"},{name:"num_layers",val:": Optional[int] = None"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"prompt_tuning_init",val:": typing.Union[peft.tuners.prompt_tuning.config.PromptTuningInit, str] = <PromptTuningInit.RANDOM: 'RANDOM'>"},{name:"prompt_tuning_init_text",val:": typing.Optional[str] = None"},{name:"tokenizer_name_or_path",val:": typing.Optional[str] = None"},{name:"tokenizer_kwargs",val:": typing.Optional[dict] = None"}],parametersDescription:[{anchor:"peft.PromptTuningConfig.prompt_tuning_init",description:`<strong>prompt_tuning_init</strong> (Union[<code>PromptTuningInit</code>, <code>str</code>]) &#x2014;
The initialization of the prompt embedding. <code>TEXT</code> will initialize with your text. <code>SAMPLE_VOCAB</code> will
initialize with randomly sampled tokens from the model&#x2019;s vocabulary. <code>RANDOM</code> will initialize with randomly
sampled continuous, soft tokens (warning: sampled soft tokens may fall outside of embedding manifold)`,name:"prompt_tuning_init"},{anchor:"peft.PromptTuningConfig.prompt_tuning_init_text",description:`<strong>prompt_tuning_init_text</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The text to initialize the prompt embedding. Only used if <code>prompt_tuning_init</code> is <code>TEXT</code>.`,name:"prompt_tuning_init_text"},{anchor:"peft.PromptTuningConfig.tokenizer_name_or_path",description:`<strong>tokenizer_name_or_path</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The name or path of the tokenizer. Only used if <code>prompt_tuning_init</code> is <code>TEXT</code>.`,name:"tokenizer_name_or_path"},{anchor:"peft.PromptTuningConfig.tokenizer_kwargs",description:`<strong>tokenizer_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
The keyword arguments to pass to <code>AutoTokenizer.from_pretrained</code>. Only used if <code>prompt_tuning_init</code> is
<code>TEXT</code>.`,name:"tokenizer_kwargs"}],source:"https://github.com/huggingface/peft/blob/v0.18.0.rc0/src/peft/tuners/prompt_tuning/config.py#L30"}}),j=new le({props:{title:"PromptEmbedding",local:"peft.PromptEmbedding",headingTag:"h2"}}),E=new be({props:{name:"class peft.PromptEmbedding",anchor:"peft.PromptEmbedding",parameters:[{name:"config",val:""},{name:"word_embeddings",val:""}],parametersDescription:[{anchor:"peft.PromptEmbedding.config",description:'<strong>config</strong> (<a href="/docs/peft/v0.18.0.rc0/en/package_reference/prompt_tuning#peft.PromptTuningConfig">PromptTuningConfig</a>) &#x2014; The configuration of the prompt embedding.',name:"config"},{anchor:"peft.PromptEmbedding.word_embeddings",description:"<strong>word_embeddings</strong> (<code>torch.nn.Module</code>) &#x2014; The word embeddings of the base transformer model.",name:"word_embeddings"}],source:"https://github.com/huggingface/peft/blob/v0.18.0.rc0/src/peft/tuners/prompt_tuning/model.py#L24"}}),M=new Ce({props:{anchor:"peft.PromptEmbedding.example",$$slots:{default:[Ie]},$$scope:{ctx:X}}}),z=new Pe({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/prompt_tuning.md"}}),{c(){p=l("meta"),k=a(),h=l("p"),g=a(),b(f.$$.fragment),i=a(),b(u.$$.fragment),L=a(),P=l("p"),P.innerHTML=me,R=a(),x=l("p"),x.textContent=de,D=a(),C=l("p"),C.innerHTML=ce,F=a(),b(I.$$.fragment),Q=a(),T=l("div"),b(U.$$.fragment),ne=a(),V=l("p"),V.innerHTML=ge,S=a(),b(j.$$.fragment),q=a(),r=l("div"),b(E.$$.fragment),oe=a(),N=l("p"),N.textContent=fe,ae=a(),Z=l("p"),Z.innerHTML=ue,se=a(),G=l("ul"),G.innerHTML=he,ie=a(),b(M.$$.fragment),re=a(),H=l("p"),H.innerHTML=_e,pe=a(),O=l("p"),O.innerHTML=Te,A=a(),b(z.$$.fragment),Y=a(),W=l("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);p=m(t,"META",{name:!0,content:!0}),t.forEach(n),k=s(e),h=m(e,"P",{}),ee(h).forEach(n),g=s(e),y(f.$$.fragment,e),i=s(e),y(u.$$.fragment,e),L=s(e),P=m(e,"P",{"data-svelte-h":!0}),_(P)!=="svelte-4zhj2m"&&(P.innerHTML=me),R=s(e),x=m(e,"P",{"data-svelte-h":!0}),_(x)!=="svelte-1cwsb16"&&(x.textContent=de),D=s(e),C=m(e,"P",{"data-svelte-h":!0}),_(C)!=="svelte-18zszvh"&&(C.innerHTML=ce),F=s(e),y(I.$$.fragment,e),Q=s(e),T=m(e,"DIV",{class:!0});var B=ee(T);y(U.$$.fragment,B),ne=s(B),V=m(B,"P",{"data-svelte-h":!0}),_(V)!=="svelte-1cefz9y"&&(V.innerHTML=ge),B.forEach(n),S=s(e),y(j.$$.fragment,e),q=s(e),r=m(e,"DIV",{class:!0});var d=ee(r);y(E.$$.fragment,d),oe=s(d),N=m(d,"P",{"data-svelte-h":!0}),_(N)!=="svelte-1sa5ela"&&(N.textContent=fe),ae=s(d),Z=m(d,"P",{"data-svelte-h":!0}),_(Z)!=="svelte-1xx6nm4"&&(Z.innerHTML=ue),se=s(d),G=m(d,"UL",{"data-svelte-h":!0}),_(G)!=="svelte-1nc06m8"&&(G.innerHTML=he),ie=s(d),y(M.$$.fragment,d),re=s(d),H=m(d,"P",{"data-svelte-h":!0}),_(H)!=="svelte-16nez6j"&&(H.innerHTML=_e),pe=s(d),O=m(d,"P",{"data-svelte-h":!0}),_(O)!=="svelte-cqt1it"&&(O.innerHTML=Te),d.forEach(n),A=s(e),y(z.$$.fragment,e),Y=s(e),W=m(e,"P",{}),ee(W).forEach(n),this.h()},h(){te(p,"name","hf:doc:metadata"),te(p,"content",je),te(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),te(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){c(document.head,p),o(e,k,t),o(e,h,t),o(e,g,t),w(f,e,t),o(e,i,t),w(u,e,t),o(e,L,t),o(e,P,t),o(e,R,t),o(e,x,t),o(e,D,t),o(e,C,t),o(e,F,t),w(I,e,t),o(e,Q,t),o(e,T,t),w(U,T,null),c(T,ne),c(T,V),o(e,S,t),w(j,e,t),o(e,q,t),o(e,r,t),w(E,r,null),c(r,oe),c(r,N),c(r,ae),c(r,Z),c(r,se),c(r,G),c(r,ie),w(M,r,null),c(r,re),c(r,H),c(r,pe),c(r,O),o(e,A,t),w(z,e,t),o(e,Y,t),o(e,W,t),K=!0},p(e,[t]){const B={};t&2&&(B.$$scope={dirty:t,ctx:e}),M.$set(B)},i(e){K||(v(f.$$.fragment,e),v(u.$$.fragment,e),v(I.$$.fragment,e),v(U.$$.fragment,e),v(j.$$.fragment,e),v(E.$$.fragment,e),v(M.$$.fragment,e),v(z.$$.fragment,e),K=!0)},o(e){$(f.$$.fragment,e),$(u.$$.fragment,e),$(I.$$.fragment,e),$(U.$$.fragment,e),$(j.$$.fragment,e),$(E.$$.fragment,e),$(M.$$.fragment,e),$(z.$$.fragment,e),K=!1},d(e){e&&(n(k),n(h),n(g),n(i),n(L),n(P),n(R),n(x),n(D),n(C),n(F),n(Q),n(T),n(S),n(q),n(r),n(A),n(Y),n(W)),n(p),J(f,e),J(u,e),J(I,e),J(U),J(j,e),J(E),J(M),J(z,e)}}}const je='{"title":"Prompt tuning","local":"prompt-tuning","sections":[{"title":"PromptTuningConfig","local":"peft.PromptTuningConfig","sections":[],"depth":2},{"title":"PromptEmbedding","local":"peft.PromptEmbedding","sections":[],"depth":2}],"depth":1}';function Ee(X){return we(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class He extends $e{constructor(p){super(),Je(this,p,Ee,Ue,ye,{})}}export{He as component};
