https://haruk1y.github.io/wiki/assets/js/5bd5f02d.85e6a133.js

1"use strict";(self.webpackChunkwiki=self.webpackChunkwiki||[]).push([["2064"],{94902(s,e,a){a.r(e),a.d(e,{metadata:()=>i,default:()=>d,frontMatter:()=>r,contentTitle:()=>t,toc:()=>c,assets:()=>m});var i=JSON.parse('{"id":"machine-learning/rlhf-and-alignment/rlhf-overview","title":"RLHF and Alignment Overview","description":"RLHF\u3001reward model\u3001DPO \u7CFB\u306E direct preference optimization\u3001Constitutional AI\u3001RLAIF \u306E\u5168\u4F53\u50CF\u3092\u6574\u7406\u3057\u307E\u3059\u3002","source":"@site/docs/machine-learning/rlhf-and-alignment/rlhf-overview.mdx","sourceDirName":"machine-learning/rlhf-and-alignment","slug":"/machine-learning/rlhf-and-alignment/rlhf-overview","permalink":"/wiki/machine-learning/rlhf-and-alignment/rlhf-overview","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":1,"frontMatter":{"sidebar_position":1,"title":"RLHF and Alignment Overview","description":"RLHF\u3001reward model\u3001DPO \u7CFB\u306E direct preference optimization\u3001Constitutional AI\u3001RLAIF \u306E\u5168\u4F53\u50CF\u3092\u6574\u7406\u3057\u307E\u3059\u3002"},"sidebar":"tutorialSidebar","previous":{"title":"RLHF and Alignment","permalink":"/wiki/category/rlhf-and-alignment"},"next":{"title":"Preference Data","permalink":"/wiki/machine-learning/rlhf-and-alignment/preference-data"}}'),n=a(74848),l=a(28453);let r={sidebar_position:1,title:"RLHF and Alignment Overview",description:"RLHF\u3001reward model\u3001DPO \u7CFB\u306E direct preference optimization\u3001Constitutional AI\u3001RLAIF \u306E\u5168\u4F53\u50CF\u3092\u6574\u7406\u3057\u307E\u3059\u3002"},t="RLHF and Alignment Overview",m={},c=[{value:"\u4F55\u3092\u89E3\u3044\u3066\u3044\u308B\u306E\u304B",id:"\u4F55\u3092\u89E3\u3044\u3066\u3044\u308B\u306E\u304B",level:2},{value:"\u4E09\u3064\u306E\u4E3B\u8981 family",id:"\u4E09\u3064\u306E\u4E3B\u8981-family",level:2},{value:"\u8A73\u7D30\u30DA\u30FC\u30B8",id:"\u8A73\u7D30\u30DA\u30FC\u30B8",level:2},{value:"\u306A\u305C DPO \u304C\u8A71\u984C\u306A\u306E\u304B",id:"\u306A\u305C-dpo-\u304C\u8A71\u984C\u306A\u306E\u304B",level:2},{value:"\u6570\u5F0F\u3067\u898B\u308B RLHF pipeline",id:"\u6570\u5F0F\u3067\u898B\u308B-rlhf-pipeline",level:2},{value:"\u95A2\u9023\u30DA\u30FC\u30B8",id:"\u95A2\u9023\u30DA\u30FC\u30B8",level:2},{value:"\u4E3B\u306A\u30BD\u30FC\u30B9",id:"\u4E3B\u306A\u30BD\u30FC\u30B9",level:2}];function h(s){let e={a:"a",annotation:"annotation",em:"em",h1:"h1",h2:"h2",header:"header",img:"img",li:"li",math:"math",mermaid:"mermaid",mi:"mi",mo:"mo",mrow:"mrow",msub:"msub",mtext:"mtext",munder:"munder",p:"p",semantics:"semantics",span:"span",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,l.R)(),...s.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(e.header,{children:(0,n.jsx)(e.h1,{id:"rlhf-and-alignment-overview",children:"RLHF and Alignment Overview"})}),"\n",(0,n.jsxs)(e.p,{children:["RLHF (Reinforcement Learning from Human Feedback) \u3068 alignment \u306F\u3001",(0,n.jsx)(e.strong,{children:"LLM \u3092\u4EBA\u9593\u306E\u597D\u307F\u3084\u5B89\u5168\u8981\u4EF6\u306B\u5408\u308F\u305B\u308B"})," \u305F\u3081\u306E post-training \u306E\u4E2D\u5FC3 topic \u3067\u3059\u3002InstructGPT \u4EE5\u964D\u3001chat \u7CFB LLM \u306E\u54C1\u8CEA\u3092\u652F\u3048\u308B\u4E8B\u5B9F\u4E0A\u306E\u6A19\u6E96\u306B\u306A\u308A\u307E\u3057\u305F\u3002"]}),"\n",(0,n.jsx)(e.p,{children:(0,n.jsx)(e.img,{alt:"RLHF / DPO map",src:a(74076).A+"",width:"1100",height:"500"})}),"\n",(0,n.jsx)(e.p,{children:(0,n.jsx)(e.em,{children:"\u81EA\u4F5C\u6982\u5FF5\u56F3\u3002Pretrained LLM \u2192 SFT \u2192 reward model / preference data \u2192 PPO / GRPO / DPO / IPO / KTO \u2192 aligned LLM\u3002"})}),"\n",(0,n.jsx)(e.h2,{id:"\u4F55\u3092\u89E3\u3044\u3066\u3044\u308B\u306E\u304B",children:"\u4F55\u3092\u89E3\u3044\u3066\u3044\u308B\u306E\u304B"}),"\n",(0,n.jsx)(e.p,{children:"LLM \u306E next-token prediction \u306F\u300C\u3082\u3063\u3068\u3082\u3042\u308A\u305D\u3046\u306A text\u300D\u3092\u751F\u6210\u3057\u307E\u3059\u304C\u3001\u5FC5\u305A\u3057\u3082\u300C\u4EBA\u9593\u306B\u3068\u3063\u3066\u597D\u307E\u3057\u3044 response\u300D\u3067\u306F\u3042\u308A\u307E\u305B\u3093\u3002Alignment \u306F\u3001"}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"\u6307\u793A\u306B\u5F93\u3046"}),"\n",(0,n.jsx)(e.li,{children:"\u5F79\u306B\u7ACB\u3064 (helpful)"}),"\n",(0,n.jsx)(e.li,{children:"\u5BB3\u304C\u306A\u3044 (harmless)"}),"\n",(0,n.jsx)(e.li,{children:"\u5618\u3092\u3064\u304B\u306A\u3044 (honest)"}),"\n",(0,n.jsx)(e.li,{children:"\u30D5\u30A9\u30FC\u30DE\u30C3\u30C8\u3092\u5B88\u308B"}),"\n"]}),"\n",(0,n.jsx)(e.p,{children:"\u3068\u3044\u3063\u305F\u6027\u8CEA\u3092\u5F8C\u5929\u7684\u306B\u6559\u3048\u8FBC\u307F\u307E\u3059\u3002"}),"\n",(0,n.jsx)(e.h2,{id:"\u4E09\u3064\u306E\u4E3B\u8981-family",children:"\u4E09\u3064\u306E\u4E3B\u8981 family"}),"\n",(0,n.jsx)(e.mermaid,{value:"flowchart TB\n  A[Alignment methods] --\x3e B[Classical RLHF]\n  A --\x3e C[Direct preference optimization]\n  A --\x3e D[Reasoning RL]\n  B --\x3e B1[Reward model + PPO]\n  C --\x3e C1[DPO / IPO / KTO / ORPO / SimPO]\n  D --\x3e D1[GRPO / DAPO / RLOO / verifiable rewards]"}),"\n",(0,n.jsxs)(e.table,{children:[(0,n.jsx)(e.thead,{children:(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.th,{children:"Family"}),(0,n.jsx)(e.th,{children:"\u7279\u5FB4"})]})}),(0,n.jsxs)(e.tbody,{children:[(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:"Classical RLHF"}),(0,n.jsx)(e.td,{children:"Reward model + PPO \u306E\u4E8C\u6BB5\u69CB\u6210"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:"Direct preference optimization"}),(0,n.jsx)(e.td,{children:"Reward model \u3092\u967D\u306B\u6301\u305F\u305A\u3001preference loss \u3092\u76F4\u63A5\u6700\u9069\u5316"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:"Reasoning RL"}),(0,n.jsx)(e.td,{children:"\u691C\u8A3C\u53EF\u80FD reward (math / code) \u3067 long CoT \u3092\u5F37\u5316"})]})]})]}),"\n",(0,n.jsx)(e.h2,{id:"\u8A73\u7D30\u30DA\u30FC\u30B8",children:"\u8A73\u7D30\u30DA\u30FC\u30B8"}),"\n",(0,n.jsxs)(e.table,{children:[(0,n.jsx)(e.thead,{children:(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.th,{children:"\u30DA\u30FC\u30B8"}),(0,n.jsx)(e.th,{children:"\u5185\u5BB9"})]})}),(0,n.jsxs)(e.tbody,{children:[(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/preference-data",children:"Preference Data"})}),(0,n.jsx)(e.td,{children:"Pairwise / ranking / point-wise \u306E preference \u53CE\u96C6"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/reward-model",children:"Reward Model"})}),(0,n.jsx)(e.td,{children:"Bradley-Terry\u3001reward model \u306E\u5B66\u7FD2\u3068\u843D\u3068\u3057\u7A74"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/rlhf-with-ppo",children:"RLHF with PPO"})}),(0,n.jsx)(e.td,{children:"InstructGPT \u7CFB\u306E RLHF pipeline"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/dpo",children:"DPO"})}),(0,n.jsx)(e.td,{children:"Direct Preference Optimization \u306E\u7406\u8AD6\u3068\u5B9F\u88C5"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/ipo-kto-orpo-simpo",children:"IPO, KTO, ORPO, SimPO"})}),(0,n.jsx)(e.td,{children:"DPO \u306E variants"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/iterative-and-online-dpo",children:"Iterative and Online DPO"})}),(0,n.jsx)(e.td,{children:"policy sampling \u3068 preference \u66F4\u65B0\u3092\u7E

170\u308A\u8FD4\u3059 DPO \u7CFB\u624B\u6CD5"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/grpo",children:"GRPO"})}),(0,n.jsx)(e.td,{children:"Group Relative Policy Optimization\u3001DeepSeek-R1 \u7CFB"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/dapo",children:"DAPO"})}),(0,n.jsx)(e.td,{children:"GRPO \u3092\u5927\u898F\u6A21 reasoning RL \u5411\u3051\u306B\u6539\u826F\u3057\u305F\u624B\u6CD5"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/constitutional-ai-and-rlaif",children:"Constitutional AI and RLAIF"})}),(0,n.jsx)(e.td,{children:"AI feedback \u306B\u3088\u308B alignment"})]}),(0,n.jsxs)(e.tr,{children:[(0,n.jsx)(e.td,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/reward-hacking-and-alignment-failures",children:"Reward Hacking and Alignment Failures"})}),(0,n.jsx)(e.td,{children:"Goodhart\u3001sycophancy\u3001jailbreak"})]})]})]}),"\n",(0,n.jsx)(e.h2,{id:"\u306A\u305C-dpo-\u304C\u8A71\u984C\u306A\u306E\u304B",children:"\u306A\u305C DPO \u304C\u8A71\u984C\u306A\u306E\u304B"}),"\n",(0,n.jsx)(e.p,{children:"PPO \u7CFB RLHF \u306F\u3001"}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsxs)(e.li,{children:["Reward model + value model + reference model + policy \u3067 ",(0,n.jsx)(e.strong,{children:"4 \u3064\u306E\u5927\u898F\u6A21 model"})," \u304C\u540C\u6642\u306B\u5FC5\u8981"]}),"\n",(0,n.jsx)(e.li,{children:"\u5B66\u7FD2\u304C\u4E0D\u5B89\u5B9A"}),"\n",(0,n.jsx)(e.li,{children:"Hyperparameter sensitive"}),"\n"]}),"\n",(0,n.jsxs)(e.p,{children:["\u3068\u3044\u3046\u6B20\u70B9\u304C\u3042\u308A\u307E\u3059\u3002DPO \u306F\u3001preference data \u3092 ",(0,n.jsx)(e.strong,{children:"policy \u30D1\u30E9\u30E1\u30FC\u30BF\u306E\u5206\u985E\u554F\u984C"})," \u3068\u3057\u3066\u76F4\u63A5\u6271\u3046\u3053\u3068\u3067\u3001"]}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Reward model \u4E0D\u8981"}),"\n",(0,n.jsx)(e.li,{children:"PPO \u4E0D\u8981"}),"\n",(0,n.jsx)(e.li,{children:"\u5B9F\u88C5\u304C\u30B7\u30F3\u30D7\u30EB\u3067\u5B89\u5B9A"}),"\n",(0,n.jsx)(e.li,{children:"\u30E1\u30E2\u30EA\u8981\u4EF6\u304C\u5927\u5E45\u306B\u524A\u6E1B"}),"\n"]}),"\n",(0,n.jsx)(e.p,{children:"\u3092\u5B9F\u73FE\u3057\u307E\u3057\u305F\u3002\u3053\u308C\u306B\u3088\u308A\u3001open-source LLM \u754C\u9688\u3067\u5E83\u304F\u4F7F\u308F\u308C\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3057\u305F\u3002"}),"\n",(0,n.jsx)(e.h2,{id:"\u6570\u5F0F\u3067\u898B\u308B-rlhf-pipeline",children:"\u6570\u5F0F\u3067\u898B\u308B RLHF pipeline"}),"\n",(0,n.jsxs)(e.p,{children:["RLHF \u306F\u3001pretrained policy ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"\u03C0"}),(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mi,{mathvariant:"normal",children:"S"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"F"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"T"})]})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\pi_{\\mathrm{SFT}}"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.5806em",verticalAlign:"-0.15em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"\u03C0"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3283em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0359em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mathrm mtight",children:"SFT"})})})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," \u3092\u3001\u4EBA\u9593\u306E preference \u304B\u3089\u5B66\u7FD2\u3057\u305F reward model ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"r"}),(0,n.jsx)(e.mi,{children:"\u03D5"})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"r_\\phi"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.7167em",verticalAlign:"-0.2861em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0278em"},children:"r"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"}

1,children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0278em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"\u03D5"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.2861em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," \u306B\u5408\u308F\u305B\u3066\u66F4\u65B0\u3059\u308B\u624B\u7D9A\u304D\u3068\u3057\u3066\u898B\u3089\u308C\u307E\u3059\u3002PPO \u578B\u3067\u306F\u3001policy ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"\u03C0"}),(0,n.jsx)(e.mi,{children:"\u03B8"})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\pi_\\theta"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.5806em",verticalAlign:"-0.15em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"\u03C0"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0359em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0278em"},children:"\u03B8"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," \u306F\u6B21\u306E objective \u3092\u6700\u5927\u5316\u3057\u307E\u3059\u3002"]}),"\n",(0,n.jsx)(e.span,{className:"katex-display",children:(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",display:"block",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsxs)(e.munder,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mi,{children:"max"}),(0,n.jsx)(e.mo,{children:"\u2061"})]}),(0,n.jsx)(e.mi,{children:"\u03B8"})]}),(0,n.jsx)(e.mtext,{children:"\u2005\u200A"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{mathvariant:"double-struck",children:"E"}),(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mi,{children:"y"}),(0,n.jsx)(e.mo,{children:"\u223C"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"\u03C0"}),(0,n.jsx)(e.mi,{children:"\u03B8"})]})]})]}),(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mo,{fence:"true",children:"["}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"r"}),(0,n.jsx)(e.mi,{children:"\u03D5"})]}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mi,{children:"y"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsx)(e.mi,{children:"\u03B2"}),(0,n.jsx)(e.mtext,{children:"\u2009"}),(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mi,{mathvariant:"normal",children:"K"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"L"})]}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"\u03C0"}),(0,n.jsx)(e.mi,{children:"\u03B8"})]}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mo,{children:"\u22C5"}),(0,n.jsx)(e.mo,{children:"\u2223"}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2225"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"\u03C0"}),(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mi,{mathvariant:"normal",children:"r"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"e"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"f"})]})]}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mo,{children:"\u22C5"}),(0,n.jsx)(e.mo,{children:"\u2223"}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mo,{fence:"true",children:"]"})]})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\max_\\theta\\;\\mathbb{E}_{x,y\\sim\\pi_\\theta}\\left[r_\\phi(x,y)\n-\\beta\\,\\mathrm{KL}(\\pi_\\theta(\\cdot\\mid x)\\|\\pi_{\\mathrm{ref}}(\\cdot\\mid x))\\right]"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1.5021em",verticalAlign:"-0.7521em"}}),(0,n.jsx)(e.span,{className:"mop op-limits",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsxs)(e.span,{className:"vlist",style:{height:"0.4306em"}

1,children:[(0,n.jsxs)(e.span,{style:{top:"-2.3479em",marginLeft:"0em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0278em"},children:"\u03B8"})})]}),(0,n.jsxs)(e.span,{style:{top:"-3em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{children:(0,n.jsx)(e.span,{className:"mop",children:"max"})})]})]}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.7521em"},children:(0,n.jsx)(e.span,{})})})]})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathbb",children:"E"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.1514em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsxs)(e.span,{className:"mord mtight",children:[(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"x"}),(0,n.jsx)(e.span,{className:"mpunct mtight",children:","}),(0,n.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0359em"},children:"y"}),(0,n.jsx)(e.span,{className:"mrel mtight",children:"\u223C"}),(0,n.jsxs)(e.span,{className:"mord mtight",children:[(0,n.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0359em"},children:"\u03C0"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3448em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.3488em",marginLeft:"-0.0359em",marginRight:"0.0714em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.5em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size3 size1 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0278em"},children:"\u03B8"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.1512em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.2861em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsxs)(e.span,{className:"minner",children:[(0,n.jsx)(e.span,{className:"mopen delimcenter",style:{top:"0em"},children:"["}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0278em"},children:"r"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0278em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"\u03D5"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.2861em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"y"}),(0,n.jsx)(e.span,{className:"mclose",children:")"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0528em"},children:"\u03B2"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsx)(e.span,{className:"mord mathrm",children:"KL"})}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"\u03C0"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"}

1,children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0359em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0278em"},children:"\u03B8"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord",children:"\u22C5"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mclose",children:")"}),(0,n.jsx)(e.span,{className:"mord",children:"\u2225"}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"\u03C0"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0359em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mathrm mtight",style:{marginRight:"0.0778em"},children:"ref"})})})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord",children:"\u22C5"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mclose",children:"))"}),(0,n.jsx)(e.span,{className:"mclose delimcenter",style:{top:"0em"},children:"]"})]})]})})]})}),"\n",(0,n.jsxs)(e.p,{children:["\u3053\u3053\u3067\u3001",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsx)(e.mi,{children:"x"})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"x"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.4306em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"})]})})]})," \u306F prompt\u3001",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsx)(e.mi,{children:"y"})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"y"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.625em",verticalAlign:"-0.1944em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"y"})]})})]})," \u306F response\u3001",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"\u03C0"}),(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mi,{mathvariant:"normal",children:"r"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"e"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"f"})]})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\pi_{\\mathrm{ref}}"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.5806em",verticalAlign:"-0.15em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"\u03C0"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"}

1,children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0359em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mathrm mtight",style:{marginRight:"0.0778em"},children:"ref"})})})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," \u306F\u53C2\u7167 policy \u3067\u3059\u3002\u3053\u306E\u5F0F\u306E\u6C17\u6301\u3061\u306F\u3001\u300Creward model \u304C\u597D\u3080\u56DE\u7B54\u3092\u5897\u3084\u3057\u305F\u3044\u304C\u3001\u5143\u306E model \u304B\u3089\u96E2\u308C\u3059\u304E\u308B\u3068\u6587\u6CD5\u3084\u5B89\u5168\u6027\u304C\u5D29\u308C\u308B\u306E\u3067 KL penalty \u3067\u6291\u3048\u308B\u300D\u3068\u3044\u3046\u3082\u306E\u3067\u3059\u3002"]}),"\n",(0,n.jsxs)(e.p,{children:["RLHF \u306E\u96E3\u3057\u3055\u306F\u3001",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"r"}),(0,n.jsx)(e.mi,{children:"\u03D5"})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"r_\\phi"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.7167em",verticalAlign:"-0.2861em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0278em"},children:"r"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.0278em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"\u03D5"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.2861em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," \u304C\u4EBA\u9593\u306E\u672C\u5F53\u306E\u4FA1\u5024\u95A2\u6570\u305D\u306E\u3082\u306E\u3067\u306F\u306A\u304F\u3001\u6709\u9650\u306E preference data \u304B\u3089\u5B66\u7FD2\u3055\u308C\u305F proxy \u3067\u3042\u308B\u70B9\u306B\u3042\u308A\u307E\u3059\u3002\u305D\u306E\u305F\u3081\u3001reward hacking \u3084 over-optimization \u3092\u9632\u3050\u306B\u306F\u3001KL\u3001\u8A55\u4FA1\u3001red teaming \u304C\u91CD\u8981\u306B\u306A\u308A\u307E\u3059\u3002"]}),"\n",(0,n.jsx)(e.h2,{id:"\u95A2\u9023\u30DA\u30FC\u30B8",children:"\u95A2\u9023\u30DA\u30FC\u30B8"}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/llm/llm-overview",children:"LLM Overview"})}),"\n",(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/llm/supervised-fine-tuning",children:"Supervised Fine-Tuning"})}),"\n",(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/reinforcement-learning/trpo-and-ppo",children:"TRPO and PPO"})}),"\n",(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/llm/reasoning-models",children:"Reasoning Models"})}),"\n",(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/reinforcement-learning/offline-rl",children:"Offline RL"})}),"\n",(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"/wiki/machine-learning/ai-agents/ai-agents-overview",children:"AI Agents Overview"})}),"\n"]}),"\n",(0,n.jsx)(e.h2,{id:"\u4E3B\u306A\u30BD\u30FC\u30B9",children:"\u4E3B\u306A\u30BD\u30FC\u30B9"}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsxs)(e.li,{children:["InstructGPT: ",(0,n.jsx)(e.a,{href:"https://arxiv.org/abs/2203.02155",children:"https://arxiv.org/abs/2203.02155"})]}),"\n",(0,n.jsxs)(e.li,{children:["DPO: ",(0,n.jsx)(e.a,{href:"https://arxiv.org/abs/2305.18290",children:"https://arxiv.org/abs/2305.18290"})]}),"\n",(0,n.jsxs)(e.li,{children:["Constitutional AI: ",(0,n.jsx)(e.a,{href:"https://arxiv.org/abs/2212.08073",children:"https://arxiv.org/abs/2212.08073"})]}),"\n",(0,n.jsxs)(e.li,{children:["DeepSeek-R1: ",(0,n.jsx)(e.a,{href:"https://arxiv.org/abs/2501.12948",children:"https://arxiv.org/abs/2501.12948"})]}),"\n",(0,n.jsxs)(e.li,{children:["DAPO: ",(0,n.jsx)(e.a,{href:"https://arxiv.org/abs/2503.14476",children:"https://arxiv.org/abs/2503.14476"})]}),"\n",(0,n.jsxs)(e.li,{children:["Note, LLM\u306E\u305F\u3081\u306E\u5F37\u5316\u5B66\u7FD2\u624B\u6CD5 2025: ",(0,n.jsx)(e.a,{href:"https://note.com/olachin/n/n9706c13c8678",children:"https://note.com/olachin/n/n9706c13c8678"})]}),"\n"]})]})}function d(s={}){let{wrapper:e}={...(0,l.R)(),...s.components};return e?(0,n.jsx)(e,{...s,children:(0,n.jsx)(h,{...s})}):h(s)}},74076(s,e,a){a.d(e,{A:()=>i});let i="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxMTAwIiBoZWlnaHQ9IjUwMCIgdmlld0JveD0iMCAwIDExMDAgNTAwIiByb2xlPSJpbWciIGFyaWEtbGFiZWw9IlJMSEYgYW5kIERQTyBtYXAiPgogIDxkZWZzPjxzdHlsZT4uYm94e2ZpbGw6I2ZmZjtzdHJva2U6IzMzNDE1NTtzdHJva2Utd2lkdGg6MjtyeDoxNX0uc2Z0e2ZpbGw6I2UwZjJmZTtzdHJva2U6IzAzNjlhMX0ucm17ZmlsbDojZjVmM2ZmO3N0cm9rZTojN2MzYWVkfS5ybHtmaWxsOiNmZmY3ZWQ7c3Ryb2tlOiNjMjQxMGN9LmRwb3tmaWxsOiNlY2ZkZjU7c3Ryb2tlOiMwNDc4NTd9LnR4dHtmb250LWZhbWlseTotYXBwbGUtc3lzdGVtLEJsaW5rTWFjU3lzdGVtRm9udCwnU2Vnb2

1UgVUknLHNhbnMtc2VyaWY7Zm9udC1zaXplOjE5cHg7ZmlsbDojMGYxNzJhO2ZvbnQtd2VpZ2h0OjcwMH0uc21hbGx7Zm9udC1zaXplOjE0cHg7Zm9udC13ZWlnaHQ6NTAwO2ZpbGw6IzMzNDE1NX0uYXJyb3d7c3Ryb2tlOiM0NzU1Njk7c3Ryb2tlLXdpZHRoOjIuNTtmaWxsOm5vbmU7bWFya2VyLWVuZDp1cmwoI2EpfTwvc3R5bGU+PG1hcmtlciBpZD0iYSIgbWFya2VyV2lkdGg9IjEwIiBtYXJrZXJIZWlnaHQ9IjEwIiByZWZYPSI4IiByZWZZPSIzIiBvcmllbnQ9ImF1dG8iPjxwYXRoIGQ9Ik0wLDAgTDAsNiBMOSwzIHoiIGZpbGw9IiM0NzU1NjkiLz48L21hcmtlcj48L2RlZnM+CiAgPHJlY3Qgd2lkdGg9IjExMDAiIGhlaWdodD0iNTAwIiBmaWxsPSIjZmFmYWZhIi8+CiAgPHJlY3QgY2xhc3M9ImJveCBzZnQiIHg9IjQwIiB5PSIxODAiIHdpZHRoPSIxODAiIGhlaWdodD0iMTMwIi8+PHRleHQgY2xhc3M9InR4dCIgeD0iMTMwIiB5PSIyMjUiIHRleHQtYW5jaG9yPSJtaWRkbGUiPlByZXRyYWluZWQgTExNPC90ZXh0Pjx0ZXh0IGNsYXNzPSJzbWFsbCIgeD0iMTMwIiB5PSIyNTUiIHRleHQtYW5jaG9yPSJtaWRkbGUiPmJhc2UgbW9kZWw8L3RleHQ+CiAgPHJlY3QgY2xhc3M9ImJveCBzZnQiIHg9IjI4MCIgeT0iMTgwIiB3aWR0aD0iMTgwIiBoZWlnaHQ9IjEzMCIvPjx0ZXh0IGNsYXNzPSJ0eHQiIHg9IjM3MCIgeT0iMjI1IiB0ZXh0LWFuY2hvcj0ibWlkZGxlIj5TRlQ8L3RleHQ+PHRleHQgY2xhc3M9InNtYWxsIiB4PSIzNzAiIHk9IjI1NSIgdGV4dC1hbmNob3I9Im1pZGRsZSI+aW5zdHJ1Y3Rpb24tdHVuZWQ8L3RleHQ+CiAgPHJlY3QgY2xhc3M9ImJveCBybSIgeD0iNTIwIiB5PSI1NSIgd2lkdGg9IjIwMCIgaGVpZ2h0PSIxMjAiLz48dGV4dCBjbGFzcz0idHh0IiB4PSI2MjAiIHk9IjEwMCIgdGV4dC1hbmNob3I9Im1pZGRsZSI+UmV3YXJkIG1vZGVsPC90ZXh0Pjx0ZXh0IGNsYXNzPSJzbWFsbCIgeD0iNjIwIiB5PSIxMzAiIHRleHQtYW5jaG9yPSJtaWRkbGUiPmZyb20gaHVtYW4gcGFpcnM8L3RleHQ+CiAgPHJlY3QgY2xhc3M9ImJveCBybCIgeD0iNTIwIiB5PSIyMDAiIHdpZHRoPSIyMDAiIGhlaWdodD0iMTIwIi8+PHRleHQgY2xhc3M9InR4dCIgeD0iNjIwIiB5PSIyNDUiIHRleHQtYW5jaG9yPSJtaWRkbGUiPlBQTyAvIEdSUE88L3RleHQ+PHRleHQgY2xhc3M9InNtYWxsIiB4PSI2MjAiIHk9IjI3NSIgdGV4dC1hbmNob3I9Im1pZGRsZSI+b25saW5lIFJMIHVwZGF0ZTwvdGV4dD4KICA8cmVjdCBjbGFzcz0iYm94IGRwbyIgeD0iNTIwIiB5PSIzNDUiIHdpZHRoPSIyMDAiIGhlaWdodD0iMTIwIi8+PHRleHQgY2xhc3M9InR4dCIgeD0iNjIwIiB5PSIzOTAiIHRleHQtYW5jaG9yPSJtaWRkbGUiPkRQTyAvIElQTyAvIEtUTzwvdGV4dD48dGV4dCBjbGFzcz0ic21hbGwiIHg9IjYyMCIgeT0iNDIwIiB0ZXh0LWFuY2hvcj0ibWlkZGxlIj5kaXJlY3QgcHJlZmVyZW5jZSBsb3NzPC90ZXh0PgogIDxyZWN0IGNsYXNzPSJib3ggc2Z0IiB4PSI4MDAiIHk9IjE4MCIgd2lkdGg9IjIyMCIgaGVpZ2h0PSIxMzAiLz48dGV4dCBjbGFzcz0idHh0IiB4PSI5MTAiIHk9IjIyNSIgdGV4dC1hbmNob3I9Im1pZGRsZSI+QWxpZ25lZCBMTE08L3RleHQ+PHRleHQgY2xhc3M9InNtYWxsIiB4PSI5MTAiIHk9IjI1NSIgdGV4dC1hbmNob3I9Im1pZGRsZSI+aGVscGZ1bCAvIGhhcm1sZXNzPC90ZXh0PgogIDxwYXRoIGNsYXNzPSJhcnJvdyIgZD0iTTIyMCAyNDUgSDI4MCIvPjxwYXRoIGNsYXNzPSJhcnJvdyIgZD0iTTQ2MCAyNDUgQzQ5MCAyNDUsNDkwIDExNSw1MjAgMTE1Ii8+PHBhdGggY2xhc3M9ImFycm93IiBkPSJNNDYwIDI0NSBINTIwIi8+PHBhdGggY2xhc3M9ImFycm93IiBkPSJNNDYwIDI0NSBDNDkwIDI0NSw0OTAgNDA1LDUyMCA0MDUiLz4KICA8cGF0aCBjbGFzcz0iYXJyb3ciIGQ9Ik03MjAgMTE1IEM3NjAgMTE1LDc2MCAyMzAsODAwIDI0MCIvPjxwYXRoIGNsYXNzPSJhcnJvdyIgZD0iTTcyMCAyNjAgSDgwMCIvPjxwYXRoIGNsYXNzPSJhcnJvdyIgZD0iTTcyMCA0MDUgQzc2MCA0MDUsNzYwIDI4MCw4MDAgMjcwIi8+Cjwvc3ZnPgo="},28453(s,e,a){a.d(e,{R:()=>r,x:()=>t});var i=a(96540);let n={},l=i.createContext(n);function r(s){let e=i.useContext(l);return i.useMemo(function(){return"function"==typeof s?s(e):{...e,...s}},[e,s])}function t(s){let e;return e=s.disableParentContext?"function"==typeof s.components?s.components(n):s.components||n:r(s.components),i.createElement(l.Provider,{value:e},s.children)}}}]);