1"use strict";(self.webpackChunkwiki=self.webpackChunkwiki||[]).push([["2884"],{86966(s,e,a){a.r(e),a.d(e,{metadata:()=>n,default:()=>d,frontMatter:()=>m,contentTitle:()=>r,toc:()=>c,assets:()=>t});var n=JSON.parse('{"id":"machine-learning/llm/reasoning-models","title":"Reasoning Models","description":"o1\u3001DeepSeek-R1\u3001long CoT\u3001test-time compute scaling\u3001reasoning RL \u3092\u6574\u7406\u3057\u307E\u3059\u3002","source":"@site/docs/machine-learning/llm/reasoning-models.mdx","sourceDirName":"machine-learning/llm","slug":"/machine-learning/llm/reasoning-models","permalink":"/wiki/machine-learning/llm/reasoning-models","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":12,"frontMatter":{"sidebar_position":12,"title":"Reasoning Models","description":"o1\u3001DeepSeek-R1\u3001long CoT\u3001test-time compute scaling\u3001reasoning RL \u3092\u6574\u7406\u3057\u307E\u3059\u3002"},"sidebar":"tutorialSidebar","previous":{"title":"Retrieval-Augmented Generation","permalink":"/wiki/machine-learning/llm/rag"},"next":{"title":"LLM Inference Optimization","permalink":"/wiki/machine-learning/llm/llm-inference-optimization"}}'),l=a(74848),i=a(28453);let m={sidebar_position:12,title:"Reasoning Models",description:"o1\u3001DeepSeek-R1\u3001long CoT\u3001test-time compute scaling\u3001reasoning RL \u3092\u6574\u7406\u3057\u307E\u3059\u3002"},r="Reasoning Models",t={},c=[{value:"\u4F55\u304C\u5909\u308F\u3063\u305F\u306E\u304B",id:"\u4F55\u304C\u5909\u308F\u3063\u305F\u306E\u304B",level:2},{value:"Test-time compute scaling",id:"test-time-compute-scaling",level:2},{value:"\u3069\u3046 training \u3059\u308B\u306E\u304B: Reasoning RL",id:"\u3069\u3046-training-\u3059\u308B\u306E\u304B-reasoning-rl",level:2},{value:"\u5229\u70B9\u3068\u6B20\u70B9",id:"\u5229\u70B9\u3068\u6B20\u70B9",level:2},{value:"Long CoT \u306E\u843D\u3068\u3057\u7A74",id:"long-cot-\u306E\u843D\u3068\u3057\u7A74",level:2},{value:"\u6570\u5F0F\u3067\u898B\u308B test-time compute scaling",id:"\u6570\u5F0F\u3067\u898B\u308B-test-time-compute-scaling",level:2},{value:"\u95A2\u9023\u30DA\u30FC\u30B8",id:"\u95A2\u9023\u30DA\u30FC\u30B8",level:2},{value:"\u4E3B\u306A\u30BD\u30FC\u30B9",id:"\u4E3B\u306A\u30BD\u30FC\u30B9",level:2}];function h(s){let e={a:"a",annotation:"annotation",h1:"h1",h2:"h2",header:"header",li:"li",math:"math",mermaid:"mermaid",mi:"mi",mn:"mn",mo:"mo",mover:"mover",mrow:"mrow",msub:"msub",munder:"munder",munderover:"munderover",p:"p",semantics:"semantics",span:"span",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,i.R)(),...s.components};return(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(e.header,{children:(0,l.jsx)(e.h1,{id:"reasoning-models",children:"Reasoning Models"})}),"\n",(0,l.jsxs)(e.p,{children:["Reasoning Models \u306F\u3001",(0,l.jsx)(e.strong,{children:"\u9577\u3044\u601D\u8003\u904E\u7A0B (long chain-of-thought)"})," \u3092\u5185\u90E8\u751F\u6210\u3057\u3066\u304B\u3089\u6700\u7D42\u7B54\u3092\u51FA\u3059\u3088\u3046\u306B\u7279\u5316\u3057\u305F LLM \u3067\u3059\u3002OpenAI o1\u3001DeepSeek-R1\u3001Qwen QwQ\u3001Gemini 2.5 Thinking \u306A\u3069\u304C\u4EE3\u8868\u7684\u3067\u3001\u8FD1\u5E74\u306E LLM \u7814\u7A76\u3067\u6700\u3082\u5927\u304D\u306A\u30C8\u30D4\u30C3\u30AF\u306E\u4E00\u3064\u3067\u3059\u3002"]}),"\n",(0,l.jsx)(e.h2,{id:"\u4F55\u304C\u5909\u308F\u3063\u305F\u306E\u304B",children:"\u4F55\u304C\u5909\u308F\u3063\u305F\u306E\u304B"}),"\n",(0,l.jsx)(e.p,{children:"\u901A\u5E38\u306E LLM \u306F\u3001prompt \u3092\u53D7\u3051\u53D6\u3063\u3066\u5373\u5EA7\u306B\u7B54\u3048\u3092\u51FA\u3057\u307E\u3059\u3002Reasoning model \u306F\u3001"}),"\n",(0,l.jsxs)(e.ul,{children:["\n",(0,l.jsxs)(e.li,{children:["\u63A8\u8AD6\u524D\u306B ",(0,l.jsx)(e.strong,{children:"\u6570\u5343\u301C\u6570\u4E07 token \u306E\u601D\u8003\u30D7\u30ED\u30BB\u30B9"})," \u3092\u751F\u6210"]}),"\n",(0,l.jsx)(e.li,{children:"\u81EA\u5DF1\u691C\u8A3C\u3001backtrack\u3001\u8A08\u753B\u4FEE\u6B63\u3092\u542B\u3080"}),"\n",(0,l.jsx)(e.li,{children:"\u7D50\u679C\u3068\u3057\u3066 math\u3001code\u3001science\u3001agent \u7CFB task \u3067\u5927\u304D\u304F\u6027\u80FD\u5411\u4E0A"}),"\n"]}),"\n",(0,l.jsx)(e.mermaid,{value:"flowchart LR\n Q[Question] --\x3e T[Internal thinking: long CoT]\n T --\x3e V[Self-verify / backtrack]\n V --\x3e T\n V --\x3e A[Final answer]"}),"\n",(0,l.jsx)(e.h2,{id:"test-time-compute-scaling",children:"Test-time compute scaling"}),"\n",(0,l.jsxs)(e.p,{children:["Reasoning model \u306E\u9375\u306F ",(0,l.jsx)(e.strong,{children:"inference time \u306B\u8A08\u7B97\u3092\u6295\u5165\u3059\u308B"})," \u3053\u3068\u3067\u3059\u3002\u30B9\u30B1\u30FC\u30EA\u30F3\u30B0\u306E\u8EF8\u304C\u5897\u3048\u307E\u3057\u305F\u3002"]}),"\n",(0,l.jsx)(e.mermaid,{value:"flowchart LR\n S1[Train compute] --\x3e Q[Quality]\n S2[Inference tokens] --\x3e Q\n S3[Search / parallel sampling] --\x3e Q"}),"\n",(0,l.jsxs)(e.ul,{children:["\n",(0,l.jsx)(e.li,{children:"\u540C\u3058 model \u3067\u3082\u3001\u601D\u8003 token \u3092\u5897\u3084\u3059\u307B\u3069\u7CBE\u5EA6\u304C\u4F38\u3073\u308B"}),"\n",(0,l.jsx)(e.li,{children:"Self-consistency\u3001tree search\u3001program-aided reasoning \u3068\u7D44\u307F\u5408\u308F\u305B\u308B"}),"\n"]}),"\n",(0,l.jsx)(e.h2,{id:"\u3069\u3046-training-\u3059\u308B\u306E\u304B-reasoning-rl",children:"\u3069\u3046 training \u3059\u308B\u306E\u304B: Reasoning RL"}),"\n",(0,l.jsx)(e.p,{children:"DeepSeek-R1 \u304C\u516C\u958B\u3057\u305F\u30EC\u30B7\u30D4\u304C\u5927\u304D\u306A\u5F71\u97FF\u3092\u4E0E\u3048\u307E\u3057\u305F\u3002"}),"\n",(0,l.jsx)(e.mermaid,{value:"flowchart LR\n P[Pretrained base] --\x3e S[Optional SFT]\n S --\x3e R[RL with verifiable rewards]\n R --\x3e M[Reasoning model]"}),"\n",(0,l.jsxs)(e.ul,{children:["\n",(0,l.jsx)(e.li,{children:"\u691C\u8A3C\u53EF\u80FD\u306A reward (math \u6B63\u8AA4\u3001code unit test \u901A\u904E) \u3092\u4F7F\u3046"}),"\n",(0,l.jsxs)(e.li,{children:[(0,l.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/grpo",children:"GRPO"})," \u306A\u3069 policy gradient \u3067\u9577 CoT \u3092\u5F37\u5316"]}),"\n",(0,l.jsx)(e.li,{children:'"Aha moment" \u7684\u306A\u81EA\u5DF1\u53CD\u7701\u304C emergent \u306B\u73FE\u308C\u308B'}),"\n"]}),"\n",(0,l.jsx)(e.h2,{id:"\u5229\u70B9\u3068\u6B20\u70B9",children:"\u5229\u70B9\u3068\u6B20\u70B9"}),"\n",(0,l.jsxs)(e.table,{children:[(0,l.jsx)(e.thead,{children:(0,l.jsxs)(e.tr,{children:[(0,l.jsx)(e.th,{children:"\u89B3\u70B9"}),(0,l.jsx)(e.th,{children:"\u5229\u70B9"}),(0,l.jsx)(e.th,{children:"\u6B20\u70B9"})]})}),(0,l.jsxs)(e.tbody,{children:[(0,l.jsxs)(e.tr,{children:[(0,l.jsx)(e.td,{children:"\u6027\u80FD"}),(0,l.jsx)(e.td,{children:"Math / code / \u8AD6\u7406\u3067\u5927\u5E45\u5411\u4E0A"}),(0,l.jsx)(e.td,{children:"\u96D1\u8AC7\u30FBcreative \u3067\u306F\u904E\u5270"})]}),(0,l.jsxs)(e.tr,{children:[(0,l.jsx)(e.td,{children:"\u5236\u5FA1"}),(0,l.jsx)(e.td,{children:"\u601D\u8003\u91CF\u3067\u7CBE\u5EA6\u3092\u8ABF\u6574\u53EF\u80FD"}),(0,l.jsx)(e.td,{children:"\u63A8\u8AD6\u30B3\u30B9\u30C8\u304C\u7DDA\u5F62\u306B\u5897\u3048\u308B"})]}),(0,l.jsxs)(e.tr,{children:[(0,l.jsx)(e.td,{children:"\u900F\u660E\u6027"}),(0,l.jsx)(e.td,{children:"\u601D\u8003\u904E\u7A0B\u3092 inspect \u53EF\u80FD"}),(0,l.jsx)(e.td,{children:"\u601D\u8003\u3068\u6700\u7D42\u7B54\u304C\u4E56\u96E2\u3059\u308B\u3053\u3068\u304C\u3042\u308B"})]}),(0,l.jsxs)(e.tr,{children:[(0,l.jsx)(e.td,{children:"Agent"}),(0,l.jsx)(e.td,{children:"\u81EA\u5DF1\u8A08\u753B / \u53CD\u7701\u3068\u76F8\u6027\u304C\u826F\u3044"}),(0,l.jsx)(e.td,{children:"\u30EC\u30A4\u30C6\u30F3\u30B7\u304C\u4F38\u3073\u308B"})]})]})]}),"\n",(0,l.jsx)(e.h2,{id:"long-cot-\u306E\u843D\u3068\u3057\u7A74",children:"Long CoT \u306E\u843D\u3068\u3057\u7A74"}),"\n",(0,l.jsxs)(e.ul,{children:["\n",(0,l.jsx)(e.li,{children:"\u81EA\u5206\u306E\u8AA4\u308A\u3092\u6B63\u5F53\u5316\u3057\u7D9A\u3051\u308B (false reasoning lock-in)"}),"\n",(0,l.jsx)(e.li,{children:"\u7B54\u3048\u306B\u5230\u9054\u5F8C\u3082\u7121\u99C4\u306B\u601D\u8003\u3057\u7D9A\u3051\u308B"}),"\n",(0,l.jsx)(e.li,{children:"\u591A\u8A00\u8A9E mixing"}),"\n",(0,l.jsx)(e.li,{children:"Prompt injection \u304C\u601D\u8003\u9014\u4E2D\u3067\u6DF7\u5165"}),"\n"]}),"\n",(0,l.jsx)(e.h2,{id:"\u6570\u5F0F\u3067\u898B\u308B-test-time-compute-scaling",children:"\u6570\u5F0F\u3067\u898B\u308B test-time compute scaling"}),"\n",(0,l.jsxs)(e.p,{children:["Reasoning model \u306F\u3001\u6700\u7D42\u56DE\u7B54 ",(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsx)(e.mrow,{children:(0,l.jsx)(e.mi,{children:"a"})}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"a"})]})})}),(0,l.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"0.4306em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"})]})})]})," \u306E\u524D\u306B\u5185\u90E8 reasoning trace ",(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsx)(e.mrow,{children:(0,l.jsx)(e.mi,{children:"z"})}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"z"})]})})}),(0,l.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"0.4306em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.044em"},children:"z"})]})})]})," \u3092\u751F\u6210\u3057\u307E\u3059\u3002\u901A\u5E38\u306E model \u304C ",(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{children:"p"}),(0,l.jsx)(e.mo,{stretchy:"false",children:"("}),(0,l.jsx)(e.mi,{children:"a"}),(0,l.jsx)(e.mo,{children:"\u2223"}),(0,l.jsx)(e.mi,{children:"x"}),(0,l.jsx)(e.mo,{stretchy:"false",children:")"})]}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"p(a\\mid x)"})]})})}),(0,l.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"p"}),(0,l.jsx)(e.span,{className:"mopen",children:"("}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,l.jsx)(e.span,{className:"mclose",children:")"})]})]})]})," \u3092\u76F4\u63A5\u51FA\u3059\u306E\u306B\u5BFE\u3057\u3001reasoning model \u306F\u6B21\u306E\u5468\u8FBA\u5316\u3068\u3057\u3066\u898B\u3089\u308C\u307E\u3059\u3002"]}),"\n",(0,l.jsx)(e.span,{className:"katex-display",children:(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",display:"block",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{children:"p"}),(0,l.jsx)(e.mo,{stretchy:"false",children:"("}),(0,l.jsx)(e.mi,{children:"a"}),(0,l.jsx)(e.mo,{children:"\u2223"}),(0,l.jsx)(e.mi,{children:"x"}),(0,l.jsx)(e.mo,{stretchy:"false",children:")"}),(0,l.jsx)(e.mo,{children:"="}),(0,l.jsxs)(e.munder,{children:[(0,l.jsx)(e.mo,{children:"\u2211"}),(0,l.jsx)(e.mi,{children:"z"})]}),(0,l.jsxs)(e.msub,{children:[(0,l.jsx)(e.mi,{children:"p"}),(0,l.jsx)(e.mi,{children:"\u03B8"})]}),(0,l.jsx)(e.mo,{stretchy:"false",children:"("}),(0,l.jsx)(e.mi,{children:"a"}),(0,l.jsx)(e.mo,{children:"\u2223"}),(0,l.jsx)(e.mi,{children:"x"}),(0,l.jsx)(e.mo,{separator:"true",children:","}),(0,l.jsx)(e.mi,{children:"z"}),(0,l.jsx)(e.mo,{stretchy:"false",children:")"}),(0,l.jsxs)(e.msub,{children:[(0,l.jsx)(e.mi,{children:"p"}),(0,l.jsx)(e.mi,{children:"\u03B8"})]}),(0,l.jsx)(e.mo,{stretchy:"false",children:"("}),(0,l.jsx)(e.mi,{children:"z"}),(0,l.jsx)(e.mo,{children:"\u2223"}),(0,l.jsx)(e.mi,{children:"x"}),(0,l.jsx)(e.mo,{stretchy:"false",children:")"})]}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"p(a\\mid x)=\\sum_z p_\\theta(a\\mid x,z)p_\\theta(z\\mid x)"})]})})}),(0,l.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"p"}),(0,l.jsx)(e.span,{className:"mopen",children:"("}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,l.jsx)(e.span,{className:"mclose",children:")"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"="}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"2.3em",verticalAlign:"-1.25em"}}),(0,l.jsx)(e.span,{className:"mop op-limits",children:(0,l.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,l.jsxs)(e.span,{className:"vlist-r",children:[(0,l.jsxs)(e.span,{className:"vlist",style:{height:"1.05em"}
1,children:[(0,l.jsxs)(e.span,{style:{top:"-1.9em",marginLeft:"0em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3.05em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.044em"},children:"z"})})]}),(0,l.jsxs)(e.span,{style:{top:"-3.05em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3.05em"}}),(0,l.jsx)(e.span,{children:(0,l.jsx)(e.span,{className:"mop op-symbol large-op",children:"\u2211"})})]})]}),(0,l.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsx)(e.span,{className:"vlist",style:{height:"1.25em"},children:(0,l.jsx)(e.span,{})})})]})}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsxs)(e.span,{className:"mord",children:[(0,l.jsx)(e.span,{className:"mord mathnormal",children:"p"}),(0,l.jsx)(e.span,{className:"msupsub",children:(0,l.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,l.jsxs)(e.span,{className:"vlist-r",children:[(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,l.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0278em"},children:"\u03B8"})})]})}),(0,l.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,l.jsx)(e.span,{})})})]})})]}),(0,l.jsx)(e.span,{className:"mopen",children:"("}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,l.jsx)(e.span,{className:"mpunct",children:","}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.044em"},children:"z"}),(0,l.jsx)(e.span,{className:"mclose",children:")"}),(0,l.jsxs)(e.span,{className:"mord",children:[(0,l.jsx)(e.span,{className:"mord mathnormal",children:"p"}),(0,l.jsx)(e.span,{className:"msupsub",children:(0,l.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,l.jsxs)(e.span,{className:"vlist-r",children:[(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,l.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0278em"},children:"\u03B8"})})]})}),(0,l.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,l.jsx)(e.span,{})})})]})})]}),(0,l.jsx)(e.span,{className:"mopen",children:"("}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.044em"},children:"z"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,l.jsx)(e.span,{className:"mclose",children:")"})]})]})]})}),"\n",(0,l.jsxs)(e.p,{children:["\u5B9F\u969B\u306B\u306F\u5168\u3066\u306E ",(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsx)(e.mrow,{children:(0,l.jsx)(e.mi,{children:"z"})}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"z"})]})})}),(0,l.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"0.4306em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.044em"},children:"z"})]})})]})," \u3092\u8DB3\u3057\u4E0A\u3052\u3089\u308C\u306A\u3044\u305F\u3081\u3001\u8907\u6570\u306E reasoning trace \u3092 sample \u3057\u3066\u3001\u6700\u7D42\u56DE\u7B54\u3092 voting \u3057\u307E\u3059\u3002"]}),"\n",(0,l.jsx)(e.span,{className:"katex-display",children:(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",display:"block",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsxs)(e.mrow,{children:[(0,l.jsxs)(e.mover,{accent:"true",children:[(0,l.jsx)(e.mi,{children:"a"}),(0,l.jsx)(e.mo,{children:"^"})]}),(0,l.jsx)(e.mo,{children:"="}),(0,l.jsx)(e.mi,{children:"arg"}),(0,l.jsx)(e.mo,{children:"\u2061"}),(0,l.jsxs)(e.munder,{children:[(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{children:"max"}),(0,l.jsx)(e.mo,{children:"\u2061"})]}),(0,l.jsx)(e.mi,{children:"a"})]}),(0,l.jsxs)(e.munderover,{children:[(0,l.jsx)(e.mo,{children:"\u2211"}),(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{children:"k"}),(0,l.jsx)(e.mo,{children:"="}),(0,l.jsx)(e.mn,{children:"1"})]}),(0,l.jsx)(e.mi,{children:"K"})]}),(0,l.jsx)(e.mn,{mathvariant:"bold",children:"1"}),(0,l.jsx)(e.mo,{stretchy:"false",children:"["}),(0,l.jsxs)(e.msub,{children:[(0,l.jsx)(e.mi,{children:"a"}),(0,l.jsx)(e.mi,{children:"k"})]}),(0,l.jsx)(e.mo,{children:"="}),(0,l.jsx)(e.mi,{children:"a"}),(0,l.jsx)(e.mo,{stretchy:"false",children:"]"})]}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\hat{a}=\\arg\\max_a \\sum_{k=1}^{K}\\mathbf{1}[a_k=a]"})]})})}),(0,l.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"0.6944em"}}),(0,l.jsx)(e.span,{className:"mord accent",children:(0,l.jsx)(e.span,{className:"vlist-t",children:(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsxs)(e.span,{className:"vlist",style:{height:"0.6944em"}
1,children:[(0,l.jsxs)(e.span,{style:{top:"-3em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"})]}),(0,l.jsxs)(e.span,{style:{top:"-3em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,l.jsx)(e.span,{className:"accent-body",style:{left:"-0.25em"},children:(0,l.jsx)(e.span,{className:"mord",children:"^"})})]})]})})})}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"="}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"3.1304em",verticalAlign:"-1.3021em"}}),(0,l.jsxs)(e.span,{className:"mop",children:["ar",(0,l.jsx)(e.span,{style:{marginRight:"0.0139em"},children:"g"})]}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsx)(e.span,{className:"mop op-limits",children:(0,l.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,l.jsxs)(e.span,{className:"vlist-r",children:[(0,l.jsxs)(e.span,{className:"vlist",style:{height:"0.4306em"},children:[(0,l.jsxs)(e.span,{style:{top:"-2.4em",marginLeft:"0em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsx)(e.span,{className:"mord mathnormal mtight",children:"a"})})]}),(0,l.jsxs)(e.span,{style:{top:"-3em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,l.jsx)(e.span,{children:(0,l.jsx)(e.span,{className:"mop",children:"max"})})]})]}),(0,l.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.7em"},children:(0,l.jsx)(e.span,{})})})]})}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsx)(e.span,{className:"mop op-limits",children:(0,l.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,l.jsxs)(e.span,{className:"vlist-r",children:[(0,l.jsxs)(e.span,{className:"vlist",style:{height:"1.8283em"},children:[(0,l.jsxs)(e.span,{style:{top:"-1.8479em",marginLeft:"0em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3.05em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsxs)(e.span,{className:"mord mtight",children:[(0,l.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0315em"},children:"k"}),(0,l.jsx)(e.span,{className:"mrel mtight",children:"="}),(0,l.jsx)(e.span,{className:"mord mtight",children:"1"})]})})]}),(0,l.jsxs)(e.span,{style:{top:"-3.05em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3.05em"}}),(0,l.jsx)(e.span,{children:(0,l.jsx)(e.span,{className:"mop op-symbol large-op",children:"\u2211"})})]}),(0,l.jsxs)(e.span,{style:{top:"-4.3em",marginLeft:"0em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"3.05em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsx)(e.span,{className:"mord mtight",children:(0,l.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0715em"},children:"K"})})})]})]}),(0,l.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsx)(e.span,{className:"vlist",style:{height:"1.3021em"},children:(0,l.jsx)(e.span,{})})})]})}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsx)(e.span,{className:"mord mathbf",children:"1"}),(0,l.jsx)(e.span,{className:"mopen",children:"["}),(0,l.jsxs)(e.span,{className:"mord",children:[(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"}),(0,l.jsx)(e.span,{className:"msupsub",children:(0,l.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,l.jsxs)(e.span,{className:"vlist-r",children:[(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.3361em"},children:(0,l.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,l.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,l.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,l.jsx)(e.span,{className:"mord mathnormal mtight",style:{marginRight:"0.0315em"},children:"k"})})]})}),(0,l.jsx)(e.span,{className:"vlist-s",children:"\u200B"})]}),(0,l.jsx)(e.span,{className:"vlist-r",children:(0,l.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,l.jsx)(e.span,{})})})]})})]}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"="}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"a"}),(0,l.jsx)(e.span,{className:"mclose",children:"]"})]})]})]})}),"\n",(0,l.jsxs)(e.p,{children:["\u3053\u306E\u5F0F\u306E\u6C17\u6301\u3061\u306F\u3001\u300C\u4E00\u56DE\u306E\u601D\u8003\u3067\u7B54\u3048\u308B\u306E\u3067\u306F\u306A\u304F\u3001\u8907\u6570\u306E\u601D\u8003\u7D4C\u8DEF\u3092\u8A66\u3057\u3001\u6700\u3082\u4E00\u8CAB\u3057\u3066\u51FA\u308B\u7B54\u3048\u3092\u9078\u3076\u300D\u3068\u3044\u3046\u3082\u306E\u3067\u3059\u3002",(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsx)(e.mrow,{children:(0,l.jsx)(e.mi,{children:"K"})}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"K"})]})})}),(0,l.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0715em"},children:"K"})]})})]})," \u3092\u5897\u3084\u3059\u307B\u3069 test-time compute \u306F\u5897\u3048\u307E\u3059\u304C\u3001math \u3084
1code \u306E\u3088\u3046\u306B\u691C\u8A3C\u53EF\u80FD\u306A task \u3067\u306F\u7CBE\u5EA6\u304C\u4F38\u3073\u3084\u3059\u304F\u306A\u308A\u307E\u3059\u3002"]}),"\n",(0,l.jsx)(e.p,{children:"Verifiable reward \u3092\u4F7F\u3046 reasoning RL \u3067\u306F\u3001\u56DE\u7B54\u304C unit test \u3084 exact match \u306B\u901A\u308B\u304B\u3092 reward \u306B\u3057\u307E\u3059\u3002"}),"\n",(0,l.jsx)(e.span,{className:"katex-display",children:(0,l.jsxs)(e.span,{className:"katex",children:[(0,l.jsx)(e.span,{className:"katex-mathml",children:(0,l.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",display:"block",children:(0,l.jsxs)(e.semantics,{children:[(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{children:"R"}),(0,l.jsx)(e.mo,{stretchy:"false",children:"("}),(0,l.jsx)(e.mi,{children:"x"}),(0,l.jsx)(e.mo,{separator:"true",children:","}),(0,l.jsx)(e.mi,{children:"y"}),(0,l.jsx)(e.mo,{stretchy:"false",children:")"}),(0,l.jsx)(e.mo,{children:"="}),(0,l.jsx)(e.mn,{mathvariant:"bold",children:"1"}),(0,l.jsx)(e.mo,{stretchy:"false",children:"["}),(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{mathvariant:"normal",children:"V"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"e"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"r"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"i"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"f"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"y"})]}),(0,l.jsx)(e.mo,{stretchy:"false",children:"("}),(0,l.jsx)(e.mi,{children:"x"}),(0,l.jsx)(e.mo,{separator:"true",children:","}),(0,l.jsx)(e.mi,{children:"y"}),(0,l.jsx)(e.mo,{stretchy:"false",children:")"}),(0,l.jsx)(e.mo,{children:"="}),(0,l.jsxs)(e.mrow,{children:[(0,l.jsx)(e.mi,{mathvariant:"normal",children:"t"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"r"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"u"}),(0,l.jsx)(e.mi,{mathvariant:"normal",children:"e"})]}),(0,l.jsx)(e.mo,{stretchy:"false",children:"]"})]}),(0,l.jsx)(e.annotation,{encoding:"application/x-tex",children:"R(x,y)=\\mathbf{1}[\\mathrm{Verify}(x,y)=\\mathrm{true}]"})]})})}),(0,l.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0077em"},children:"R"}),(0,l.jsx)(e.span,{className:"mopen",children:"("}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,l.jsx)(e.span,{className:"mpunct",children:","}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"y"}),(0,l.jsx)(e.span,{className:"mclose",children:")"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"="}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord mathbf",children:"1"}),(0,l.jsx)(e.span,{className:"mopen",children:"["}),(0,l.jsx)(e.span,{className:"mord",children:(0,l.jsx)(e.span,{className:"mord mathrm",style:{marginRight:"0.0139em"},children:"Verify"})}),(0,l.jsx)(e.span,{className:"mopen",children:"("}),(0,l.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,l.jsx)(e.span,{className:"mpunct",children:","}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,l.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.0359em"},children:"y"}),(0,l.jsx)(e.span,{className:"mclose",children:")"}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,l.jsx)(e.span,{className:"mrel",children:"="}),(0,l.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,l.jsxs)(e.span,{className:"base",children:[(0,l.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,l.jsx)(e.span,{className:"mord",children:(0,l.jsx)(e.span,{className:"mord mathrm",children:"true"})}),(0,l.jsx)(e.span,{className:"mclose",children:"]"})]})]})]})}),"\n",(0,l.jsx)(e.p,{children:"\u3053\u306E\u3088\u3046\u306A reward \u306F\u4EBA\u9593\u306E\u597D\u307F\u3088\u308A\u3082\u
1660E\u78BA\u306A\u306E\u3067\u3001\u9577\u3044 reasoning trace \u306E\u5F37\u5316\u5B66\u7FD2\u306B\u4F7F\u3044\u3084\u3059\u3044\u3068\u3044\u3046\u5229\u70B9\u304C\u3042\u308A\u307E\u3059\u3002"}),"\n",(0,l.jsx)(e.h2,{id:"\u95A2\u9023\u30DA\u30FC\u30B8",children:"\u95A2\u9023\u30DA\u30FC\u30B8"}),"\n",(0,l.jsxs)(e.ul,{children:["\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/llm/llm-overview",children:"LLM Overview"})}),"\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/llm/in-context-learning-and-prompting",children:"In-Context Learning and Prompting"})}),"\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/llm/sampling-strategies",children:"Sampling Strategies"})}),"\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/grpo",children:"GRPO"})}),"\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/iterative-and-online-dpo",children:"Iterative and Online DPO"})}),"\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/rlhf-and-alignment/rlhf-overview",children:"RLHF Overview"})}),"\n",(0,l.jsx)(e.li,{children:(0,l.jsx)(e.a,{href:"/wiki/machine-learning/ai-agents/ai-agents-overview",children:"AI Agents Overview"})}),"\n"]}),"\n",(0,l.jsx)(e.h2,{id:"\u4E3B\u306A\u30BD\u30FC\u30B9",children:"\u4E3B\u306A\u30BD\u30FC\u30B9"}),"\n",(0,l.jsxs)(e.ul,{children:["\n",(0,l.jsxs)(e.li,{children:["OpenAI o1 system card: ",(0,l.jsx)(e.a,{href:"https://openai.com/index/openai-o1-system-card/",children:"https://openai.com/index/openai-o1-system-card/"})]}),"\n",(0,l.jsxs)(e.li,{children:["DeepSeek-R1 paper: ",(0,l.jsx)(e.a,{href:"https://arxiv.org/abs/2501.12948",children:"https://arxiv.org/abs/2501.12948"})]}),"\n",(0,l.jsxs)(e.li,{children:["Let's Verify Step by Step (PRM): ",(0,l.jsx)(e.a,{href:"https://arxiv.org/abs/2305.20050",children:"https://arxiv.org/abs/2305.20050"})]}),"\n",(0,l.jsxs)(e.li,{children:["Scaling LLM Test-Time Compute: ",(0,l.jsx)(e.a,{href:"https://arxiv.org/abs/2408.03314",children:"https://arxiv.org/abs/2408.03314"})]}),"\n"]})]})}function d(s={}){let{wrapper:e}={...(0,i.R)(),...s.components};return e?(0,l.jsx)(e,{...s,children:(0,l.jsx)(h,{...s})}):h(s)}},28453(s,e,a){a.d(e,{R:()=>m,x:()=>r});var n=a(96540);let l={},i=n.createContext(l);function m(s){let e=n.useContext(i);return n.useMemo(function(){return"function"==typeof s?s(e):{...e,...s}},[e,s])}function r(s){let e;return e=s.disableParentContext?"function"==typeof s.components?s.components(l):s.components||l:m(s.components),n.createElement(i.Provider,{value:e},s.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.