contrastive-learning-a105d81dd357688c.js

js abhik.ai collected 2026-10-03 23:05:34 UTC 51,802 bytes, 1 lines download raw bytes

1(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[60240],{9482:(e,n,s)=>{"use strict";s.r(n),s.d(n,{ContrastiveLearning:()=>p,__N_SSG:()=>d,default:()=>x,meta:()=>h});var a=s(37876),i=s(91668),l=s(12627),c=s(18847),t=s.n(c),r=s(40028),o=s(14356),d=!0;let p=t()(()=>s.e(39120).then(s.bind(s,39120)),{loadableGenerated:{webpack:()=>[39120]},ssr:!1,loading:r.A}),h={title:"Contrastive Learning",description:"Master contrastive learning for vector embeddings: how InfoNCE loss and self-supervised techniques train models to create high-quality semantic representations.",category:"embeddings",order:5,author:"Abhik Sarkar",date:"2026-06-02",readingTime:"10 min",difficulty:"advanced",prerequisites:["deep-learning/contrastive-loss","embeddings/dense-embeddings"],tags:["contrastive-learning","self-supervised","representation-learning","infonce","simclr"],relatedConcepts:["dense-embeddings","cross-encoder-vs-bi-encoder","domain-adaptation"],keywords:["contrastive learning","InfoNCE loss","SimCLR","self-supervised learning","positive pairs","negative sampling","temperature parameter","representation learning"],slug:"embeddings/contrastive-learning"},m=e=>(0,a.jsx)(l.b,Object.assign({meta:h},e));function u(e){let n=Object.assign({p:"p",h2:"h2",a:"a",h3:"h3",ol:"ol",li:"li",strong:"strong",ul:"ul",code:"code",pre:"pre",span:"span",table:"table",thead:"thead",tr:"tr",th:"th",tbody:"tbody",td:"td",em:"em"},(0,i.RP)(),e.components),{ErrorBoundary:s,RelatedConceptsGrid:l}=n;return s||j("ErrorBoundary",!0),l||j("RelatedConceptsGrid",!0),(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.p,{children:"Contrastive learning has revolutionized self-supervised representation learning by teaching models to distinguish between similar and dissimilar samples without explicit labels. This approach powers systems like CLIP, SimCLR, and modern embedding models."}),"\n",(0,a.jsx)(n.h2,{id:"interactive-learning-visualization",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#interactive-learning-visualization",children:"Interactive Learning Visualization"})}),"\n",(0,a.jsx)(s,{children:(0,a.jsx)(p,{})}),"\n",(0,a.jsx)(n.h2,{id:"core-principles",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#core-principles",children:"Core Principles"})}),"\n",(0,a.jsx)(n.h3,{id:"the-contrastive-objective",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#the-contrastive-objective",children:"The Contrastive Objective"})}),"\n",(0,a.jsx)(n.p,{children:"Contrastive learning optimizes representations by:"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Pulling Together"}),": Positive pairs (augmentations of the same sample)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Pushing Apart"}),": Negative pairs (different samples)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Learning Invariances"}),": Robust features across transformations"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"infonce-loss",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#infonce-loss",children:"InfoNCE Loss"})}),"\n",(0,a.jsx)(n.p,{children:"The InfoNCE (Noise Contrastive Estimation) loss is the foundation:"}),"\n",(0,a.jsx)(s,{children:(0,a.jsx)(o.m,{formula:"\\mathcal{L} = -\\log \\frac{\\exp(\\text{sim}(a,p)/\\tau)}{\\sum_{n}\\exp(\\text{sim}(a,n)/\\tau)}"})}),"\n",(0,a.jsx)(n.p,{children:"Where:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"a"}),": anchor sample"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"p"}),": positive sample"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"n"}),": negative samples"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"Ï"}),": temperature parameter"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"sim"}),": similarity function (usually cosine)"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"key-components",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#key-components",children:"Key Components"})}),"\n",(0,a.jsx)(n.h3,{id:"1-data-augmentation",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#1-data-augmentation",children:"1. Data Augmentation"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.strong,{children:"Visual Domain:"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Random cropping"}),"\n",(0,a.jsx)(n.li,{children:"Color jittering"}),"\n",(0,a.jsx)(n.li,{children:"Gaussian blur"}),"\n",(0,a.jsx)(n.li,{children:"Random flipping"}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.strong,{children:"Text Domain:"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Token dropout"}),"\n",(0,a.jsx)(n.li,{children:"Paraphrasing"}),"\n",(0,a.jsx)(n.li,{children:"Back-translation"}),"\n",(0,a.jsx)(n.li,{children:"Span corruption"}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"2-temperature-scaling",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#2-temperature-scaling",children:"2. Temperature Scaling"})}),"\n",(0,a.jsx)(n.p,{children:"The temperature parameter Ï controls the concentration of the distribution:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Low Ï (0.01-0.1)"}),": Sharp distribution, harder negatives"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"High Ï (0.5-1.0)"}),": Smooth distribution, softer learning"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"3-negative-sampling",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#3-negative-sampling",children:"3. Negative Sampling"})}),"\n",(0,a.jsx)(n.p,{children:"More negatives generally improve representations:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"In-batch negatives"}),": Other samples in the minibatch"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Memory bank"}),": Store and reuse past embeddings"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Hard negative mining"}),": Focus on challenging examples"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"popular-methods",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#popular-methods",children:"Popular Methods"})}),"\n",(0,a.jsx)(n.h3,{id:"simclr-vision",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#simclr-vision",children:"SimCLR (Vision)"})}),"\n",(0,a.jsx)(n.pre,{className:"language-python",children:(0,a.jsxs)(n.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(n.span,{className:"code-line",children:[(0,a.jsx)(n.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(n.span,{className:"token function",children:"simclr_loss"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z_i",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," z_j",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," temperature",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"0.07"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token triple-quoted-string string",children:'"""SimCLR loss for image representations"""'}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Normalize embeddings"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    z_i ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}

1),"normalize",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z_i",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    z_j ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"normalize",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z_j",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Concatenate representations"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    representations ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"cat",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token punctuation",children:"["}),"z_i",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," z_j",(0,a.jsx)(n.span,{className:"token punctuation",children:"]"}),(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"0"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Compute similarity matrix"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    similarity_matrix ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"matmul",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        representations",(0,a.jsx)(n.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        representations",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," temperature\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Create labels for positive pairs"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    batch_size ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," z_i",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"shape",(0,a.jsx)(n.span,{className:"token punctuation",children:"["}),(0,a.jsx)(n.span,{className:"token number",children:"0"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"]"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    labels ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"cat",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token punctuation",children:"["}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"arange",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"batch_size",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"+"})," batch_size",(0,a.jsx)(n.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"arange",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"batch_size",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token punctuation",children:"]"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Mask out self-similarity"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    mask ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"eye",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token number",children:"2"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"*"})," batch_size",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dtype",(0,a.jsx)(n.span,{className:"token operator",children:"="}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),(0,a.jsx)(n.span,{className:"token builtin",children:"bool"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    similarity_matrix ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," similarity_matrix",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"masked_fill",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        mask",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," ",(0,a.jsx)(n.span,{className:"token operator",children:"-"}),(0,a.jsx)(n.span,{className:"token builtin",children:"float"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token string",children:"'inf'"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Compute loss"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    loss ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"cross_entropy",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"similarity_matrix",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," labels",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token keyword",children:"return"})," loss\n"]})]})}),"\n",(0,a.jsx)(n.h3,{id:"clip-vision-language",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#clip-vision-language",children:"CLIP (Vision-Language)"})}),"\n",(0,a.jsx)(n.pre,{className:"language-python",children:(0,a.jsxs)(n.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(n.span,{className:"code-line",children:[(0,a.jsx)(n.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(n.span,{className:"token function",children:"clip_loss"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"image_embeddings",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," text_embeddings",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," temperature",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"0.07"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token triple-quoted-string string",children:'"""CLIP contrastive loss"""'}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Normalize embeddings"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    image_embeddings ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}

1),"normalize",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"image_embeddings",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token operator",children:"-"}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    text_embeddings ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"normalize",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"text_embeddings",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token operator",children:"-"}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Compute similarity"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    logits ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"matmul",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        image_embeddings",(0,a.jsx)(n.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        text_embeddings",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," temperature\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Labels: diagonal elements are positive pairs"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    labels ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"arange",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token builtin",children:"len"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"logits",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Symmetric loss"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    loss_i2t ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"cross_entropy",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"logits",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," labels",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    loss_t2i ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"cross_entropy",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"logits",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," labels",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token keyword",children:"return"})," ",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"loss_i2t ",(0,a.jsx)(n.span,{className:"token operator",children:"+"})," loss_t2i",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," ",(0,a.jsx)(n.span,{className:"token number",children:"2"}),"\n"]})]})}),"\n",(0,a.jsx)(n.h2,{id:"advanced-techniques",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#advanced-techniques",children:"Advanced Techniques"})}),"\n",(0,a.jsx)(n.h3,{id:"1-momentum-contrast-moco",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#1-momentum-contrast-moco",children:"1. Momentum Contrast (MoCo)"})}),"\n",(0,a.jsx)(n.p,{children:"Maintains a queue of negative samples:"}),"\n",(0,a.jsx)(n.pre,{className:"language-python",children:(0,a.jsxs)(n.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(n.span,{className:"code-line",children:[(0,a.jsx)(n.span,{className:"token keyword",children:"class"})," ",(0,a.jsx)(n.span,{className:"token class-name",children:"MoCo"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"nn",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"Module",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(n.span,{className:"token function",children:"__init__"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," encoder",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"128"}),(0,a.jsx)(n.span,{className:"token punctuation",children:","})," K",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"65536"}),(0,a.jsx)(n.span,{className:"token punctuation",children:","})," m",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"0.999"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token builtin",children:"super"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"__init__",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"K ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," K  ",(0,a.jsx)(n.span,{className:"token comment",children:"# queue size"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"m ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," m  ",(0,a.jsx)(n.span,{className:"token comment",children:"# momentum coefficient"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token comment",children:"# Create encoders"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"encoder_q ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," encoder\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"encoder_k ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," copy",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"deepcopy",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"encoder",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token comment",children:"# Stop gradients to key encoder"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token keyword",children:"for"})," param ",(0,a.jsx)(n.span,{className:"token keyword",children:"in"})," self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"encoder_k",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"parameters",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["            param",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"requires_grad ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," ",(0,a.jsx)(n.span,{className:"token boolean",children:"False"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token comment",children:"# Create queue"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"register_buffer",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["            ",(0,a.jsx)(n.span,{className:"token string",children:'"queue"'}),(0,a.jsx)(n.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["            torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"randn",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"dim",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," K",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"queue ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}

1),"normalize",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"queue",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"0"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        self",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"register_buffer",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["            ",(0,a.jsx)(n.span,{className:"token string",children:'"queue_ptr"'}),(0,a.jsx)(n.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["            torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"zeros",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dtype",(0,a.jsx)(n.span,{className:"token operator",children:"="}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),(0,a.jsx)(n.span,{className:"token builtin",children:"long"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        ",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsx)(n.h3,{id:"2-swav-clustering",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#2-swav-clustering",children:"2. SwAV (Clustering)"})}),"\n",(0,a.jsx)(n.p,{children:"Combines contrastive learning with clustering:"}),"\n",(0,a.jsx)(n.pre,{className:"language-python",children:(0,a.jsxs)(n.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(n.span,{className:"code-line",children:[(0,a.jsx)(n.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(n.span,{className:"token function",children:"swav_loss"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z1",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," z2",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," prototypes",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," temperature",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"0.1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token triple-quoted-string string",children:'"""SwAV loss with online clustering"""'}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Compute assignments using Sinkhorn-Knopp"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    q1 ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," sinkhorn",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"matmul",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z1",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," prototypes",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," temperature",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    q2 ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," sinkhorn",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"matmul",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z2",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," prototypes",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," temperature",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Softmax predictions over prototypes for each view"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    p1 ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"softmax",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"matmul",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z1",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," prototypes",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," temperature",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    p2 ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," F",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"softmax",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"matmul",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"z2",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," prototypes",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"T",(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"/"})," temperature",(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(n.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token comment",children:"# Swapped prediction: each view's assignment supervises the other's prediction"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    loss ",(0,a.jsx)(n.span,{className:"token operator",children:"="})," ",(0,a.jsx)(n.span,{className:"token operator",children:"-"}),"torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"mean",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),(0,a.jsx)(n.span,{className:"token builtin",children:"sum"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"q1 ",(0,a.jsx)(n.span,{className:"token operator",children:"*"})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"log",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"p2",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(n.span,{className:"token operator",children:"+"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["        torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),(0,a.jsx)(n.span,{className:"token builtin",children:"sum"}),(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"q2 ",(0,a.jsx)(n.span,{className:"token operator",children:"*"})," torch",(0,a.jsx)(n.span,{className:"token punctuation",children:"."}),"log",(0,a.jsx)(n.span,{className:"token punctuation",children:"("}),"p1",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),(0,a.jsx)(n.span,{className:"token punctuation",children:","})," dim",(0,a.jsx)(n.span,{className:"token operator",children:"="}),(0,a.jsx)(n.span,{className:"token number",children:"1"}),(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(n.span,{className:"code-line",children:["    ",(0,a.jsx)(n.span,{className:"token keyword",children:"return"})," loss\n"]})]})}),"\n",(0,a.jsx)(n.h2,{id:"applications",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#applications",children:"Applications"})}),"\n",(0,a.jsx)(n.h3,{id:"1-visual-representation-learning",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#1-visual-representation-learning",children:"1. Visual Representation Learning"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Pre-training on unlabeled images"}),"\n",(0,a.jsx)(n.li,{children:"Transfer learning for downstream tasks"}),"\n",(0,a.jsx)(n.li,{children:"Few-shot classification"}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"2-language-model-pre-training",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#2-language-model-pre-training",children:"2. Language Model Pre-training"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Sentence embeddings (Sentence-BERT)"}),"\n",(0,a.jsx)(n.li,{children:"Cross-lingual alignment"}),"\n",(0,a.jsx)(n.li,{children:"Document similarity"}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"3-multimodal-learning",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#3-multimodal-learning",children:"3. Multimodal Learning"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Vision-language models (CLIP, ALIGN)"}),"\n",(0,a.jsx)(n.li,{children:"Audio-visual correspondence"}),"\n",(0,a.jsx)(n.li,{children:"Cross-modal retrieval"}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"best-practices",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#best-practices",children:"Best Practices"})}),"\n",(0,a.jsx)(n.h3,{id:"training-tips",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#training-tips",children:"Training Tips"})}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Large Batch Sizes"}),": More negatives improve learning (256-8192)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Strong Augmentations"}),": Encourage invariance learning"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Projection Head"}

1),": Non-linear projection improves representations"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Learning Rate Schedule"}),": Cosine annealing with warmup"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Weight Decay"}),": Prevent representation collapse"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"common-pitfalls",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#common-pitfalls",children:"Common Pitfalls"})}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Representation Collapse"}),": All samples map to same point"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Solution: Use stop-gradient, asymmetric architectures"}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"False Negatives"}),": Similar samples treated as negatives"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Solution: Careful data curation, supervised fine-tuning"}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Temperature Sensitivity"}),": Performance varies with Ï"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Solution: Grid search, learnable temperature"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"performance-considerations",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#performance-considerations",children:"Performance Considerations"})}),"\n",(0,a.jsxs)(n.table,{children:[(0,a.jsx)(n.thead,{children:(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.th,{children:"Method"}),(0,a.jsx)(n.th,{children:"Batch Size"}),(0,a.jsx)(n.th,{children:"Memory"}),(0,a.jsx)(n.th,{children:"Training Time"}),(0,a.jsx)(n.th,{children:"ImageNet Acc"})]})}),(0,a.jsxs)(n.tbody,{children:[(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"SimCLR"}),(0,a.jsx)(n.td,{children:"4096"}),(0,a.jsx)(n.td,{children:"High"}),(0,a.jsx)(n.td,{children:"1000 epochs"}),(0,a.jsx)(n.td,{children:"76.5%"})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"MoCo v3"}),(0,a.jsx)(n.td,{children:"256"}),(0,a.jsx)(n.td,{children:"Medium"}),(0,a.jsx)(n.td,{children:"300 epochs"}),(0,a.jsx)(n.td,{children:"76.7%"})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"SwAV"}),(0,a.jsx)(n.td,{children:"4096"}),(0,a.jsx)(n.td,{children:"High"}),(0,a.jsx)(n.td,{children:"800 epochs"}),(0,a.jsx)(n.td,{children:"75.3%"})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"BYOL"}),(0,a.jsx)(n.td,{children:"4096"}),(0,a.jsx)(n.td,{children:"High"}),(0,a.jsx)(n.td,{children:"1000 epochs"}),(0,a.jsx)(n.td,{children:"74.3%"})]})]})]}),"\n",(0,a.jsx)(n.h2,{id:"where-the-field-is-heading",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#where-the-field-is-heading",children:"Where the Field Is Heading"})}),"\n",(0,a.jsx)(n.p,{children:"The sharpest recent shift is away from explicit negatives. BYOL, SimSiam, and DINO match two views of the same image with no negative pairs at all, each leaning on a different anti-collapse mechanism â BYOL on a momentum target, SimSiam on a stop-gradient predictor, DINO on teacher centering and sharpening â while VICReg replaces negatives outright with variance and covariance regularization terms. The practical payoff is dropping the large-batch and memory-queue machinery that InfoNCE depends on, traded for more delicate architectural asymmetry."}),"\n",(0,a.jsx)(n.p,{children:"Contrastive and generative pre-training are also converging. Masked autoencoders (MAE) and data2vec showed that reconstructing masked content rivals contrastive learning on transfer benchmarks, and hybrids such as iBOT pair masked-image prediction with a self-distillation objective â local detail from reconstruction, global semantics from the distillation target."}),"\n",(0,a.jsxs)(n.p,{children:["The open problems are honest ones. Performance still hinges on the augmentation pipeline, which is hand-tuned per domain and barely transfers from vision to audio or tabular data. The theory is catching up but incomplete: the alignment-and-uniformity view explains ",(0,a.jsx)(n.em,{children:"what"})," a good representation looks like, but not ",(0,a.jsx)(n.em,{children:"which"})," negatives or temperature get you there. And false negatives â semantically identical samples wrongly pushed apart â remain a quiet ceiling that supervised-contrastive and debiased objectives only partly address."]}),"\n",(0,a.jsx)(n.h2,{id:"conclusion",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#conclusion",children:"Conclusion"})}),"\n",(0,a.jsx)(n.p,{children:"Contrastive learning has become a cornerstone of modern representation learning, enabling models to learn powerful features from unlabeled data. The interactive visualization above demonstrates how the contrastive objective shapes the embedding space, pulling positive pairs together while pushing negatives apart."}),"\n",(0,a.jsx)(n.p,{children:"The success of methods like CLIP and SimCLR shows that contrastive learning can rival or exceed supervised pre-training, opening new possibilities for learning from the vast amounts of unlabeled data available in the real world."}),"\n",(0,a.jsx)(n.h2,{id:"further-reading",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#further-reading",children:"Further Reading"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"https://arxiv.org/abs/2002.05709",children:"A Simple Framework for Contrastive Learning of Visual Representations (SimCLR)"})," â Chen et al., 2020. Introduces the augmentation-based contrastive framework and projection head."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"https://arxiv.org/abs/1911.05722",children:"Momentum Contrast for Unsupervised Visual Representation Learning (MoCo)"})," â He et al., 2019. The momentum encoder and dynamic negative queue."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"https://arxiv.org/abs/2103.00020",children:"Learning Transferable Visual Models From Natural Language Supervision (CLIP)"})," â Radford et al., 2021. Imageâtext contrastive pre-training at scale."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"https://lilianweng.github.io/posts/2021-05-31-contrastive/",children:"Contrastive Representation Learning"})," â Lilian Weng. A thorough survey of InfoNCE, temperature, and negative-sampling strategies."]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"related-concepts",children:(0,a.jsx)(n.a,{className:"heading-link",href:"#related-concepts",children:"Related concepts"})}),"\n",(0,a.jsx)(s,{children:(0,a.jsx)(l,{})})]})}void 0!==h&&h&&((void 0===h.wordCount||null===h.wordCount)&&(h.wordCount=1006),h.readingTime||(h.readingTime="7 min read"));let x=function(e={}){return(0,a.jsx)(m,Object.assign({},e,{children:(0,a.jsx)(u,e)}))};function j(e,n){throw Error("Expected "+(n?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}},14356:(e,n,s)=>{"use strict";s.d(n,{G:()=>p,m:()=>d});var a=s(37876),i=s(14232);let l=null,c={displayMode:!0,throwOnError:!1,errorColor:"#ef4444",trust:!0,strict:!1},t={displayMode:!1,throwOnError:!1,errorColor:"#ef4444"},r=["i","j","Ii","Tj","sim(.)"];function o(e,n,a){let[c,t]=(0,i.useState)(null);return(0,i.useEffect)(()=>

1{if(!a)return;let i=!1;return(!l&&(l=Promise.all([s.e(62333).then(s.bind(s,91275)),Promise.all([s.e(41062),s.e(63261)]).then(s.t.bind(s,63261,23))]).then(([e])=>e.default||e)),l).then(s=>{i||t({formula:e,html:s.renderToString(e,n)})}).catch(e=>{i||console.error("KaTeX rendering error:",e)}),()=>{i=!0}},[e,n,a]),c&&c.formula===e?c.html:null}function d({formula:e,className:n="",dark:s=!1}){let i=o(e,c,!!e&&!e.includes("sim(Ii,Ti)")),l=`my-6 py-4 px-3 mx-auto max-w-full text-center sm:max-w-3xl sm:px-6 ${s?"bg-zinc-800 text-zinc-100":"bg-zinc-50 dark:bg-zinc-800 text-zinc-900 dark:text-zinc-100"} rounded-lg overflow-x-auto border border-zinc-200 dark:border-zinc-700 ${n}`;if(!e||e.includes("Loss")&&e.includes("sim(Ii,Ti)"))return(0,a.jsx)("div",{className:l,"aria-label":"CLIP Loss Formula",role:"math",children:(0,a.jsx)("div",{className:"inline-block min-w-max font-serif text-lg leading-relaxed tracking-wide sm:text-xl",children:(0,a.jsxs)("div",{className:"flex flex-col items-center",children:[(0,a.jsxs)("div",{className:"mb-2",children:["Loss = â ",(0,a.jsx)("sup",{children:"1"}),"â",(0,a.jsx)("sub",{children:"N"})," Î£",(0,a.jsx)("sub",{children:"i=1"}),(0,a.jsx)("sup",{children:"N"})," log"]}),(0,a.jsxs)("div",{className:"flex w-48 flex-col border-0",children:[(0,a.jsxs)("div",{className:"border-b border-current pb-1 text-center",children:["e",(0,a.jsxs)("sup",{children:["sim(I",(0,a.jsx)("sub",{children:"i"}),",T",(0,a.jsx)("sub",{children:"i"}),")"]})]}),(0,a.jsxs)("div",{className:"pt-1 text-center",children:["Î£",(0,a.jsx)("sub",{children:"j=1"}),(0,a.jsx)("sup",{children:"N"})," e",(0,a.jsxs)("sup",{children:["sim(I",(0,a.jsx)("sub",{children:"i"}),",T",(0,a.jsx)("sub",{children:"j"}),")"]})]})]})]})})});let t="inline-block min-w-max font-serif text-lg leading-relaxed tracking-wide sm:text-xl";return(0,a.jsx)("div",{className:l,"aria-label":"Mathematical Formula",role:"math",children:null===i?(0,a.jsx)("div",{className:t,children:h(e)},"fallback"):(0,a.jsx)("div",{className:t,dangerouslySetInnerHTML:{__html:i}},"katex")})}function p({formula:e,className:n=""}){let s=o(e,t,!!e&&!r.includes(e));if(!e)return null;let i=`font-serif italic ${n}`;return"i"===e?(0,a.jsx)("i",{className:i,children:"i"}):"j"===e?(0,a.jsx)("i",{className:i,children:"j"}):"Ii"===e?(0,a.jsxs)("span",{className:i,children:["I",(0,a.jsx)("sub",{children:"i"})]}):"Tj"===e?(0,a.jsxs)("span",{className:i,children:["T",(0,a.jsx)("sub",{children:"j"})]}):"sim(.)"===e?(0,a.jsx)("span",{className:i,children:"sim(\xb7)"}):null!==s?(0,a.jsx)("span",{className:i,role:"math",dangerouslySetInnerHTML:{__html:s}},"katex"):(0,a.jsx)("span",{className:i,role:"math",children:h(e)},"fallback")}function h(e){if(!e)return"";try{let n=e.replace(/^\$\$(.*)\$\$$/g,"$1").replace(/^\$(.*)\$$/g,"$1").trim();n=(n=(n=(n=(n=(n=(n=n.replace(/\\\\/g,"\\")).replace(/\_\{([^}]+)\}/g,"<sub>$1</sub>").replace(/\^{([^}]+)}/g,"<sup>$1</sup>").replace(/\_([a-zA-Z0-9])/g,"<sub>$1</sub>").replace(/\^([a-zA-Z0-9])/g,"<sup>$1</sup>")).replace(/\\sum/g,"Î£").replace(/\\prod/g,"Î ").replace(/\\int/g,"â«").replace(/\\infty/g,"â").replace(/\\times/g,"\xd7").replace(/\\cdot/g,"\xb7").replace(/\\odot/g,"â").replace(/\\sqrt\{([^}]+)\}/g,"â($1)").replace(/\\sqrt/g,"â").replace(/\\theta/g,"Î¸").replace(/\\phi/g,"Ï").replace(/\\alpha/g,"Î±").replace(/\\beta/g,"Î²").replace(/\\gamma/g,"Î³").replace(/\\delta/g,"Î´").replace(/\\epsilon/g,"Îµ").replace(/\\mu/g,"Î¼").replace(/\\sigma/g,"Ï").replace(/\\pi/g,"Ï").replace(/\\lambda/g,"Î»").replace(/\\tau/g,"Ï").replace(/\\omega/g,"Ï").replace(/\\hat\{([^}]+)\}/g,"$1Ì")).replace(/\\mathbb\{([^}]+)\}/g,(e,n)=>({E:"\uD835\uDD3C",R:"â",N:"â",Z:"â¤",Q:"â",C:"â",P:"â"})[n]||n)).replace(/\\mathcal\{([^}]+)\}/g,(e,n)=>({L:"â",N:"\uD835\uDCA9",O:"\uD835\uDCAA",D:"\uD835\uDC9F",F:"â±",G:"\uD835\uDCA2",H:"â"})[n]||`ð${n.toLowerCase()}`)).replace(/\\log/g,"log").replace(/\\exp/g,"exp").replace(/\\sin/g,"sin").replace(/\\cos/g,"cos").replace(/\\tan/g,"tan").replace(/\\min/g,"min").replace(/\\max/g,"max").replace(/\\arg\\s*min/g,"argmin").replace(/\\arg\\s*max/g,"argmax").replace(/\\sim/g,"â¼").replace(/\\approx/g,"â").replace(/\\neq/g,"â ").replace(/\\leq/g,"â¤").replace(/\\geq/g,"â¥").replace(/\\ll/g,"âª").replace(/\\gg/g,"â«").replace(/\\quad/g,"  ").replace(/\\parallel/g,"â¥").replace(/\\perp/g,"â¥").replace(/\\in/g,"â").replace(/\\notin/g,"â").replace(/\\subset/g,"â").replace(/\\subseteq/g,"â").replace(/\\forall/g,"â").replace(/\\exists/g,"â").replace(/\\nabla/g,"â").replace(/\\partial/g,"â").replace(/\\emptyset/g,"â").replace(/\\rightarrow/g,"â").replace(/\\leftarrow/g,"â").replace(/\\Rightarrow/g,"â").replace(/\\Leftarrow/g,"â").replace(/\\leftrightarrow/g,"â").replace(/\\Leftrightarrow/g,"â").replace(/\|\|/g,"â").replace(/D_\{KL\}/g,"D<sub>KL</sub>")).replace(/\\left\(/g,"(").replace(/\\right\)/g,")").replace(/\\left\[/g,"[").replace(/\\right\]/g,"]").replace(/\\left\{/g,"{").replace(/\\right\}/g,"}").replace(/\\left\|/g,"â").replace(/\\right\|/g,"â").replace(/\\left\\langle/g,"â¨").replace(/\\right\\rangle/g,"â©");let s=10;for(;n.includes("\\frac{")&&s>0;)n=n.replace(/\\frac\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}/,'<span class="inline-flex flex-col items-center align-middle mx-1 text-base"><span class="border-b border-current px-1 pb-0.5 leading-tight">$1</span><span class="px-1 pt-0.5 leading-tight">$2</span></span>'),s--;return(0,a.jsx)("span",{dangerouslySetInnerHTML:{__html:n}})}catch(n){return console.error("Error processing formula:",n),(0,a.jsx)("span",{className:"font-mono text-sm",children:e})}}},40028:(e,n,s)=>{"use strict";s.d(n,{A:()=>i,b:()=>l});var a=s(37876);function i({className:e="h-64"}){return(0,a.jsx)("div",{className:`${e} animate-pulse rounded-xl bg-zinc-100 dark:bg-zinc-800`})}function l(){return(0,a.jsx)("div",{className:"h-48 animate-pulse rounded-xl bg-zinc-100 dark:bg-zinc-800"})}},95431:(e,n,s)=>{(window.__NEXT_P=window.__NEXT_P||[]).push(["/concepts/embeddings/contrastive-learning",function(){return s(9482)}])}},e=>{e.O(0,[41147,82667,2276,12627,90636,46593,38792],()=>e(e.s=95431)),_N_E=e.O()}]);

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.