1<!DOCTYPE html> <html lang="fr-FR"> <head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0, shrink-to-fit=no"><meta http-equiv="X-UA-Compatible" content="IE=edge"><title>Finetuner un modèle CamemBERT pour l'embedding de phrases ⢠HacheNotes</title><!-- Icons / Favicon --><link rel="icon" href="/favicon.ico" sizes="any"><link rel="icon" href="/icon.svg" type="image/svg+xml"><link rel="apple-touch-icon" href="/hache_120x120.png"><link rel="manifest" href="/manifest.webmanifest"><link rel="canonical" href="https://hachenotes.vercel.app/posts/sts-camembert/"><!-- Primary Meta Tags --><meta name="title" content="Finetuner un modèle CamemBERT pour l'embedding de phrases ⢠HacheNotes"><meta name="description" content="Le modèle CamemBERT est à l'état de l'art pour la langue française. Dans ce post on finetune ce modèle pour l'embedding de phrases."><meta name="author" content="h4c5"><!-- Theme Colour --><meta name="theme-color" content="#fafafa"><!-- Open Graph / Facebook --><meta property="og:type" content="article"><meta property="og:title" content="Finetuner un modèle CamemBERT pour l'embedding de phrases"><meta property="og:description" content="Le modèle CamemBERT est à l'état de l'art pour la langue française. Dans ce post on finetune ce modèle pour l'embedding de phrases."><meta property="og:url" content="https://hachenotes.vercel.app/posts/sts-camembert/"><meta property="og:site_name" content="HacheNotes"><meta property="og:locale" content="fr_FR"><meta property="og:image" content="https://hachenotes.vercel.app/og-image/sts-camembert.png"><meta property="og:image:width" content="1200"><meta property="og:image:height" content="630"><meta property="article:author" content="h4c5"><meta property="article:published_time" content="2024-03-09T00:00:00.000Z"><!-- Twitter --><meta property="twitter:card" content="summary_large_image"><meta property="twitter:url" content="https://hachenotes.vercel.app/posts/sts-camembert/"><meta property="twitter:title" content="Finetuner un modèle CamemBERT pour l'embedding de phrases"><meta property="twitter:description" content="Le modèle CamemBERT est à l'état de l'art pour la langue française. Dans ce post on finetune ce modèle pour l'embedding de phrases."><meta property="twitter:image" content="https://hachenotes.vercel.app/og-image/sts-camembert.png"><!-- RSS auto-discovery --><link rel="alternate" type="application/rss+xml" title="HacheNotes" href="/rss.xml"><!-- Sitemap --><link rel="sitemap" href="/sitemap-index.xml"><link rel="stylesheet" href="/_astro/_slug_.CgQj9uuD.css"> 2<link rel="stylesheet" href="/_astro/_slug_.CdoNP_Yz.css">
2<script type="module" src="/_astro/hoisted.Bu64Prmm.js"></script>
vendor: 1 bytes, line 2
2
3<script type="module" src="/_astro/page.CY1iZwUD.js"></script>
vendor: 1 bytes, line 3
3
4<script>window.va = window.va || function () { (window.vaq = window.vaq || []).push(arguments); }; 5 var script = document.createElement('script'); 6 script.defer = true; 7 script.src = '/_vercel/insights/script.js'; 8 var head = document.querySelector('head'); 9 head.appendChild(script); 10 </script>
10</head> <body>
10<script>(function(){const themeColorDark = "#1d1f21"; 11const themeColorLight = "#fafafa"; 12 13 const root = document.documentElement; 14 const colorThemeMetaTag = document.querySelector("meta[name='theme-color']"); 15 const storedTheme = 16 typeof localStorage !== "undefined" && localStorage.getItem("theme"); 17 const userTheme = 18 storedTheme || 19 (window.matchMedia("(prefers-color-scheme: light)").matches 20 ? "light" 21 : "dark"); 22 function setTheme(theme) { 23 root.classList.toggle("dark", theme === "dark"); 24 colorThemeMetaTag.setAttribute( 25 "content", 26 theme === "dark" ? themeColorDark : themeColorLight 27 ); 28 if (typeof localStorage !== "undefined") { 29 localStorage.setItem("theme", theme); 30 } 31 } 32 setTheme(userTheme); 33 root.addEventListener("theme-change", (e) => { 34 setTheme(e.detail.theme); 35 }); 36})();</script>
36 <a href="#main" class="sr-only focus:not-sr-only focus:fixed focus:start-1 focus:top-1.5">skip to content 37</a> <header id="main-header" class="group relative mb-28 flex items-center sm:ps-[4.5rem]"> <div class="flex sm:flex-col"> <a href="/" class="inline-flex items-center grayscale hover:filter-none sm:relative sm:inline-block"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 60" class="me-3 h-10 w-6 sm:absolute sm:start-[-4.5rem] sm:me-0 sm:h-20 sm:w-12"> <text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" font-size="8em">ðª</text> </svg> <title>Logo</title> <span class="text-xl font-bold sm:text-2xl">HacheNotes</span> </a> <nav id="navigation-menu" class="absolute -inset-x-4 top-14 hidden flex-col items-end gap-y-4 rounded-md bg-[--theme-menu-bg] py-4 text-accent shadow backdrop-blur group-[.menu-open]:z-50 group-[.menu-open]:flex sm:static sm:z-auto sm:-ms-4 sm:mt-1 sm:flex sm:flex-row sm:items-center sm:divide-x sm:divide-dashed sm:divide-accent sm:rounded-none sm:bg-transparent sm:py-0 sm:shadow-none sm:backdrop-blur-none" aria-label="Main menu"> <a href="/" class="px-4 py-4 sm:py-0 sm:hover:underline" rel="prefetch"> Home </a><a href="/posts/" class="px-4 py-4 sm:py-0 sm:hover:underline" rel="prefetch"> Posts </a><a href="/about/" class="px-4 py-4 sm:py-0 sm:hover:underline" rel="prefetch"> About </a> </nav> </div> <site-search id="search" class="ms-auto" data-astro-cid-otpdt6jm> <button data-open-modal disabled class="flex h-9 w-9 items-center justify-center rounded-md ring-zinc-400 transition-all hover:ring-2" data-astro-cid-otpdt6jm> <svg aria-label="search" class="h-6 w-6" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" data-astro-cid-otpdt6jm> <path stroke="none" d="M0 0h24v24H0z" data-astro-cid-otpdt6jm></path> <path d="M3 10a7 7 0 1 0 14 0 7 7 0 1 0-14 0M21 21l-6-6" data-astro-cid-otpdt6jm></path> </svg> </button> <dialog aria-label="search" class="h-full max-h-full w-full max-w-full border border-zinc-400 bg-[--theme-bg] shadow backdrop:bg-[--theme-menu-bg] backdrop:backdrop-blur sm:mx-auto sm:mb-auto sm:mt-16 sm:h-max sm:max-h-[calc(100%-8rem)] sm:min-h-[15rem] sm:w-5/6 sm:max-w-[48rem] sm:rounded-md" data-astro-cid-otpdt6jm> <div class="dialog-frame flex flex-col gap-4 p-6 pt-12 sm:pt-6" data-astro-cid-otpdt6jm> <button data-close-modal class="ms-auto cursor-pointer rounded-md bg-zinc-200 p-2 font-semibold dark:bg-zinc-700" data-astro-cid-otpdt6jm>Close</button> <div class="search-container" data-astro-cid-otpdt6jm> <div id="cactus__search" data-astro-cid-otpdt6jm></div> </div> </div> </dialog> </site-search> <theme-toggle class="ms-2 sm:ms-4"> <button type="button" id="toggle-theme" class="group relative h-9 w-9 rounded-md bg-zinc-200 p-2 ring-zinc-400 transition-all hover:ring-2 dark:bg-zinc-700" aria-label="Toggle Dark Mode"> <svg id="sun-svg" class="absolute start-1/2 top-1/2 h-7 w-7 -translate-x-1/2 -translate-y-1/2 scale-0 opacity-0 transition-all group-aria-pressed:scale-100 group-aria-pressed:opacity-100" aria-hidden="true" focusable="false" stroke-width="1.5" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M12 18C15.3137 18 18 15.3137 18 12C18 8.68629 15.3137 6 12 6C8.68629 6 6 8.68629 6 12C6 15.3137 8.68629 18 12 18Z" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M22 12L23 12" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M12 2V1" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M12 23V22" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M20 20L19 19" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M20 4L19 5" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M4 20L5 19" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M4 4L5 5" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M1 12L2 12" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> </svg> <svg id="moon-svg" class="absolute start-1/2 top-1/2 h-7 w-7 -translate-x-1/2 -translate-y-1/2 scale-0 opacity-0 transition-all group-aria-[pressed=false]:scale-100 group-aria-[pressed=false]:opacity-100" aria-hidden="true" focusable="false" stroke-width="1.5" viewB
37ox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <circle cx="12" cy="12" r="10" stroke="currentColor" stroke-width="1.5"></circle> <path d="M7.63262 3.06689C8.98567 3.35733 9.99999 4.56025 9.99999 6.00007C9.99999 7.65693 8.65685 9.00007 6.99999 9.00007C5.4512 9.00007 4.17653 7.82641 4.01685 6.31997" stroke="currentColor" stroke-width="1.5"></path> <path d="M22 13.0505C21.3647 12.4022 20.4793 12 19.5 12C17.567 12 16 13.567 16 15.5C16 17.2632 17.3039 18.7219 19 18.9646" stroke="currentColor" stroke-width="1.5"></path> <path d="M14.5 8.51L14.51 8.49889" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> <path d="M10 17C11.1046 17 12 16.1046 12 15C12 13.8954 11.1046 13 10 13C8.89543 13 8 13.8954 8 15C8 16.1046 8.89543 17 10 17Z" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"></path> </svg> </button> </theme-toggle>
37<script> 38 const bodyThemeIsDark = document.documentElement.classList.contains("dark"); 39 const button = document.getElementById("toggle-theme"); 40 button.setAttribute("aria-pressed", String(bodyThemeIsDark)); 41</script>
41 <button id="toggle-navigation-menu" class="group relative ms-4 h-7 w-7 sm:invisible sm:hidden" type="button" aria-label="Open main menu" aria-expanded="false" aria-haspopup="menu"> <svg id="line-svg" class="absolute start-1/2 top-1/2 h-full w-full -translate-x-1/2 -translate-y-1/2 transition-all group-aria-expanded:scale-0 group-aria-expanded:opacity-0" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"> <path stroke-linecap="round" stroke-linejoin="round" d="M3.75 9h16.5m-16.5 6.75h16.5"></path> </svg> <svg id="cross-svg" class="absolute start-1/2 top-1/2 h-full w-full -translate-x-1/2 -translate-y-1/2 scale-0 text-accent opacity-0 transition-all group-aria-expanded:scale-100 group-aria-expanded:opacity-100" class="text-accent" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"> <path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12"></path> </svg> </button> </header> <main id="main" class="flex-1"> <div class="gap-x-10 lg:flex lg:items-start"> <aside class="sticky top-20 order-2 -me-32 hidden basis-64 lg:block"> <h2 class="font-semibold">Table of Contents</h2> <ul class="mt-4 text-xs"> <li class="line-clamp-2 hover:text-accent"> <a class="block mt-3" href="#sentence-bert" aria-label="Scroll to section: Sentence-BERT"> <span class="me-1">#</span> Sentence-BERT </a> </li><li class="line-clamp-2 hover:text-accent"> <a class="block mt-3" href="#finetuner-un-modèle-bert-en-modèle-sentence-bert" aria-label="Scroll to section: Finetuner un modèle BERT en modèle Sentence-BERT"> <span class="me-1">#</span> Finetuner un modèle BERT en modèle Sentence-BERT </a> </li><li class="line-clamp-2 hover:text-accent"> <a class="block mt-2 ps-3 text-[0.6875rem]" href="#pré-requis" aria-label="Scroll to section: Pré-requis"> <span class="me-1">##</span> Pré-requis </a> </li><li class="line-clamp-2 hover:text-accent"> <a class="block mt-2 ps-3 text-[0.6875rem]" href="#récupération-des-données" aria-label="Scroll to section: Récupération des données"> <span class="me-1">##</span> Récupération des données </a> </li><li class="line-clamp-2 hover:text-accent"> <a class="block mt-2 ps-3 text-[0.6875rem]" href="#evaluation-du-modèle" aria-label="Scroll to section: Evaluation du modèle"> <span class="me-1">##</span> Evaluation du modèle </a> </li><li class="line-clamp-2 hover:text-accent"> <a class="block mt-3" href="#modèle-sentence-bert-distillé" aria-label="Scroll to section: Modèle Sentence-BERT distillé"> <span class="me-1">#</span> Modèle Sentence-BERT distillé </a> </li> <li class="line-clamp-2 hover:text-accent"> <a class="block mt-3" href="#commentaires" aria-label="Scroll to section: Commentaires"> <span class="me-1">#</span>Commentaires 42</a> </li> </ul> </aside> <article class="flex-grow break-words" data-pagefind-body> <div id="blog-hero"><h1 class="title mb-3 sm:mb-1">Finetuner un modèle CamemBERT pour l'embedding de phrases</h1> <time datetime="2024-03-09T00:00:00.000Z">9 mars 2024</time> <div class="mt-1 sm:mt-0 sm:inline sm:before:mx-2 sm:before:content-['|']"> <svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" class="me-1 inline-block h-6 w-6" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"> <path stroke="none" d="M0 0h24v24H0z" fill="none"></path> <path d="M7.859 6h-2.834a2.025 2.025 0 0 0 -2.025 2.025v2.834c0 .537 .213 1.052 .593 1.432l6.116 6.116a2.025 2.025 0 0 0 2.864 0l2.834 -2.834a2.025 2.025 0 0 0 0 -2.864l-6.117 -6.116a2.025 2.025 0 0 0 -1.431 -.593z"></path> <path d="M17.573 18.407l2.834 -2.834a2.025 2.025 0 0 0 0 -2.864l-7.117 -7.116"></path> <path d="M6 9h-.01"></path> </svg> <a class="cactus-link inline-block before:content-['#']" aria-label="View more blogs with the tag python" href="/tags/python/" data-pagefind-filter="tag"> python </a> , <a class="cactus-link inline-block before:content-['#']" aria-label="View more blogs with the tag transformers" href="/tags/transformers/" data-pagefind-filter="tag"> transformers </a> , <a class="cactus-link inline-block before:content-['#']" aria-label="View more blogs with the tag nlp" href="/tags/nlp/" data-pagefind-filter="tag"> nlp </a> , <a class="cactus-link inline-block before:content-['#']" aria-label="View more blogs with the tag finetuning" href="/tags/finetuning/" data-pagefind-filter="tag"> finetuning </a> , <a class="cactus-link inline-block before:content-['#']" aria-label="View more blogs with the tag sentence-transformers" href="/tags/sentence-transformers/" data-pagefind-filter="tag"> sentence-transformers </a> , <a class="cactus-link inline-block before:content-['#']" aria-label="View more blogs with the tag sentence-embedding" href="/tags/sentence-embedding/" data-pagefind-filter="tag"> sentence-embedding </a> </div> <!-- <p class="mt-8">{data.description}</p> --></div> <div class="prose prose-sm prose-cactus mt-12 w-full max-w-2xl text-ju
42stify prose-headings:font-semibold prose-headings:before:absolute prose-headings:before:-ms-4 prose-headings:before:text-accent prose-headings:before:content-['#'] prose-th:before:content-none"> <p>Le modèle <a href="https://camembert-model.fr">CamemBERT</a> est un modèle de langage à lâétat de lâart pour la modélisation de 43la langue française.</p> 44<p>Il sâagit dâun modèle <a href="https://arxiv.org/abs/1907.11692">RoBERTa</a> entrainé sur un grand nombre de texte en 45français et qui peut-être aisément adapté à un grand nombre de tâche grâce au finetuning.</p> 46<p>Ici on cherche à finetuner le modèle pour lâembedding de phrases (sentence embedding en anglais).</p> 47<h2 id="sentence-bert">Sentence-BERT</h2> 48<p>En sortie dâun modèle BERT, on a un vecteur embedding pour chaque token. Pour obtenir un embedding du texte dans son ensemble il faut définir une stratégie de transformation pour passer des embeddings individuels des tokens à un vecteur 49dâembedding pour la phrase dans son ensemble.</p> 50<p>La stratégie la plus simple et la plus efficace est de simplement prendre la moyenne des embeddings des tokens, câest ce que lâon appelle le <em>mean pooling</em>.</p> 51<img src="/_astro/sbert.Dc7jB3bI_Z2fsvs9.svg" alt="Schéma du modèle BERT" width="831" height="720" loading="lazy" decoding="async"> 52<p>Si vous voulez en savoir plus sur les statégies qui ont été envisagées, vous pouvez consulter ce papier : <a href="https://arxiv.org/pdf/1908.10084.pdf">Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks</a>.</p> 53<h2 id="finetuner-un-modèle-bert-en-modèle-sentence-bert">Finetuner un modèle BERT en modèle Sentence-BERT</h2> 54<p>Les auteurs du papier évoqué ci-dessus on construit une librairie python appelée <a href="https://www.sbert.net/">sentence-transformers</a>, permettant de manipuler des modèles Sentence-BERT.</p> 55<p>Nous allons lâutiliser pour obtenir un modèle Sentence-CamemBERT à partir dâun modèle CamemBERT disponible sur <a href="https://huggingface.co/almanach/camembert-base">huggingface</a>.</p> 56<h3 id="pré-requis">Pré-requis</h3> 57<p>Nous allons utilisés les packages suivants :</p> 58<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="plaintext"><code><span class="line"><span>datasets</span></span> 59<span class="line"><span>sentence-transformers</span></span> 60<span class="line"><span></span></span></code></pre> 61<h3 id="récupération-des-données">Récupération des données</h3> 62<p>Dans un premier temps, nous allons récupérer les données dâentrainement. 63On utilisera la partie française du jeu de données <a href="https://huggingface.co/datasets/stsb_multi_mt">STSb Multi MT</a>. 64Il sâagit dâun jeu de données contenant des paires de phrases ainsi quâune note entre 0 et 5 qui représente la similarité entre les deux phrases.</p> 65<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="python"><code><span class="line"><span style="color:#FF79C6">from</span><span style="color:#F8F8F2"> datasets </span><span style="color:#FF79C6">import</span><span style="color:#F8F8F2"> load_dataset</span></span> 66<span class="line"></span> 67<span class="line"><span style="color:#F8F8F2">sts_train_dataset </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> load_dataset(</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">stsb_multi_mt</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">name</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">fr</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">split</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">train</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">)</span></span> 68<span class="line"><span style="color:#F8F8F2">sts_dev_dataset </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> load_dataset(</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">stsb_multi_mt</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">name</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">fr</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">split</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">dev</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">)</span></span> 69<span class="line"><span style="color:#F8F8F2">sts_test_dataset </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> load_dataset(</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">stsb_multi_mt</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">name</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">fr</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">split</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">test</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">)</span></span> 70<span class="line"></span></code></pre> 71<p>Nous allons ensuite convertir les données récupérées en objets <code>InputExample</code> qui pourront être utilisés pour lâentrainement.</p> 72<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="python"><code><span class="line"><span style="color:#FF79C6">from</span><span style="color:#F8F8F2"> typing </span><span style="color:#FF79C6">
72import</span><span style="color:#F8F8F2"> List</span></span> 73<span class="line"><span style="color:#FF79C6">from</span><span style="color:#F8F8F2"> sentence_transformers </span><span style="color:#FF79C6">import</span><span style="color:#F8F8F2"> InputExample</span></span> 74<span class="line"></span> 75<span class="line"><span style="color:#FF79C6">def</span><span style="color:#50FA7B"> dataset_to_input_examples</span><span style="color:#F8F8F2">(</span><span style="color:#FFB86C;font-style:italic">dataset</span><span style="color:#F8F8F2">) </span><span style="color:#FF79C6">-></span><span style="color:#F8F8F2"> List[InputExample]:</span></span> 76<span class="line"><span style="color:#FF79C6"> return</span><span style="color:#F8F8F2"> [</span></span> 77<span class="line"><span style="color:#F8F8F2"> InputExample(</span></span> 78<span class="line"><span style="color:#FFB86C;font-style:italic"> texts</span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2">[example[</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">sentence1</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">], example[</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">sentence2</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">]],</span></span> 79<span class="line"><span style="color:#FFB86C;font-style:italic"> label</span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2">example[</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">similarity_score</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">] </span><span style="color:#FF79C6">/</span><span style="color:#BD93F9"> 5.0</span><span style="color:#F8F8F2">,</span></span> 80<span class="line"><span style="color:#F8F8F2"> )</span></span> 81<span class="line"><span style="color:#FF79C6"> for</span><span style="color:#F8F8F2"> example </span><span style="color:#FF79C6">in</span><span style="color:#F8F8F2"> dataset</span></span> 82<span class="line"><span style="color:#F8F8F2">]</span></span> 83<span class="line"></span> 84<span class="line"><span style="color:#F8F8F2">sts_train_examples </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> dataset_to_input_examples(sts_train_dataset)</span></span> 85<span class="line"><span style="color:#F8F8F2">sts_dev_examples </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> dataset_to_input_examples(sts_dev_dataset)</span></span> 86<span class="line"><span style="color:#F8F8F2">sts_test_examples </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> dataset_to_input_examples(sts_test_dataset)</span></span> 87<span class="line"></span></code></pre> 88<p>A present, on récupère le modèle CamemBERT depuis huggingface. On utilise ici le modèle <a href="https://huggingface.co/almanach/camembert-base"><code>almanach/camembert-base</code></a> pour le finetuning :</p> 89<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="python"><code><span class="line"><span style="color:#FF79C6">from</span><span style="color:#F8F8F2"> sentence_transformers </span><span style="color:#FF79C6">import</span><span style="color:#F8F8F2"> evaluation, losses, SentenceTransformer</span></span> 90<span class="line"><span style="color:#FF79C6">from</span><span style="color:#F8F8F2"> torch.utils.data </span><span style="color:#FF79C6">import</span><span style="color:#F8F8F2"> DataLoader</span></span> 91<span class="line"></span> 92<span class="line"><span style="color:#F8F8F2">batch_size </span><span style="color:#FF79C6">=</span><span style="color:#BD93F9"> 32</span></span> 93<span class="line"></span> 94<span class="line"><span style="color:#F8F8F2">model </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> SentenceTransformer(</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">almanach/camembert-base</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">)</span></span> 95<span class="line"></span> 96<span class="line"><span style="color:#F8F8F2">train_dataloader </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> DataLoader(sts_train_examples, </span><span style="color:#FFB86C;font-style:italic">shuffle</span><span style="color:#FF79C6">=</span><span style="color:#BD93F9">True</span><span style="color:#F8F8F2">, </span><span style="color:#FFB86C;font-style:italic">batch_size</span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2">batch_size)</span></span> 97<span class="line"><span style="color:#F8F8F2">train_loss </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> losses.CosineSimilarityLoss(</span><span style="color:#FFB86C;font-style:italic">model</span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2">model)</span></span> 98<span class="line"></span></code></pre> 99<p>On en profite pour constuire les objets <code>train_dataloader</code> et <code>train_loss</code> qui serviront pour lâentrainement.</p> 100<p>Enfin, on construit un evaluateur qui permettra de suivre les performances du modèle sur le jeu de données de dev pendant lâentrainement.</p> 101<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="python"><code><span class="line"><span style="color:#F8F8F2">sts_dev_evaluator </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> evaluation.EmbeddingSimilarityEvaluator.from_input_examples(</span></span> 102<span class="line"><span style="color:#F8F8F2"> sts_dev_examples, </span><span style="color:#FFB86C;font-style:italic">name</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">sts-dev</span><span style="color:#E9F284">"</span></span> 103<span class="line"><span style="color:#F8F8F2">)</span></span> 104<span class="line"></span></code></pre> 105<p>On peut à présent lancer lâentrainement du modèle :</p> 106<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="python"><code><span class="line"><span style="color:#F8F8F2">model.fit(</span></span> 107<span class="line"><span style="color:#FFB86C;font-style:italic"> train_objectives</span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2">[(train_dataloader, train_loss)],</span></span> 108<span class="line"><span style="color:#FFB86C;font-style:italic"> evaluator</span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2">sts_dev_evaluator,</span></span> 109<span class="line"><span style="color:#FFB86C;font-style:italic"> epochs</span><span style="color:#FF79C6">=</span><span style="color:#BD93F9">10</span><span style="color:#F8F8F2">,</span></span> 110<span class="line"><span style="color:#FFB86C;font-style:italic"> warmup_steps</span><span style="color:#FF79C6">=</span><span style="color:#BD93F9">500</span><span style="color:#F8F8F2">,</span></span> 111<span class="line"><span style="color:#FFB86C;font-style:italic"> save_best_model</span><span style="color:#FF79C6">=</span><span style="color:#BD93F9">True</span><span style="color:#F8F8F2">,</span></span> 112<span class="line"><span style="color:#F8F8F2">)</span></span> 113<span class="line"></span></code></pre> 114<h3 id="evaluation-du-modèle">Evaluation du modèle</h3> 115<p>Une fois lâentrainement terminé, on peut mesure la performance du modèle sur le jeu de données de test que lâon a gardé 116à lâécart :</p> 117<pre class="astro-code dracula" style="background-color:#282A36;color:#F8F8F2;overflow-x:auto;white-space:pre-wrap;word-wrap:break-word" tabindex="0" data-language="python"><code><span class="line"><span style="color:#F8F8F2">sts_test_evaluator </span><span style="color:#FF79C6">=</span><span style="color:#F8F8F2"> evaluation.EmbeddingSimilarityEvaluator.from_input_examples(</span></span> 118<span class="line"><span style="color:#F8F8F2"> sts_test_examples, </span><span style="color:#FFB86C;font-style:italic">name</span><span style="color:#FF79C6">=</span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">sts-test</span><span style="color:#E9F284">"</span></span> 119<span class="line"><span style="color:#F8F8F2">)</span></span> 120<span class="line"></span> 121<span class="line"><span style="color:#F8F8F2">sts_test_evaluator(model, </span><span style="color:#E9F284">"</span><span style="color:#F1FA8C">.</span><span style="color:#E9F284">"</span><span style="color:#F8F8F2">)</span></span> 122<span class="line"></span></code></pre> 123<p>Jâobtiens une correlation Pearson de <code>0.837</code> ce qui est au même niveau que les Sentence-CamemBERT que jâai pu trouvé sur huggingface :</p> 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159<table><thead><tr><th style="text-align:left">Model</th><th style="text-align:center">Pearson Correlation</th><th style="text-align:right">Paramètres</th></tr></thead><tbody><tr><td style="text-align:left"><a href="https://huggingface.co/h4c5/sts-camembert-base"><code>h4c5/sts-camembert-base</code></a></td><td style="text-align:center"><strong>0.837</strong></td><td style="text-align:right">110M</td></tr><tr><td style="text-align:left"><a href="https://huggingface.co/Lajavaness/sentence-camembert-base"><code>Lajavaness/sentence-camembert-base</code></a></td><td style="text-align:center">0.835</td><td style="text-align:right">110M</td></tr><tr><td style="text-align:left"><a href="https://huggingface.co/inokufu/flaubert-base-uncased-xnli-sts"><code>inokufu/flaubert-base-uncased-xnli-sts</code></a></td><td style="text-align:center">0.828</td><td style="text-align:right">137M</td></tr><tr><td style="text-align:left"><a href="https://huggingface.co/h4c5/sts-distilcamembert-base"><code>h4c5/sts-distilcamembert-base</code></a></td><td style="text-align:center">0.817</td><td style="text-align:right">68M</td></tr><tr><td style="text-align:left"><a href="https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased"><code>sentence-transformers/distiluse-base-multilingual-cased-v2</code></a></td><td style="text-align:center">0.786</td><td style="text-align:right">135M</td></tr></tbody></table> 160<h2 id="modèle-sentence-bert-distillé">Modèle Sentence-BERT distillé</h2> 161<p>Comme vous lâavez peut-être remarqué dans le tableau ci-dessus, jâai également entrainé un modèle Sentence-CamemBERT environ deux fois plus petit (68M paramètres contre 110M) et qui a pourtant de très bonnes performances : <a href="https://huggingface.co/h4c5/sts-distilcamembert-base"><code>h4c5/sts-distilcamembert-base</code></a>.</p> 162<p>Il sâagit en fait dâun modèle obtenu en suivant la procédure ci-dessus mais en partant du modèle CamemBERT distillé : <a href="https://huggingface.co/cmarkea/distilcamembert-base">cmarkea/distilcamembert-base</a>.</p> 163<p>Ce modèle dit âdistilléâ a été obtenu en supprimant une couche sur deux du modèle CamemBERT de base et en lâentrainant pour malgré tout maintenir ses performances.</p> 164<p>Pour en savoir plus sur le processus de distillation nâhésitez pas à consulter les papiers suivants :</p> 165<ul> 166<li><a href="https://arxiv.org/abs/1910.01108">DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter</a></li> 167<li><a href="https://hal.archives-ouvertes.fr/hal-03674695">DistilCamemBERT : une distillation du modèle français CamemBERT</a></li> 168</ul> 169<p>En attendant vous retrouverez mes deux modèles Sentence-CamemBERT sur huggingface :</p> 170<ul> 171<li><a href="https://huggingface.co/h4c5/sts-camembert-base">h4c5/sts-camembert-base</a></li> 172<li><a href="https://huggingface.co/h4c5/sts-distilcamembert-base">h4c5/sts-distilcamembert-base</a></li> 173</ul> <section class="mt-16" id="commentaires"> <h2>Commentaires</h2> <div id="comment-section"></div>
173<script type="text/javascript" src="https://latest.cactus.chat/cactus.js"></script>
173
173<script> 174 175const commentSectionId = window.location.pathname.split('/').filter(v => v !== '').join('.'); 176 177initComments({ 178 node: document.getElementById("comment-section"), 179 defaultHomeserverUrl: "https://matrix.cactus.chat:8448", 180 serverName: "cactus.chat", 181 siteName: "hachenotes", 182 commentSectionId: commentSectionId, 183}) 184</script>
184 </section> </div> </article> </div> <button id="to-top-btn" class="z-90 fixed bottom-8 end-4 flex h-10 w-10 translate-y-28 items-center justify-center rounded-full border-2 border-transparent bg-zinc-200 text-3xl opacity-0 transition-all duration-300 hover:border-zinc-400 data-[show=true]:translate-y-0 data-[show=true]:opacity-100 dark:bg-zinc-700 sm:end-8 sm:h-12 sm:w-12" aria-label="Back to Top" data-show="false"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="none" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" class="h-6 w-6"> <path stroke-linecap="round" stroke-linejoin="round" d="M4.5 15.75l7.5-7.5 7.5 7.5"></path> </svg> </button> </main> <footer class="mt-auto flex w-full flex-col items-center justify-center gap-y-2 pb-4 pt-20 text-center align-top text-gray-500 sm:flex-row sm:justify-between sm:text-xs"> <div class="me-0 sm:me-4"> <a rel="license" href="http://creativecommons.org/licenses/by/4.0/"> <img alt="Licence Creative Commons" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/80x15.png"> </a> </div> <div class="flex flex-wrap items-center gap-x-4 sm:items-center"> <ul class="flex flex-1 items-center gap-x-4 sm:flex-initial"> <li> <a class="inline-block p-2 sm:hover:text-link" href="https://github.com/h4c5" target="_blank" rel="noopener noreferrer"> <svg class="h-6 w-6" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"> <title>Github logo</title> <path stroke="none" d="M0 0h24v24H0z" fill="none"></path> <path d="M9 19c-4.3 1.4 -4.3 -2.5 -6 -3m12 5v-3.5c0 -1 .1 -1.4 -.5 -2c2.8 -.3 5.5 -1.4 5.5 -6a4.6 4.6 0 0 0 -1.3 -3.2a4.2 4.2 0 0 0 -.1 -3.2s-1.1 -.3 -3.5 1.3a12.3 12.3 0 0 0 -6.2 0c-2.4 -1.6 -3.5 -1.3 -3.5 -1.3a4.2 4.2 0 0 0 -.1 3.2a4.6 4.6 0 0 0 -1.3 3.2c0 4.6 2.7 5.7 5.5 6c-.6 .6 -.6 1.2 -.5 2v3.5"></path> </svg> <span class="sr-only">Github</span> </a> </li> <li> <a class="inline-block p-2 sm:hover:text-link" href="https://gitlab.com/h4c5" target="_blank" rel="noopener noreferrer"> <svg class="h-6 w-6" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"> <title>Gitlab logo</title> <path stroke="none" d="M0 0h24v24H0z" fill="none"></path> <path d="M22.65 14.39L12 22.13 1.35 14.39a.84.84 0 0 1-.3-.94l1.22-3.78 2.44-7.51A.42.42 0 0 1 4.82 2a.43.43 0 0 1 .58 0 .42.42 0 0 1 .11.18l2.44 7.49h8.1l2.44-7.51A.42.42 0 0 1 18.6 2a.43.43 0 0 1 .58 0 .42.42 0 0 1 .11.18l2.44 7.51L23 13.45a.84.84 0 0 1-.35.94z"></path> </svg> <span class="sr-only">Gitlab</span> </a> </li> <li> <a class="inline-block p-2 sm:hover:text-link" href="https://stackoverflow.com/users/7685195/h4kim" target="_blank" rel="noopener noreferrer"> <svg class="h-6 w-6" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor"> <title>StackOverflow logo</title> <path stroke="none" d="M0 0h24v24H0z" fill="none"></path> <path d="M17.36,20.2V14.82h1.79V22H3V14.82H4.8V20.2H17.36M6.77,14.32l.37-1.76,8.79,1.85-.37,1.76L6.77,14.32m1.16-4.21L8.69,8.5l8.14,3.78-.76,1.62L7.93,10.11m2.26-4,1.15-1.38,6.9,5.76-1.15,1.37-6.9-5.75m4.45-4.25L20,9.08l-1.44,1.07L13.2,2.94l1.44-1.07m-8,16.54v-1.8h9v1.8Z"></path> </svg> <span class="sr-only">StackOverflow</span> </a> </li> <li> <a class="inline-block p-2 sm:hover:text-link" href="https://www.linkedin.com/in/hakimcheikh/" target="_blank" rel="noopener noreferrer"> <svg class="h-6 w-6" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" stroke-linecap="round" stroke-linejoin="round"> <title>LinkedIn logo</title> <path stroke="none" d="M0 0h24v24H0z" fill="none"></path> <path d="M6.5 8C7.32843 8 8 7.32843 8 6.5C8 5.67157 7.32843 5 6.5 5C5.67157 5 5 5.67157 5 6.5C5 7.32843 5.67157 8 6.5 8Z"></path> <path d="M5 10C5 9.44772 5.44772 9 6 9H7C7.55228 9 8 9.44771 8 10V18C8 18.5523 7.55228 19 7 19H6C5.44772 19 5 18.5523 5 18V10Z"></path> <path d="M11 19H12C12.5523 19 13 18.5523 13 18V13.5C13 12 16 11 16 13V18.0004C16 18.5527 16.4477 19 17 19H18C18.5523 19 19 18.5523 19 18V12C19 10 17.5 9 15.5 9C13.5 9 13 10.5 13 10.5V10C13 9.44771 12.5523 9 12 9H11C10.4477 9 10 9.44772 10 10V18C10 18.5523 10.4477 19 11 19Z"></path> <path fill-rule="evenodd" clip-rule="evenodd" d="M20 1C21.6569 1 23 2.34315 23 4V20C23 21.6569 21.6569 23 20 23H4C2.34315 23 1 21.6569
1841 20V4C1 2.34315 2.34315 1 4 1H20ZM20 3C20.5523 3 21 3.44772 21 4V20C21 20.5523 20.5523 21 20 21H4C3.44772 21 3 20.5523 3 20V4C3 3.44772 3.44772 3 4 3H20Z"></path> </svg> <span class="sr-only">LinkedIn</span> </a> </li> <li> <a class="inline-block p-2 sm:hover:text-link" href="https://matrix.to/#/@h4c5:matrix.org" target="_blank" rel="noopener noreferrer"> <svg class="h-6 w-6" aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 27.9 32" fill="currentColor" stroke-linecap="round" stroke-linejoin="round" transform="translate(-.095 .005)"> <title>Matrix (protocol) logo</title> <path stroke="none" d="M0 0h24v24H0z" fill="none"></path> <path d="m27.1 31.2v-30.5h-2.19v-0.732h3.04v32h-3.04v-0.732z"></path> <path d="m8.23 10.4v1.54h0.044c0.385-0.564 0.893-1.03 1.49-1.37 0.58-0.323 1.25-0.485 1.99-0.485 0.72 0 1.38 0.14 1.97 0.42 0.595 0.279 1.05 0.771 1.36 1.48 0.338-0.5 0.796-0.941 1.38-1.32 0.58-0.383 1.27-0.574 2.06-0.574 0.602 0 1.16 0.074 1.67 0.22 0.514 0.148 0.954 0.383 1.32 0.707 0.366 0.323 0.653 0.746 0.859 1.27 0.205 0.522 0.308 1.15 0.308 1.89v7.63h-3.13v-6.46c0-0.383-0.015-0.743-0.044-1.08-0.0209-0.307-0.103-0.607-0.242-0.882-0.133-0.251-0.336-0.458-0.584-0.596-0.257-0.146-0.606-0.22-1.05-0.22-0.44 0-0.796 0.085-1.07 0.253-0.272 0.17-0.485 0.39-0.639 0.662-0.159 0.287-0.264 0.602-0.308 0.927-0.052 0.347-0.078 0.697-0.078 1.05v6.35h-3.13v-6.4c0-0.338-7e-3 -0.673-0.021-1-0.0114-0.314-0.0749-0.623-0.188-0.916-0.108-0.277-0.3-0.512-0.55-0.673-0.258-0.168-0.636-0.253-1.14-0.253-0.198 0.0083-0.394 0.042-0.584 0.1-0.258 0.0745-0.498 0.202-0.705 0.374-0.228 0.184-0.422 0.449-0.584 0.794-0.161 0.346-0.242 0.798-0.242 1.36v6.62h-3.13v-11.4z"></path> <path d="m0.936 0.732v30.5h2.19v0.732h-3.04v-32h3.03v0.732z"></path> </svg> <span class="sr-only">Matrix</span> </a> </li> </ul> </div> <nav aria-label="More on this site" class="flex gap-x-2 sm:gap-x-0 sm:divide-x sm:divide-gray-500"> <a href="/" class="px-4 py-2 sm:px-2 sm:py-0 sm:hover:text-textColor sm:hover:underline"> Home </a><a href="/posts/" class="px-4 py-2 sm:px-2 sm:py-0 sm:hover:text-textColor sm:hover:underline"> Posts </a><a href="/about/" class="px-4 py-2 sm:px-2 sm:py-0 sm:hover:text-textColor sm:hover:underline"> About </a> </nav> </footer>
184<script> 185 function speedInsightsBeforeSend(data){ 186 console.log("Speed Insights before send", data) 187 return data; 188 } 189</script>
189 <vercel-speed-insights data-props="{}" data-params="{"slug":"sts-camembert"}" data-pathname="/posts/sts-camembert/"></vercel-speed-insights> </body></html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.