1<!DOCTYPE HTML> 2<!-- 3Prologue by HTML5 UP 4html5up.net | @ajlkn 5Free for personal and commercial use under the CCA 3.0 license (html5up.net/license) 6Jekyll integration by Chris Bobbe | chrisbobbe.github.io 7--> 8<html><head><!-- Robots --> 9 <meta name="robots" content="index, follow" /><link rel="canonical" href="https://www.sabilab.fr/project/2020/03/19/distnet.html" /><!-- Title, description, author --><title>DistNet: Real-time Deep Detection of bacteria | SABILab - Scientific Analysis of Bio-Images Laboratory</title> 10 <meta name="description" content="Segmentation and Tracking of Bacteria growing in the Mother Machine by regression of Euclidean Distance Map and Displacement using Deep Neural Network and Self-Attention" /> 11 <meta name="author" content="Jean Ollion" /> 12 13 <!-- Open Graph --> 14 <meta property="og:title" content="DistNet: Real-time Deep Detection of bacteria | SABILab - Scientific Analysis of Bio-Images Laboratory" /> 15 <meta property="og:type" content="website" /> 16 <meta property="og:image" content="https://www.sabilab.fr/assets/images/SABILab.png" /> 17 <meta property="og:url" content="https://www.sabilab.fr/project/2020/03/19/distnet.html" /> 18 <meta property="og:site_name" content="SABILab" /> 19 <meta property="og:description" content="Segmentation and Tracking of Bacteria growing in the Mother Machine by regression of Euclidean Distance Map and Displacement using Deep Neural Network and Self-Attention" /> 20
20<script type="application/ld+json"> 21 { 22 "@type":"WebSite", 23 "url":"https://www.sabilab.fr", 24 "publisher":{ 25 "@type":"Organization", 26 "logo":{ 27 "@type":"ImageObject", 28 "url":"https://www.sabilab.fr/assets/images/SABILab.png" 29 }, 30 "name":"SABILab" 31 }, 32 "author":{ 33 "@type":"Person", 34 "name":"Jean Ollion" 35 }, 36 "headline":"DistNet: Real-time Deep Detection of bacteria | SABILab - Scientific Analysis of Bio-Images Laboratory", 37 "description":"Segmentation and Tracking of Bacteria growing in the Mother Machine by regression of Euclidean Distance Map and Displacement using Deep Neural Network and Self-Attention", 38 "image":"https://www.sabilab.fr/assets/images/SABILab.png", 39 "name":"SABILab", 40 "@context":"http://schema.org" 41 } 42 </script>
42 43 <!-- Styles --> 44 <meta charset="utf-8" /> 45 <meta name="viewport" content="width=device-width, initial-scale=1" /> 46 <!--[if lte IE 8]>
46<script src="/assets/js/ie/html5shiv.js" defer></script>
46<![endif]--> 47 <link rel="stylesheet" href="/assets/css/main.css" /> 48 <!--[if lte IE 8]><link rel="stylesheet" href="/assets/css/ie8.css" /><![endif]--> 49 <!--[if lte IE 9]><link rel="stylesheet" href="/assets/css/ie9.css" /><![endif]--> 50 <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.1.0/css/all.css"> 51 <!--my css--> 52 <link rel="stylesheet" href="/assets/css/tooltip.css"/> 53 <link rel="stylesheet" href="/assets/css/twentytwenty.css"/> 54 <link rel="stylesheet" href="/assets/css/mystyles.css"/> 55 <!-- Scripts --> 56
56<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js" defer></script>
56 57
57<script src="/assets/js/jquery.scrolly.min.js" defer></script>
57 58
58<script src="/assets/js/jquery.scrollzer.min.js" defer></script>
58 59 <!--script src="https://unpkg.com/imagesloaded@4/imagesloaded.pkgd.min.js"></script--> 60 <!--script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.imagesloaded/3.2.0/imagesloaded.pkgd.min.js"></script--> 61
61<script src="/assets/js/skel.min.js" defer></script>
61 62
62<script src="/assets/js/util.js" defer></script>
62 63 <!--[if lte IE 8]>
63<script src="/assets/js/ie/respond.min.js" defer></script>
63<![endif]--> 64
64<script src="/assets/js/main.js" defer></script>
64 65 <!--twentytwenty--> 66
66<script src="/assets/js/jquery-3.4.1.min.js"></script>
66 67
67<script src="/assets/js/jquery.event.move.js"></script>
67 68
68<script src="/assets/js/jquery.twentytwenty.js"></script>
68 69 <!--toc--> 70 <link rel="stylesheet" href="/assets/css/bootstrap-toc.css"/> 71
71<script src="/assets/js/bootstrap-toc.js"></script>
71 72 <!--script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/umd/popper.min.js" integrity="sha384-Q6E9RHvbIyZFJoft+2mJbHaEWldlvI9IOYy5n3zV9zzTtmI3UksdQRVvoxMfooAo" crossorigin="anonymous"></script--> 73
73<script src="/assets/js/bootstrap.min.js"></script>
73 74</head> 75<body><!-- Header --> 76<div id="header"> 77 <div class="top"><!-- Logo --> 78<div id="logo"> 79 <a href="https://www.sabilab.fr/" id="home-link"> 80 <span><img src="/assets/images/SABILab.png" alt="Jean Ollion" width="96" height="96" /></span> 81 <!--h1 id="title">SABILab</h1> 82 <p>Scientific Analysis of Bio-Images Laboratory</p--> 83 </a> 84</div> 85<!-- Nav --> 86<nav id="nav"> 87 <ul><li><a href="https://www.sabilab.fr/" id="about-link">About<!--span class="icon fa-home">About</span--> 88 </a></li><li><a href='https://www.sabilab.fr/blog' id='research-link'>Research<!--span class="icon fa-pencil-ruler">Research</span--> 89 </a></li></ul> 90</nav> 91<hr> 92 <nav id="post-toc"></nav></div> 93 <div class="bottom"><!-- Social Icons --> 94<ul class="icons"><li><a href="https://www.linkedin.com/in/jean-ollion-0b9629175/" class="icon-b fa-linkedin-in"><span class="label">LinkedIn</span></a></li><li><a href="https://www.github.com/jeanollion" class="icon-b fa-github"><span class="label">GitHub</span></a></li><li><a href="https://www.researchgate.net/profile/Jean_Ollion" class="icon-b fa-researchgate"><span class="label">Research Gate</span></a></li><li><a href="https://scholar.google.com/citations?user=QlnYf4EAAAAJ&hl" class="icon fa-graduation-cap"><span class="label">Google Scholar</span></a></li></ul> 95</div> 96</div> 97<!-- Main --> 98<div id="main"> 99 <!-- Post --> 100 <article class="shade-two"> 101 <div id="post-content" class="container"> 102 <header> 103 <h2 data-toc-skip>DistNet: Real-time Deep Detection of bacteria</h2> 104 <p>19 March 2020</p> 105 </header><p class="justify">Along with Charles Ollion (CMAP, Ecole Polytechnique, Université Paris-Saclay), I recently developed <em>DistNet</em><sup><a class="citation" href="#ollion2020distnet">[1]</a></sup>, a method based on a novel <em>Deep Neural Network</em> (DNN) architecture that allows simultaneous segmentation and tracking of bacteria growing in the <a class="tooltip">Mother Machine<span><img src="/assets/images/distnet/mother_machine.png" /><br />The <i>Mother machine</i> is a popular microfluidic device that allows long-term time-lapse imaging of thousands of cells in parallel by microscopy. It has become a valuable tool for single-cell level quantitative analysis and characterization of many cellular processes such as gene expression and regulation, mutagenesis or response to antibiotics. <br />White arrows represent the flow of growing medium in the <i>mother machine</i> microfluidic chip. Left: corresponding phase-contrast microscopy image showing <i>Escherichia Coli</i> bacteria growing in the microchannels. Scale bar: 5 μm.</span></a><sup><a class="citation" href="#wang2010robust">[2]</a></sup> microfluidic device at very low error rate. The article describing the method has been accepted in the MICCAI 2020 Conference.</p> 106 107<p> 108<div id="distnet-processing" class="twentytwenty-container" style="max-width: 858px; margin: 0 auto; text-align:center;"> 109 <img id="raw_i" src="/assets/images/distnet/raw_0.png" width="100%" height="284px" /> 110 <img id="raw_o" src="/assets/images/distnet/raw_overlay_0.png" width="100%" height="284px" /> 111</div> 112</p> 113
114<script> 115var container = $('#distnet-processing'); 116container.twentytwenty({ 117 default_offset_pct: 0.5, 118 orientation: 'vertical', 119 before_label: "Phase-contrast movie of E. Coli bacteria. Cells grow and divide over time.", 120 after_label: "Processed image: each color corresponds to a distinct bacteria lineage (from a division to another)", 121 no_overlay: false, 122 move_slider_on_hover: false, 123 move_with_handle_only: true, 124 click_to_move: true 125}); 126</script>
126 127 128<!-- sync gifs -->
129<script> 130var raw = new Image(); 131raw.src = "/assets/images/distnet/raw_.gif"; 132var raw_overlay_ = new Image(); 133raw_overlay_.src = "/assets/images/distnet/raw_overlay_.gif"; 134 135var allLoaded = function() { 136 $('#raw_i').attr("src", raw.src); 137 $('#raw_o').attr("src", raw_overlay_.src); 138 $('#raw_i').attr("height", "auto"); 139 $('#raw_o').attr("height", "auto"); 140} 141var imagesLoaded = 0; 142var onload = function() { 143 imagesLoaded++; 144 if(imagesLoaded == 2){ 145 allLoaded(); 146 } 147} 148raw.onload = onload; 149raw_overlay_.onload = onload; 150</script>
150 151 152<!--more--> 153 154<p class="justify">The figure above shows a phase-contrast movie of bacteria growing in the microchannels of the <em>Mother Machine</em>, as well as the result of the spatio-temporal detection (segmentation and tracking) made by <em>DistNet</em>. The next figure corresponds to another way of showing this data: successive frames of one single microchannel are displayed next to each other, referred to as <em>kymograph</em>. Result of segmentation is visible as cell outlines and result of tracking as coloured arrows.</p> 155<div id="distnet-kymo" class="twentytwenty-container" style="max-width: 848px; margin: 0 auto; margin-bottom:5px;"> 156 <img id="k1" src="/assets/images/distnet/kymo_annot_small.png" width="100%" height="279px" /> 157 <img id="k2" src="/assets/images/distnet/kymo_small.png" width="100%" height="279px" /> 158</div> 159<div style="max-width: 848px; margin: 0 auto;"> 160 <img src="/assets/images/distnet/time_arrow.png" width="100%" class="imgrespsync" /> 161</div> 162<p class="center">Time</p>
163<script> 164$("#distnet-kymo").twentytwenty({ 165 default_offset_pct: 0.5, 166 orientation: 'horizontal', 167 after_label: "Successive frames of one single microchannel juxtaposed", 168 before_label: "Kymograph processed by DistNet: each color corresponds to a distinct bacteria lineage (from a division to another)", 169 no_overlay: false, 170 move_slider_on_hover: false, 171 move_with_handle_only: true, 172 click_to_move: true 173}); 174$(document).ready(function() { 175 $("#k1").attr("height", "auto"); 176 $("#k2").attr("height", "auto"); 177}); 178</script>
178 179 180<h3 id="context">Context</h3> 181 182<p>Tracking of bacteria growing in the mother machine faces three major challenges:</p> 183<ul><li>Cell growth induces changes in bacteria morphology</li> 184<li>Bacteria can divide</li> 185<li>Due to cell growth, bacteria located at the open-end of microchannels are pushed out by other bacteria, thus their next observation is sometimes outside or partly outside the image.</li> 186</ul> 187<p>Studying some biological processes such as mutagenesis requires very fine statistics in order to detect rare events as in <a class="citation" href="#robert2018mutation">[3]</a>. 188To achieve this, one need to analyze massive datasets with typically 10<sup>6</sup>-10<sup>7</sup> observations of bacteria, at a very low error rate in order to limit manual curation time.</p> 189 190<h3 id="problem-formulation">Problem formulation</h3> 191 192<p>The main contribution of our approach is that we perform tracking by regression of the bacteria displacement between two successive frames. 193This allows to perform the tracking of multiple objects in one single prediction, with very common and simple network architectures (see next section). 194We also perform segmentation simultaneously by regression of the Euclidean Distance Map (EDM). We showed that performing segmentation and tracking simultaneously yielded in lower error rates compared to performing them separately.</p> 195 196<p>This formulation contrasts with most current methods, that perform detection first, then tracking with one prediction per detected object.</p> 197<p> 198<div id="distnet-kymo-edm" class="twentytwenty-container" style="max-width: 850px; margin: 0 auto; margin-bottom:5px;"> 199 <img id="edm1" src="/assets/images/distnet/kymo2_edm.png" width="100%" height="172px" /> 200 <img id="edm2" src="/assets/images/distnet/kymo2.png" width="100%" height="172px" /> 201</div> 202<div style="max-width: 850px; margin: 0 auto;"> 203 <img src="/assets/images/distnet/time_arrow.png" width="100%" class="imgrespsync" /> 204</div> 205<div id="distnet-kymo-dy" class="twentytwenty-container" style="max-width: 850px; margin: 0 auto; margin-bottom:5px;"> 206 <img id="dy1" src="/assets/images/distnet/kymo2_dy.png" width="100%" height="172px" /> 207 <img id="dy2" src="/assets/images/distnet/kymo2.png" width="100%" height="172px" /> 208</div> 209</p>
210<script> 211$("#distnet-kymo-edm").twentytwenty({ 212 default_offset_pct: 0.5, 213 orientation: 'horizontal', 214 after_label: "Successive frames of one single microchannel juxtaposed", 215 before_label: "EDM prediction. Value corresponds to the distance to the cell border", 216 no_overlay: false, 217 move_slider_on_hover: false, 218 move_with_handle_only: true, 219 click_to_move: true 220}); 221$("#distnet-kymo-dy").twentytwenty({ 222 default_offset_pct: 0.5, 223 orientation: 'horizontal', 224 after_label: "Successive frames of one single microchannel juxtaposed", 225 before_label: "Displacement prediction relative to the previous cell: blue = upward; red = downward", 226 no_overlay: false, 227 move_slider_on_hover: false, 228 move_with_handle_only: true, 229 click_to_move: true 230}); 231$(document).ready(function() { 232 $("#edm1").attr("height", "auto"); 233 $("#edm2").attr("height", "auto"); 234 $("#dy1").attr("height", "auto"); 235 $("#dy2").attr("height", "auto"); 236}); 237</script>
237 238 239<h3 id="network-architecture">Network architecture</h3> 240 241<p>Our network is based on <a class="tooltip">U-Net<span><img src="/assets/images/distnet/unet.png" /><br />U-Net architecture. <br />Each blue block corresponds to a 2D multi-channel feature map (resulting from convolutions). The network has an encoder-decoder structure. The encoder reduces spatial dimensions and increases the number of channels at each contraction (red arrows). The decoder reduces the number of channels at each up-sampling level, and restores spatial dimensions using both feature maps of the previous level (green arrows) and of the corresponding level in the encoder (yellow arrows). Inputs are couples of successive grayscale images (a: previous frame, b: current frame). The upper bacteria in (a) divides in (b). Outputs are: EDM predictions for the previous (c) and current frame (d); category prediction (e): background, (f): cell that do not divide and are associated to a cell at the previous frame, (g): cells that divided, (h): cells that are not associated to a cell at the previous frame); (i): prediction of bacteria Y-displacement between the two frames, in pixels and within an image of height 256.</span></a><sup><a class="citation" href="#ronneberger2015u">[4]</a></sup>, a widely-used network architecture in bio-image processing, that has the advantage of being simple and easy to train. 242The originality of our method is to introduce a <a class="tooltip">self-attention layer<span><img src="/assets/images/distnet/self_attention.png" /><br />Self-attention layer. <br />A spatial feature map (left, blue) is interpreted as a set of feature vectors. These vectors are combined with positional embedding that only depend on their index (for instance i â [0, 7] if the spatial dimensions are 8 Ã 1). The self-attention effectively transforms the set into a new one, where global information may be used. The final output has a skip connection with the input and can be re-interpreted as a spatial map.</span></a><sup><a class="citation" href="#vaswani2017attention">[5]</a></sup> in this network.</p> 243 244<p>This layer enables the DNN to combine information from the whole image, while a convolution only mixes information locally. To illustrate this intuition, the next figure shows a few example of predictions and the associated attention weight matrix. A weight matrix can be read in the following way: for each output region (columns), it shows where the attention was mostly focused on the inputs (rows). For instance, a perfect diagonal attention matrix would mean that most of the information needed to produce an output region comes from the same input region.</p> 245<p> 246<div id="attention-weights" class="twentytwenty-container" style="max-width: 849px; margin: 0 auto; margin-bottom:5px;"> 247 <img id="aw1" src="/assets/images/distnet/attention_weights_annot.png" width="100%" height="170px" /> 248 <img id="aw2" src="/assets/images/distnet/attention_weights_light.png" width="100%" height="170px" /> 249</div> 250</p>
251<script> 252$("#attention-weights").twentytwenty({ 253 default_offset_pct: 0.01, 254 orientation: 'horizontal', 255 after_label: "Attention weight matrices", 256 before_label: "Highlight of areas where attention is focused", 257 no_overlay: false, 258 move_slider_on_hover: false, 259 move_with_handle_only: true, 260 click_to_move: true 261}); 262$(document).ready(function() { 263 $("#aw1").attr("height", "auto"); 264 $("#aw2").attr("height", "auto"); 265}); 266</script>
266 267 268<p>We observe here that in the case of long cells, self-attention focuses on the edges of cells rather than on their interior (panel C). This is particularly visible, in panel A where a division occurs, and the attention for upper daughter cell prediction is focused on the edges of the mother cell. Panel B corresponds to the next observation, and consistently, we observe that the attention moves upwards and is still focused on the edges of the previous cell.</p> 269 270<h3 id="conclusion">Conclusion</h3> 271 272<p>This method presents several benefits:</p> 273<ul> 274 <li><strong>Global consistency</strong>: this method yields more coherent results, i.e. less conflicting predictions compared to a method that make one prediction per bacterium, because tracking is done simultaneously for all bacteria.</li> 275 <li><strong>Speed</strong>: This method is faster because one prediction by image is needed instead of one for the detection, then a second one per detected bacterium for the tracking.</li> 276 <li><strong>Simplicity / versatility</strong>: Our method enables to jointly train a single model, which is derived straightforwardly from a U-Net architecture, and could be adapted easily to different problem settings (e.g. 2D/3D geometries) or backbone networks. In contrast, tracking methods involving two steps and several models induce more hyper-parameters and complexity.</li> 277</ul> 278 279<p><strong>We applied successfully this method to the problem of bacteria growing in the mother machine, and achieved error rates inferior to 0.005% for tracking and of 0.03% for segmentation, outperforming current state-of-the-art methods, and making this method well-suited for high-throughput data analysis.</strong></p> 280 281<h3 id="implementation--availability">Implementation / Availability</h3> 282<p>An implementation of DistNet for tensorflow/keras is are available in <a href="https://github.com/jeanollion/distnet" class="astrong" target="_blank">this repository</a>. In order to run training on cloud we developed an <a href="https://github.com/jeanollion/dataset_iterator" class="astrong" target="_blank">iterator</a> that allows to read images in .h5 files, which is much faster on cloud than single image files.</p> 283 284<p>To test DistNet, we provide a <a href="https://github.com/jeanollion/bacmman/wiki/DistNet" class="astrong" target="_blank">tutorial</a> for a <a href="https://github.com/jeanollion/bacmman" class="astrong" target="_blank">BACMMAN</a><sup><a class="citation" href="#ollion2019high">[6]</a></sup> module along with a sample dataset, as well as a <a href="https://github.com/jeanollion/bacmman/wiki/FineTune-DistNet" class="astrong" target="_blank">tutorial</a> to adapt it to other datasets with fine-tuning, on google colab, a service that provides free GPU.</p> 285 286<hr /> 287<h3 id="references">References</h3> 288<ol class="bibliography"><li> 289 290 291<span id="ollion2020distnet">J. Ollion and C. Ollion, âDistNet: Deep Tracking by displacement regression: application to bacteria growing in the Mother Machine,â 2020.</span> 292 293<br /> 294 295</li> 296<li> 297 298 299<span id="wang2010robust">P. Wang <i>et al.</i>, âRobust growth of Escherichia coli,â <i>Current biology</i>, vol. 20, no. 12, pp. 1099â1103, 2010.</span> 300 301<br /> 302 303</li> 304<li> 305 306 307<span id="robert2018mutation">L. Robert, J. Ollion, J. Robert, X. Song, I. Matic, and M. Elez, âMutation dynamics and fitness effects followed in single cells,â <i>Science</i>, vol. 359, no. 6381, pp. 1283â1286, 2018, [Online]. Available at: https://science.sciencemag.org/content/359/6381/1283.editor-summary.</span> 308 309<br /> 310 311<a href="https://science.sciencemag.org/content/359/6381/1283.editor-summary"> https://science.sciencemag.org/content/359/6381/1283.editor-summary</a> 312 313</li> 314<li> 315 316 317<span id="ronneberger2015u">O. Ronneberger, P. Fischer, and T. Brox, âU-net: Convolutional networks for biomedical image segmentation,â in <i>International Conference on Medical image computing and computer-assisted intervention</i>, 2015, pp. 234â241, [Online]. Available at: https://arxiv.org/abs/1505.04597.</span> 318 319<br /> 320 321<a href="https://arxiv.org/abs/1505.04597"> https://arxiv.org/abs/1505.04597</a> 322 323</li> 324<li> 325 326 327<span id="vaswani2017attention">A. Vaswani <i>et al.</i>, âAttention is all you need,â in <i>Advances in neural information processing systems</i>, 2017, pp. 5998â6008, [Online]. Available at: https://arxiv.org/abs/1706.03762.</span> 328 329<br /> 330 331<a href="https://arxiv.org/abs/1706.03762"> https://arxiv.org/abs/1706.03762</a> 332 333</li> 334<li> 335 336 337<span id="ollion2019high">J. Ollion, M. Elez, and L. Robert, âHigh-throughput detection and tracking of cells and intracellular spots in mother machine experiments,â <i>Nature protocols</i>, vol. 14, no. 11, pp. 3144â3161, 2019, [Online]. Available at: https://rdcu.be/bRSze.</span> 338 339<br /> 340 341<a href="https://rdcu.be/bRSze"> https://rdcu.be/bRSze</a> 342 343</li></ol> 344 345<!--next script to preload gifs in order to sync them-->
346<script src="/assets/js/imgrespsync.js"></script>
346 347 348<!--script src="https://utteranc.es/client.js" 349 repo="https://github.com/jeanollion/jeanollion.github.io" 350 issue-term="title" 351 theme="github-light" 352 crossorigin="anonymous"> 353</script--> 354<!--script> 355// NOT WORKING YET!! 356$(document).ready(function() { 357 var alltwtw = $(".twentytwenty-container"); 358 for (var i=0; i<alltwtw.length; i++) { 359 alltwtw[i].find("img:first").attr("height", "auto"); 360 alltwtw[i].find("img:last").attr("height", "auto"); 361 } 362}); 363</script--> 364</div> 365 </article> 366</div>
367<script> 368$(function() { 369 var navSelector = "#post-toc"; 370 var $myNav = $(navSelector); 371 Toc.init({ 372 $nav: $myNav, 373 $scope: $("#post-content") 374 }); 375}); 376$('body').scrollspy({ target: "#post-toc" }) 377</script>
377 378<!-- Footer --> 379<div id="footer"> 380 <!-- Copyright --> 381 <ul class="copyright"> 382 383 <li><sub><sub><sup>©SABILab. All rights reserved.</sup></sup></sub></li> 384 385 <li><sub><sub><sup>Template Design: <a href="http://html5up.net">HTML5 UP</a> Jekyll integration: <a href="https://github.com/chrisbobbe/jekyll-theme-prologue">Chris Bobbe</a></sup></sup></sub></li> 386 </ul> 387</div> 388</body> 389</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.