PageSourceSearch

https://sabilab.fr/project/2020/03/19/distnet

html sabilab.fr collected 2026-09-28 07:04:53 UTC 22,827 bytes, 389 lines download raw bytes

1<!DOCTYPE HTML>
2<!--
3Prologue by HTML5 UP
4html5up.net | @ajlkn
5Free for personal and commercial use under the CCA 3.0 license (html5up.net/license)
6Jekyll integration by Chris Bobbe | chrisbobbe.github.io
7-->
8<html><head><!-- Robots -->
9  <meta name="robots" content="index, follow" /><link rel="canonical" href="https://www.sabilab.fr/project/2020/03/19/distnet.html" /><!-- Title, description, author --><title>DistNet: Real-time Deep Detection of bacteria | SABILab - Scientific Analysis of Bio-Images Laboratory</title>
10  <meta name="description" content="Segmentation and Tracking of Bacteria growing in the Mother Machine by regression of Euclidean Distance Map and Displacement using Deep Neural Network and Self-Attention" />
11  <meta name="author" content="Jean Ollion" />
12
13  <!-- Open Graph -->
14  <meta property="og:title" content="DistNet: Real-time Deep Detection of bacteria | SABILab - Scientific Analysis of Bio-Images Laboratory" />
15  <meta property="og:type" content="website" />
16  <meta property="og:image" content="https://www.sabilab.fr/assets/images/SABILab.png" />
17  <meta property="og:url" content="https://www.sabilab.fr/project/2020/03/19/distnet.html" />
18  <meta property="og:site_name" content="SABILab" />
19  <meta property="og:description" content="Segmentation and Tracking of Bacteria growing in the Mother Machine by regression of Euclidean Distance Map and Displacement using Deep Neural Network and Self-Attention" />
20  
20<script type="application/ld+json">
21    {
22      "@type":"WebSite",
23      "url":"https://www.sabilab.fr",
24      "publisher":{
25        "@type":"Organization",
26        "logo":{
27          "@type":"ImageObject",
28          "url":"https://www.sabilab.fr/assets/images/SABILab.png"
29        },
30        "name":"SABILab"
31      },
32      "author":{
33        "@type":"Person",
34      "name":"Jean Ollion"
35      },
36      "headline":"DistNet: Real-time Deep Detection of bacteria | SABILab - Scientific Analysis of Bio-Images Laboratory",
37      "description":"Segmentation and Tracking of Bacteria growing in the Mother Machine by regression of Euclidean Distance Map and Displacement using Deep Neural Network and Self-Attention",
38      "image":"https://www.sabilab.fr/assets/images/SABILab.png",
39      "name":"SABILab",
40      "@context":"http://schema.org"
41    }
42  </script>
42
43  <!-- Styles -->
44  <meta charset="utf-8" />
45  <meta name="viewport" content="width=device-width, initial-scale=1" />
46  <!--[if lte IE 8]>
46<script src="/assets/js/ie/html5shiv.js" defer></script>
46<![endif]-->
47  <link rel="stylesheet" href="/assets/css/main.css" />
48  <!--[if lte IE 8]><link rel="stylesheet" href="/assets/css/ie8.css" /><![endif]-->
49  <!--[if lte IE 9]><link rel="stylesheet" href="/assets/css/ie9.css" /><![endif]-->
50  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.1.0/css/all.css">
51  <!--my css-->
52  <link rel="stylesheet" href="/assets/css/tooltip.css"/>
53  <link rel="stylesheet" href="/assets/css/twentytwenty.css"/>
54  <link rel="stylesheet" href="/assets/css/mystyles.css"/>
55  <!-- Scripts -->
56  
56<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js" defer></script>
56
57  
57<script src="/assets/js/jquery.scrolly.min.js" defer></script>
57
58  
58<script src="/assets/js/jquery.scrollzer.min.js" defer></script>
58
59  <!--script src="https://unpkg.com/imagesloaded@4/imagesloaded.pkgd.min.js"></script-->
60  <!--script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.imagesloaded/3.2.0/imagesloaded.pkgd.min.js"></script-->
61  
61<script src="/assets/js/skel.min.js" defer></script>
61
62  
62<script src="/assets/js/util.js" defer></script>
62
63  <!--[if lte IE 8]>
63<script src="/assets/js/ie/respond.min.js" defer></script>
63<![endif]-->
64  
64<script src="/assets/js/main.js" defer></script>
64
65  <!--twentytwenty-->
66  
66<script src="/assets/js/jquery-3.4.1.min.js"></script>
66
67  
67<script src="/assets/js/jquery.event.move.js"></script>
67
68  
68<script src="/assets/js/jquery.twentytwenty.js"></script>
68
69  <!--toc-->
70  <link rel="stylesheet" href="/assets/css/bootstrap-toc.css"/>
71  
71<script src="/assets/js/bootstrap-toc.js"></script>
71
72  <!--script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/umd/popper.min.js" integrity="sha384-Q6E9RHvbIyZFJoft+2mJbHaEWldlvI9IOYy5n3zV9zzTtmI3UksdQRVvoxMfooAo" crossorigin="anonymous"></script-->
73  
73<script src="/assets/js/bootstrap.min.js"></script>
73
74</head>
75<body><!-- Header -->
76<div id="header">
77  <div class="top"><!-- Logo -->
78<div id="logo">
79  <a href="https://www.sabilab.fr/" id="home-link">
80    <span><img src="/assets/images/SABILab.png" alt="Jean Ollion" width="96" height="96" /></span>
81    <!--h1 id="title">SABILab</h1>
82    <p>Scientific Analysis of Bio-Images Laboratory</p-->
83  </a>
84</div>
85<!-- Nav -->
86<nav id="nav">
87  <ul><li><a href="https://www.sabilab.fr/" id="about-link">About<!--span class="icon fa-home">About</span-->
88          </a></li><li><a href='https://www.sabilab.fr/blog' id='research-link'>Research<!--span class="icon fa-pencil-ruler">Research</span-->
89          </a></li></ul>
90</nav>
91<hr>
92        <nav id="post-toc"></nav></div>
93  <div class="bottom"><!-- Social Icons -->
94<ul class="icons"><li><a href="https://www.linkedin.com/in/jean-ollion-0b9629175/" class="icon-b fa-linkedin-in"><span class="label">LinkedIn</span></a></li><li><a href="https://www.github.com/jeanollion" class="icon-b fa-github"><span class="label">GitHub</span></a></li><li><a href="https://www.researchgate.net/profile/Jean_Ollion" class="icon-b fa-researchgate"><span class="label">Research Gate</span></a></li><li><a href="https://scholar.google.com/citations?user=QlnYf4EAAAAJ&hl" class="icon fa-graduation-cap"><span class="label">Google Scholar</span></a></li></ul>
95</div>
96</div>
97<!-- Main -->
98<div id="main">
99	<!-- Post -->
100	<article class="shade-two">
101	  <div id="post-content" class="container">
102			<header>
103				<h2 data-toc-skip>DistNet: Real-time Deep Detection of bacteria</h2>
104				<p>19 March 2020</p>
105			</header><p class="justify">Along with Charles Ollion (CMAP, Ecole Polytechnique, Université Paris-Saclay), I recently developed <em>DistNet</em><sup><a class="citation" href="#ollion2020distnet">[1]</a></sup>, a method based on a novel <em>Deep Neural Network</em> (DNN) architecture that allows simultaneous segmentation and tracking of bacteria growing in the <a class="tooltip">Mother Machine<span><img src="/assets/images/distnet/mother_machine.png" /><br />The <i>Mother machine</i> is a popular microfluidic device that allows long-term time-lapse imaging of thousands of cells in parallel by microscopy. It has become a valuable tool for single-cell level quantitative analysis and characterization of many cellular processes such as gene expression and regulation, mutagenesis or response to antibiotics. <br />White arrows represent the flow of growing medium in the <i>mother machine</i> microfluidic chip. Left: corresponding phase-contrast microscopy image showing <i>Escherichia Coli</i> bacteria growing in the microchannels. Scale bar: 5 μm.</span></a><sup><a class="citation" href="#wang2010robust">[2]</a></sup> microfluidic device at very low error rate. The article describing the method has been accepted in the MICCAI 2020 Conference.</p>
106
107<p>
108<div id="distnet-processing" class="twentytwenty-container" style="max-width: 858px; margin: 0 auto; text-align:center;">
109 <img id="raw_i" src="/assets/images/distnet/raw_0.png" width="100%" height="284px" />
110 <img id="raw_o" src="/assets/images/distnet/raw_overlay_0.png" width="100%" height="284px" />
111</div>
112</p>
113
114<script>
115var container = $('#distnet-processing');
116container.twentytwenty({
117	default_offset_pct: 0.5,
118	orientation: 'vertical',
119	before_label: "Phase-contrast movie of E. Coli bacteria. Cells grow and divide over time.",
120	after_label: "Processed image: each color corresponds to a distinct bacteria lineage (from a division to another)",
121	no_overlay: false,
122	move_slider_on_hover: false,
123	move_with_handle_only: true,
124	click_to_move: true
125});
126</script>
126
127
128<!-- sync gifs -->
129<script>
130var raw = new Image();
131raw.src = "/assets/images/distnet/raw_.gif";
132var raw_overlay_ = new Image();
133raw_overlay_.src = "/assets/images/distnet/raw_overlay_.gif";
134
135var allLoaded = function() {
136    $('#raw_i').attr("src", raw.src);
137    $('#raw_o').attr("src", raw_overlay_.src);
138    $('#raw_i').attr("height", "auto");
139    $('#raw_o').attr("height", "auto");
140}
141var imagesLoaded = 0;
142var onload = function() {
143    imagesLoaded++;
144    if(imagesLoaded == 2){
145       allLoaded();
146    }
147}
148raw.onload = onload;
149raw_overlay_.onload = onload;
150</script>
150
151
152<!--more-->
153
154<p class="justify">The figure above shows a phase-contrast movie of bacteria growing in the microchannels of the <em>Mother Machine</em>, as well as the result of the spatio-temporal detection (segmentation and tracking) made by <em>DistNet</em>. The next figure corresponds to another way of showing this data: successive frames of one single microchannel are displayed next to each other, referred to as <em>kymograph</em>. Result of segmentation is visible as cell outlines and result of tracking as coloured arrows.</p>
155<div id="distnet-kymo" class="twentytwenty-container" style="max-width: 848px; margin: 0 auto; margin-bottom:5px;">
156 <img id="k1" src="/assets/images/distnet/kymo_annot_small.png" width="100%" height="279px" />
157 <img id="k2" src="/assets/images/distnet/kymo_small.png" width="100%" height="279px" />
158</div>
159<div style="max-width: 848px; margin: 0 auto;">
160 <img src="/assets/images/distnet/time_arrow.png" width="100%" class="imgrespsync" />
161</div>
162<p class="center">Time</p>
163<script>
164$("#distnet-kymo").twentytwenty({
165	default_offset_pct: 0.5,
166	orientation: 'horizontal',
167	after_label: "Successive frames of one single microchannel juxtaposed",
168	before_label: "Kymograph processed by DistNet: each color corresponds to a distinct bacteria lineage (from a division to another)",
169	no_overlay: false,
170	move_slider_on_hover: false,
171	move_with_handle_only: true,
172	click_to_move: true
173});
174$(document).ready(function() {
175	$("#k1").attr("height", "auto");
176	$("#k2").attr("height", "auto");
177});
178</script>
178
179
180<h3 id="context">Context</h3>
181
182<p>Tracking of bacteria growing in the mother machine faces three major challenges:</p>
183<ul><li>Cell growth induces changes in bacteria morphology</li>
184<li>Bacteria can divide</li>
185<li>Due to cell growth, bacteria located at the open-end of microchannels are pushed out by other bacteria, thus their next observation is sometimes outside or partly outside the image.</li>
186</ul>
187<p>Studying some biological processes such as mutagenesis requires very fine statistics in order to detect rare events as in <a class="citation" href="#robert2018mutation">[3]</a>.
188To achieve this, one need to analyze massive datasets with typically 10<sup>6</sup>-10<sup>7</sup> observations of bacteria, at a very low error rate in order to limit manual curation time.</p>
189
190<h3 id="problem-formulation">Problem formulation</h3>
191
192<p>The main contribution of our approach is that we perform tracking by regression of the bacteria displacement between two successive frames.
193This allows to perform the tracking of multiple objects in one single prediction, with very common and simple network architectures (see next section).
194We also perform segmentation simultaneously by regression of the Euclidean Distance Map (EDM). We showed that performing segmentation and tracking simultaneously yielded in lower error rates compared to performing them separately.</p>
195
196<p>This formulation contrasts with most current methods, that perform detection first, then tracking with one prediction per detected object.</p>
197<p>
198<div id="distnet-kymo-edm" class="twentytwenty-container" style="max-width: 850px; margin: 0 auto; margin-bottom:5px;">
199 <img id="edm1" src="/assets/images/distnet/kymo2_edm.png" width="100%" height="172px" />
200 <img id="edm2" src="/assets/images/distnet/kymo2.png" width="100%" height="172px" />
201</div>
202<div style="max-width: 850px; margin: 0 auto;">
203 <img src="/assets/images/distnet/time_arrow.png" width="100%" class="imgrespsync" />
204</div>
205<div id="distnet-kymo-dy" class="twentytwenty-container" style="max-width: 850px; margin: 0 auto; margin-bottom:5px;">
206 <img id="dy1" src="/assets/images/distnet/kymo2_dy.png" width="100%" height="172px" />
207 <img id="dy2" src="/assets/images/distnet/kymo2.png" width="100%" height="172px" />
208</div>
209</p>
210<script>
211$("#distnet-kymo-edm").twentytwenty({
212	default_offset_pct: 0.5,
213	orientation: 'horizontal',
214	after_label: "Successive frames of one single microchannel juxtaposed",
215	before_label: "EDM prediction. Value corresponds to the distance to the cell border",
216	no_overlay: false,
217	move_slider_on_hover: false,
218	move_with_handle_only: true,
219	click_to_move: true
220});
221$("#distnet-kymo-dy").twentytwenty({
222	default_offset_pct: 0.5,
223	orientation: 'horizontal',
224	after_label: "Successive frames of one single microchannel juxtaposed",
225	before_label: "Displacement prediction relative to the previous cell: blue = upward; red = downward",
226	no_overlay: false,
227	move_slider_on_hover: false,
228	move_with_handle_only: true,
229	click_to_move: true
230});
231$(document).ready(function() {
232	$("#edm1").attr("height", "auto");
233	$("#edm2").attr("height", "auto");
234	$("#dy1").attr("height", "auto");
235	$("#dy2").attr("height", "auto");
236});
237</script>
237
238
239<h3 id="network-architecture">Network architecture</h3>
240
241<p>Our network is based on <a class="tooltip">U-Net<span><img src="/assets/images/distnet/unet.png" /><br />U-Net architecture. <br />Each blue block corresponds to a 2D multi-channel feature map (resulting from convolutions). The network has an encoder-decoder structure. The encoder reduces spatial dimensions and increases the number of channels at each contraction (red arrows). The decoder reduces the number of channels at each up-sampling level, and restores spatial dimensions using both feature maps of the previous level (green arrows) and of the corresponding level in the encoder (yellow arrows). Inputs are couples of successive grayscale images (a: previous frame, b: current frame). The upper bacteria in (a) divides in (b). Outputs are: EDM predictions for the previous (c) and current frame (d); category prediction (e): background, (f): cell that do not divide and are associated to a cell at the previous frame, (g): cells that divided, (h): cells that are not associated to a cell at the previous frame); (i): prediction of bacteria Y-displacement between the two frames, in pixels and within an image of height 256.</span></a><sup><a class="citation" href="#ronneberger2015u">[4]</a></sup>, a widely-used network architecture in bio-image processing, that has the advantage of being simple and easy to train.
242The originality of our method is to introduce a <a class="tooltip">self-attention layer<span><img src="/assets/images/distnet/self_attention.png" /><br />Self-attention layer. <br />A spatial feature map (left, blue) is interpreted as a set of feature vectors. These vectors are combined with positional embedding that only depend on their index (for instance i ∈ [0, 7] if the spatial dimensions are 8 × 1). The self-attention effectively transforms the set into a new one, where global information may be used. The final output has a skip connection with the input and can be re-interpreted as a spatial map.</span></a><sup><a class="citation" href="#vaswani2017attention">[5]</a></sup> in this network.</p>
243
244<p>This layer enables the DNN to combine information from the whole image, while a convolution only mixes information locally. To illustrate this intuition, the next figure shows a few example of predictions and the associated attention weight matrix. A weight matrix can be read in the following way: for each output region (columns), it shows where the attention was mostly focused on the inputs (rows). For instance, a perfect diagonal attention matrix would mean that most of the information needed to produce an output region comes from the same input region.</p>
245<p>
246<div id="attention-weights" class="twentytwenty-container" style="max-width: 849px; margin: 0 auto; margin-bottom:5px;">
247 <img id="aw1" src="/assets/images/distnet/attention_weights_annot.png" width="100%" height="170px" />
248 <img id="aw2" src="/assets/images/distnet/attention_weights_light.png" width="100%" height="170px" />
249</div>
250</p>
251<script>
252$("#attention-weights").twentytwenty({
253	default_offset_pct: 0.01,
254	orientation: 'horizontal',
255	after_label: "Attention weight matrices",
256	before_label: "Highlight of areas where attention is focused",
257	no_overlay: false,
258	move_slider_on_hover: false,
259	move_with_handle_only: true,
260	click_to_move: true
261});
262$(document).ready(function() {
263	$("#aw1").attr("height", "auto");
264	$("#aw2").attr("height", "auto");
265});
266</script>
266
267
268<p>We observe here that in the case of long cells, self-attention focuses on the edges of cells rather than on their interior (panel C). This is particularly visible, in panel A where a division occurs, and the attention for upper daughter cell prediction is focused on the edges of the mother cell. Panel B corresponds to the next observation, and consistently, we observe that the attention moves upwards and is still focused on the edges of the previous cell.</p>
269
270<h3 id="conclusion">Conclusion</h3>
271
272<p>This method presents several benefits:</p>
273<ul>
274  <li><strong>Global consistency</strong>: this method yields more coherent results, i.e. less conflicting predictions compared to a method that make one prediction per bacterium, because tracking is done simultaneously for all bacteria.</li>
275  <li><strong>Speed</strong>: This method is faster because one prediction by image is needed instead of one for the detection, then a second one per detected bacterium for the tracking.</li>
276  <li><strong>Simplicity / versatility</strong>: Our method enables to jointly train a single model, which is derived straightforwardly from a U-Net architecture, and could be adapted easily to different problem settings (e.g. 2D/3D geometries) or backbone networks. In contrast, tracking methods involving two steps and several models induce more hyper-parameters and complexity.</li>
277</ul>
278
279<p><strong>We applied successfully this method to the problem of bacteria growing in the mother machine, and achieved error rates inferior to 0.005% for tracking and of 0.03% for segmentation, outperforming current state-of-the-art methods, and making this method well-suited for high-throughput data analysis.</strong></p>
280
281<h3 id="implementation--availability">Implementation / Availability</h3>
282<p>An implementation of DistNet for tensorflow/keras is are available in <a href="https://github.com/jeanollion/distnet" class="astrong" target="_blank">this repository</a>. In order to run training on cloud we developed an <a href="https://github.com/jeanollion/dataset_iterator" class="astrong" target="_blank">iterator</a> that allows to read images in .h5 files, which is much faster on cloud than single image files.</p>
283
284<p>To test DistNet, we provide a <a href="https://github.com/jeanollion/bacmman/wiki/DistNet" class="astrong" target="_blank">tutorial</a> for a <a href="https://github.com/jeanollion/bacmman" class="astrong" target="_blank">BACMMAN</a><sup><a class="citation" href="#ollion2019high">[6]</a></sup> module along with a sample dataset, as well as a <a href="https://github.com/jeanollion/bacmman/wiki/FineTune-DistNet" class="astrong" target="_blank">tutorial</a> to adapt it to other datasets with fine-tuning, on google colab, a service that provides free GPU.</p>
285
286<hr />
287<h3 id="references">References</h3>
288<ol class="bibliography"><li>
289
290
291<span id="ollion2020distnet">J. Ollion and C. Ollion, “DistNet: Deep Tracking by displacement regression: application to bacteria growing in the Mother Machine,” 2020.</span>
292
293<br />
294
295</li>
296<li>
297
298
299<span id="wang2010robust">P. Wang <i>et al.</i>, “Robust growth of Escherichia coli,” <i>Current biology</i>, vol. 20, no. 12, pp. 1099–1103, 2010.</span>
300
301<br />
302
303</li>
304<li>
305
306
307<span id="robert2018mutation">L. Robert, J. Ollion, J. Robert, X. Song, I. Matic, and M. Elez, “Mutation dynamics and fitness effects followed in single cells,” <i>Science</i>, vol. 359, no. 6381, pp. 1283–1286, 2018, [Online]. Available at: https://science.sciencemag.org/content/359/6381/1283.editor-summary.</span>
308
309<br />
310 
311<a href="https://science.sciencemag.org/content/359/6381/1283.editor-summary"> https://science.sciencemag.org/content/359/6381/1283.editor-summary</a> 
312
313</li>
314<li>
315
316
317<span id="ronneberger2015u">O. Ronneberger, P. Fischer, and T. Brox, “U-net: Convolutional networks for biomedical image segmentation,” in <i>International Conference on Medical image computing and computer-assisted intervention</i>, 2015, pp. 234–241, [Online]. Available at: https://arxiv.org/abs/1505.04597.</span>
318
319<br />
320 
321<a href="https://arxiv.org/abs/1505.04597"> https://arxiv.org/abs/1505.04597</a> 
322
323</li>
324<li>
325
326
327<span id="vaswani2017attention">A. Vaswani <i>et al.</i>, “Attention is all you need,” in <i>Advances in neural information processing systems</i>, 2017, pp. 5998–6008, [Online]. Available at: https://arxiv.org/abs/1706.03762.</span>
328
329<br />
330 
331<a href="https://arxiv.org/abs/1706.03762"> https://arxiv.org/abs/1706.03762</a> 
332
333</li>
334<li>
335
336
337<span id="ollion2019high">J. Ollion, M. Elez, and L. Robert, “High-throughput detection and tracking of cells and intracellular spots in mother machine experiments,” <i>Nature protocols</i>, vol. 14, no. 11, pp. 3144–3161, 2019, [Online]. Available at: https://rdcu.be/bRSze.</span>
338
339<br />
340 
341<a href="https://rdcu.be/bRSze"> https://rdcu.be/bRSze</a> 
342
343</li></ol>
344
345<!--next script to preload gifs in order to sync them-->
346<script src="/assets/js/imgrespsync.js"></script>
346
347
348<!--script src="https://utteranc.es/client.js"
349        repo="https://github.com/jeanollion/jeanollion.github.io"
350        issue-term="title"
351        theme="github-light"
352        crossorigin="anonymous">
353</script-->
354<!--script>
355// NOT WORKING YET!!
356$(document).ready(function() {
357  var alltwtw = $(".twentytwenty-container");
358  for (var i=0; i<alltwtw.length; i++) {
359    alltwtw[i].find("img:first").attr("height", "auto");
360    alltwtw[i].find("img:last").attr("height", "auto");
361  }
362});
363</script-->
364</div>
365	</article>
366</div>
367<script>
368$(function() {
369  var navSelector = "#post-toc";
370  var $myNav = $(navSelector);
371  Toc.init({
372     $nav: $myNav, 
373     $scope: $("#post-content")
374  });
375});
376$('body').scrollspy({ target: "#post-toc" })
377</script>
377
378<!-- Footer -->
379<div id="footer">
380  <!-- Copyright -->
381  <ul class="copyright">
382    
383      <li><sub><sub><sup>&copy;SABILab. All rights reserved.</sup></sup></sub></li>
384    
385    <li><sub><sub><sup>Template Design: <a href="http://html5up.net">HTML5 UP</a> Jekyll integration: <a href="https://github.com/chrisbobbe/jekyll-theme-prologue">Chris Bobbe</a></sup></sup></sub></li>
386  </ul>
387</div>
388</body>
389</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.