PageSourceSearch

https://gael-varoquaux.info/programming/hiring-an-engineer-and-pos…to-simplify-data-science-on-dirty-data.html

html gael-varoquaux.info collected 2026-10-01 11:29:27 UTC 20,895 bytes, 465 lines download raw bytes

1<!doctype html>
2<html lang="en">
3<head>
4    <meta charset="utf-8">
5    <meta name="viewport" content="width=device-width, initial-scale=1.0">
6    <meta name="description" content="Gaël Varoquaux, computer / data / health science">
7
8        <link rel="alternate"  href="https://gael-varoquaux.info/feeds/all.atom.xml" type="application/atom+xml" title="Gaël Varoquaux Full Atom Feed"/>
9
10        <title>Hiring an engineer and post-doc to simplify data science on dirty data -- Gaël Varoquaux: computer / data / health science</title>
11
12    <link href="https://mastodon.social/@GaelVaroquaux" rel="me">
13
14    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/pure/0.3.0/pure-min.css">
15    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.0/css/all.min.css">
16    <link rel="stylesheet" href="https://gael-varoquaux.info/theme/css/pure.css">
17
18    
18<script src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js"></script>
18
19    <!-- OpenGraph -->
20	<meta property="og:type" content="article" />
21	<meta property="og:url" content="https://gael-varoquaux.info/programming/hiring-an-engineer-and-post-doc-to-simplify-data-science-on-dirty-data.html" />
22	<meta property="og:title" content="Hiring an engineer and post-doc to simplify data science on dirty data" />
23	<meta property="og:description" content="Note Join us to work on reinventing data-science practices and tools to produce robust analysis with less data curation. It is well known that data cleaning and preparation are a heavy burden..." />
24	<meta property="og:image" content="attachments/big_data_borat_cleaning_data.png" />
25	<meta property="og:site_name" content="Gaël Varoquaux" />
26    <meta property="og:logo" content="https://gael-varoquaux.info/images/gael.png" />
27</head>
28
29<body>
30<div class="pure-g-r" id="layout">
31    <div class="sidebar sidebar-article pure-u">
32        <header class="header-article">
33            <hgroup>
34                <a href="https://gael-varoquaux.info">
35		    <img class="avatar" alt="Gaël Varoquaux" 
36                     src="https://gael-varoquaux.info/images/gael.png">
37                </a>
38                <a href="https://gael-varoquaux.info" class="article-info">
39		    <h2 class="article-info">Gaël Varoquaux</h2>
40		</a>
41                <p>Fri 29 October 2021</p>
42                <a href="/">&larr;Home</a>
43            </hgroup>
44        </header>
45    </div>
46    <div class="pure-u">
47        <div class="content">
48            <section class="post">
49                <header class="post-header">
50                    <h1>Hiring an engineer and post-doc to simplify data science on dirty data</h1>
51                        <p class="post-meta">
52                            under                                 <a class="post-category" href="https://gael-varoquaux.info/tag/machine-learning.html">machine learning</a>
53                                <a class="post-category" href="https://gael-varoquaux.info/tag/data-science.html">data science</a>
54                                <a class="post-category" href="https://gael-varoquaux.info/tag/dirty-data.html">dirty data</a>
55                                <a class="post-category" href="https://gael-varoquaux.info/tag/hiring.html">hiring</a>
56		    <span class="readtime">
57			&nbsp Read time: 3 min.
58		    </span>
59<!--
59<script src="https://apis.google.com/js/platform.js" async defer></script>
59-->
60<span class="social_links">
61<!--<a href="http://twitter.com/share" class="twitter-share-button" data-count="horizontal" data-via="GaelVaroquaux">Tweet</a>
61<script type="text/javascript" src="http://platform.twitter.com/widgets.js" async defer></script>
61
62-->
63<!-- Place this tag where you want the +1 button to render. -->
64<span class="g-plusone" data-size="medium"></span>
65</span>                        </p>
66                </header>
67            </section>
68            <div class="admonition note">
69<p class="first admonition-title">Note</p>
70<p class="last">Join us to work on reinventing data-science practices and tools to
71produce robust analysis with less data curation.</p>
72</div>
73<p>It is well known that data cleaning and preparation are a heavy burden to
74the data scientist.</p>
75<img alt="" class="align-center" src="attachments/big_data_borat_cleaning_data.png" style="width: 400px;" />
76<div class="section" id="dirty-data-research">
77<h2>Dirty data research</h2>
78<p>In the <a class="reference external" href="https://project.inria.fr/dirtydata/">dirty data project</a>, we
79have been conducting machine-learning research to see how better
80statistical models could readily ingest non-curated data, and reduce the
81need of data preparation for data science. We now have a growing
82understanding of the problems, theoretical and practical, which lie
83across statistical and database topics.</p>
84<p>Machine learning leads to different tradeoffs than traditional
85inferential statistics (because it can rely on more powerful model). For
86instance, we now have a good understanding of the case of missing values:
87in <a class="reference external" href="https://arxiv.org/abs/2106.00311">Le Morvan et al</a>, we showed that
88with traditional methods, ignorable missingness <a class="footnote-reference" href="#footnote-1" id="footnote-reference-1">[1]</a> and “good”
89imputation are important, but it turns out for prediction, flexible
90predictors are what matters and they can work on any missingness
91mechanism.</p>
92<table class="docutils footnote" frame="void" id="footnote-1" rules="none">
93<colgroup><col class="label" /><col /></colgroup>
94<tbody valign="top">
95<tr><td class="label"><a class="fn-backref" href="#footnote-reference-1">[1]</a></td><td>“Missing at Random”, where missingness is independent of the
96hidden values</td></tr>
97</tbody>
98</table>
99<p>Similarly, we have made good progress on tolerating normalization errors
100and typos. We find that rather to attempt to deduplicate the entries or
101fix the typos, it is best to represent similarities and ambiguities to
102a flexible learning algorithm. The simplest and most reliable methods are
103implemented in the <a class="reference external" href="http://dirty-cat.github.io/">dirty-cat</a> library, to
104facilitate the life of data-scientists</p>
105</div>
106<div class="section" id="reinventing-data-science">
107<h2>Reinventing data science</h2>
108<p>With this understanding (and even more exciting on-going research), we
109want to revisit data science. Machine-learning can provide flexible
110models for many usages of data science. Our goal is to use it to help
111assembling and analyzing datasets while minimizing human efforts. For
112this, we need tools that can answer typical data-science questions using
113machine learning and starting from the raw data, often spread in multiple
114files or multiple tables of a databases. Building these tools requires
115data-science research, a new vision of data-science APIs, and careful
116software crafting.</p>
117</div>
118<div class="section" id="join-us-in-this-adventure">
119<h2>Join us in this adventure</h2>
120<p>We have an <a class="reference external" href="https://project.inria.fr/dirtydata/team/">awesome team</a>,
121with a great mix of people of different seniority, different expertise
122(statistics, machine learning, databases, software engineering), sharing
123offices with the <a class="reference external" href="https://scikit-learn.fondation-inria.fr/home/">scikit-learn at Inria</a>. But we have too many
124exciting ideas, so we are growing this team.</p>
125<div class="section" id="a-data-science-engineer-new-software-with-new-ideas">
126<h3>A data-science engineer: new software with new ideas</h3>
127<p>We are looking for someone with a background in data science or numerical
128Python programming to join us, to help with designing a new data-science
129library, evolving from <a class="reference external" href="http://dirty-cat.github.io/">dirty-cat</a>, and
130to help with data-science experimentation for the research.</p>
131<p>We like people who care about data, designing good tools, and have vision
132about data science. We are happy to consider different level of
133experience. Apply on <a class="reference external" href="https://jobs.inria.fr/public/classic/fr/offres/2021-04182">the job offer</a>.</p>
134</div>
135<div class="section" id="a-post-doc-researcher-science-joining-data-engineering-to-deep-learning">
136<h3>A post-doc researcher: science joining data engineering to deep learning</h3>
137<p>We will soon be announcing a post-doc position to join the team for
138research in this scope. We are interested in questions around learning on
139relational or tabular data, or learning data integration. We have plenty
140of ideas to explore around embeddings in databases, learning to
141aggregate, learning on sets, graph neural networks for databases, or
142distributional matching for entity and schema alignment.
143We expect to be borrowing tools (conceptual and practical) from deep
144learning, but to blending them with techniques from data integration,
145knowledge graphs, and databases.</p>
146<p>The job posting will be out soon, but I am running out of the office
147right now for vacations (work-life balance also matters to us).</p>
148<div class="topic">
149<p class="topic-title">Diversity is important</p>
150<p><a class="reference external" href="https://project.inria.fr/dirtydata/team/">Our team</a> is not as
151diverse as I would like it to be (though probably doing better than
152typical computer-science team). We love diverse candidates. Do not
153hesitate.</p>
154</div>
155</div>
156</div>
157
158            <div class="hr" style="margin-bottom: -.5em;"></div>
159<!--
159<script src="https://apis.google.com/js/platform.js" async defer></script>
159-->
160<span class="social_links">
161<!--<a href="http://twitter.com/share" class="twitter-share-button" data-count="horizontal" data-via="GaelVaroquaux">Tweet</a>
161<script type="text/javascript" src="http://platform.twitter.com/widgets.js" async defer></script>
161
162-->
163<!-- Place this tag where you want the +1 button to render. -->
164<span class="g-plusone" data-size="medium"></span>
165</span>            <a href="#" class="go-top">Go Top</a>
166<footer class="footer">
167    <p>&copy; Gaël Varoquaux &ndash;
168        Built with <a href="https://github.com/PurePelicanTheme/pure">Pure Theme</a>
169        for <a href="http://blog.getpelican.com/">Pelican</a>
170    &ndash;
171    <a href="https://gael-varoquaux.info/legal.html">Mentions légales</a>
172    </p>
173</footer>        </div>
174    </div>
175</div>
176    
176<script
177src="//cdnjs.cloudflare.com/ajax/libs/fitvids/1.0.1/jquery.fitvids.min.js"></script>
177
178    
178<script>
179        $(document).ready(function(){
180            $(".content").fitVids();
181        });
182    </script>
182
183    
183<script>
184    // Highlight the current section in the TOC
185(function () {
186    'use strict';
187    var SectionScroller = {
188        anchorTops: [],
189
190        el: {
191            anchors: document.querySelectorAll('.toc-backref'),
192            anchorLinks: document.querySelectorAll('a.reference.internal')
193        },
194        
195        forEach: function(array, callback, scope) {
196            for (var i = 0, ii = array.length; i < ii; i++) {
197                callback.call(scope, i, array[i]);
198            }
199        },
200
201        throttle: function (fn, threshhold, scope) {
202          threshhold = threshhold || (threshhold = 250);
203          var last;
204          var deferTimer;
205          return function () {
206            var context = scope || this;
207            var now = +new Date();
208            var args = arguments;
209            if (last && now < last + threshhold) {
210              // hold on to it
211              clearTimeout(deferTimer);
212              deferTimer = setTimeout(function () {
213                last = now;
214                fn.apply(context, args);
215              }, threshhold);
216            } else {
217              last = now;
218              fn.apply(context, args);
219            }
220          };
221        },
222        
223        mathSign: function (x) {
224            x = +x; // convert to a number
225            if (x === 0 || isNaN(x)) {
226                return x;
227            }
228            return x > 0 ? 1 : -1;
229        },
230
231        anchorGetter: function () {
232            var SS = SectionScroller;
233            for (var i = 0, max = SS.el.anchors.length; i < max; i++) {
234                SS.anchorTops[i] = SS.el.anchors[i].offsetTop;
235            }
236            for (var j = 0, jj = SS.anchorTops.length; j < jj; j++) {
237                if (SS.anchorTops[j] - 350 < window.scrollY) {
238                    for (var x = 0, xx = SS.el.anchors.length; x < xx; x++) {
239                        SS.el.anchorLinks[x].classList.remove('selected');
240                    }
241                    SS.el.anchorLinks[j].classList.add('selected');
242                }
243            }
244        },
245        
246        smooth: function (e) {
247            var id = e.currentTarget.getAttribute('href');
248            var node = document.querySelector(id);
249            var nodeTop = node.offsetTop;
250            var winTop = window.scrollY;
251            var sign = SectionScroller.mathSign(nodeTop);
252            var scrollAmnt;
253            var down; 
254            if (nodeTop > winTop) {
255                down = true;
256                scrollAmnt = nodeTop - winTop;
257            } else {
258                down = false;
259                scrollAmnt = Math.abs(winTop - nodeTop);
260            }
261            
262            var scroller = function () {
263                if (down) {
264                    window.scrollTo(0, window.scrollY + 1);
265                } else {
266                    window.scrollTo(0, window.scrollY - 1);
267                }
268                scrollAmnt--;
269                if (scrollAmnt > 0)  {
270                    window.requestAnimationFrame(scroller);
271                }
272            };
273            window.requestAnimationFrame(scroller);
274            e.preventDefault();
275        },
276        
277        smoothScroll: function(e) {
278            var id = e.currentTarget.getAttribute('href');
279            var node = document.querySelector(id);
280            var scrollContainer = node;
281            do { //find scroll container
282                scrollContainer = scrollContainer.parentNode;
283                if (!scrollContainer) return;
284                scrollContainer.scrollTop += 1;
285            } while (scrollContainer.scrollTop === 0);
286
287            var targetY = 0;
288            do { //find the top of target relatively to the container
289                if (node == scrollContainer) break;
290                targetY += node.offsetTop;
291            } while (node === node.offsetParent);
292
293            var scroll = function(c, a, b, i) {
294                i++; if (i > 30) return;
295                c.scrollTop = a + (b - a) / 30 * i;
296                setTimeout(function(){ scroll(c, a, b, i); }, 20);
297            };
298            // start scrolling
299            scroll(scrollContainer, scrollContainer.scrollTop, targetY, 0);
300            e.preventDefault();
301        },
302
303        events: function () {
304            var SS = SectionScroller;
305            window.addEventListener('scroll', SS.throttle(SS.anchorGetter, 150));
306            SS.forEach(SS.el.anchorLinks, function (index, link) {
307                link.addEventListener('click', SS.smooth);
308            });
309        },
310
311        init: function () {
312            SectionScroller.anchorGetter();
313            SectionScroller.events();
314        }
315    };
316
317    SectionScroller.init();
318})();
319</script>
319
320    
320<script>
321        var $top = $('.go-top');
322
323        // Show or hide the sticky footer button
324        $(window).scroll(function() {
325            if ($(this).scrollTop() > 200) {
326                $top.fadeIn(200);
327            } else {
328                $top.fadeOut(200);
329            }
330        });
331
332        // Animate the scroll to top
333        $top.click(function(event) {
334            event.preventDefault();
335            $('html, body').animate({scrollTop: 0}, 300);
336        })
337
338        // Makes sure that the href="#" attached to the <a> elements
339        // don't scroll you back up the page.
340        $('body').on('click', 'a[href="#"]', function(event) {
341            event.preventDefault();
342        });
343    </script>
343
344    
344<script  language="JavaScript" type="text/javascript">
345    /* From http://evirtus.net/pub/eflickrstream.asp */
346    /* Global variables */
347    var bolOnLoadRun=false;
348    var bolFlickrRun=false;
349    var objFlickrOutput=null;
350    
351    /* OnLoad events */
352    window.onload = function() {
353    bolOnLoadRun=true;
354    if (! /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent) ) {
355	eFlickrStream();
356    };
357    }
358    
359    /*
360    eFlickrStream - Lovely, lovely photos! :-)
361    - 02.08.2007 @ 14:33: Initial version
362    */
363    function eFlickrStream() {
364    if (!bolFlickrRun ) { return false; } else { bolFlickrRun=false; }
365    if ((!document.createElement) && (!document.getElementById) && (!document.getElementsByTagName) ) { return false; }// Test for required browser capabilities
366    
367    /*   *** Options/start ***   */
368    var strTarget='flickrstream';// Target container for the flickr badge (must allready exist in the document)
369    /*   *** Options/end ***     */
370    
371    if (!document.getElementById(strTarget)){return false;}// Exsist if target doesn't exsist
372    
373    var objTarget=document.getElementById(strTarget);
374    objTarget.appendChild(objFlickrOutput);// Append output
375    
376    objFlickrOutput=null;// We're done, delete stored variable value
377    objFlickr=null;
378    }
379    
380    /*
381    jsonFlickrFeed - Triggered when the Flickr json "feed" is loaded, generates Flickr photos output list item (Deefer "trigger method" from Patrick Quinn-Graham's DOM Flickr Badge)
382    - 02.08.2007 @ 14:33: Initial version
383    - 23.11.2007 @ 08:24: Updated to use eSetAttr(), enables the flickr link in IE
384    - 29.11.2007 @ 05:52: Some changes for easier configuration (of both options and text-strings)
385    */
386    function jsonFlickrFeed(objFlickr) {
387    if ((!document.createElement) && (!document.getElementById) && (!document.getElementsByTagName) ) { return false; }// Test for required browser capabilities
388    
389    /*   *** Options/start ***   */
390    var intMaxImages=10;// Maximum number of images to show (1-20)
391    var bolThumbSquare=true;// Load the square thumbnails
392    var intThumbWidth=75;// Thumbnail width (omitted if set to "0")
393    var strLightbox='flickrstream';// Lightbox "group name" (define to mark the thumbnails for lightbox usage, adds rel="lightbox[value]" to image links)
394    var strImgAlt='[title]';// Alternative title for images ("[title]" will be replaced by image title)
395    var strImgLinkTitle='View a larger version of \"[title]\"';// Link title for the image link
396    /*   *** Options/end ***     */
397    
398    var objElm,objTxt,objImg,objLnk,objTmp,strTmp;
399    var intPhotos=objFlickr.items.length;
400    var objOut=document.createElement('ul');
401    objOut.setAttribute('class','flickrlist');
402    
403    for (var iCntA=0; ( (iCntA<intPhotos) && (iCntA<intMaxImages)); iCntA++) {
404    
405	objElm=document.createElement('li');// Create item container element (<li>)
406    
407	objLnk=document.createElement('a');// Create link (to the large(r) photo)
408	objLnk.setAttribute('href',objFlickr.items[iCntA].link);
409	objLnk.setAttribute('title',strImgLinkTitle.replace('[title]',objFlickr.items[iCntA].title));
410	if (strLightbox!=''){objLnk.setAttribute('rel','lightbox['+strLightbox+']');}// Add relation value for lightbox usage
411    
412	strTmp=objFlickr.items[iCntA].media.m;// Retreive thumbnail URI
413	if (bolThumbSquare){strTmp=strTmp.replace('_m.jpg','_s.jpg');}// Swap default thumbnail for square...
414    
415	objImg=document.createElement('img');// Main thumbnail
416	eSetAttr(objImg,'class','photo');
417	objImg.setAttribute('src',strTmp);
418	objImg.setAttribute('alt',strImgAlt.replace('[title]',objFlickr.items[iCntA].title));
419	if (intThumbWidth!=0){objImg.setAttribute('width',intThumbWidth);}
420    
421	objLnk.appendChild(objImg);// Append thumbnail to link
422	objElm.appendChild(objLnk);// Append link to container (li)
423
424	objOut.appendChild(objElm);// Append container to output object
425	}
426    
427    objFlickrOutput=objOut;// Store output for later use...
428    
429    if (bolOnLoadRun) {
430	eFlickrStream();
431	}
432    else {
433	bolFlickrRun=true;
434	}
435    }
436    
437    /*
438    eSetAttr/eGetAttr - An attempt to tame IE's weird DOM model
439    - 22.11.2007 @ 12:50 - Initial version
440    */
441    function eSetAttr(objTarget,strAttr,strValue) {
442    if (typeof window.attachEvent!='undefined' && typeof window.opera=='undefined') {
443    //    //Set attributes for IE
444	switch (strAttr) {
445	    case('class'):
446		objTarget.setAttribute('className',strValue);
447		break;
448	    case('style'):
449		objTarget.style.cssText=strValue;
450		break;
451	    default:
452		objTarget.setAttribute(strAttr,strValue);
453		break;
454	    }
455	}
456    else {
457    //    //Set attributes for /Other browsers/
458	objTarget.setAttribute(strAttr,strValue);
459	}
460    //66885349@N03
461    }
462    </script>
462
463    
463<script type="text/javascript" src="https://api.flickr.com/services/feeds/photos_public.gne?id=66885349@N03&amp;format=json&amp;" defer="defer"></script>
463
464</body>
465</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.