https://neu-pdi.github.io/cs2000-public-resources/assets/js/6cc19de0.9a9b5b6b.js

1"use strict";(self.webpackChunkinstructor_resources=self.webpackChunkinstructor_resources||[]).push([[3662],{31956:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>d,contentTitle:()=>l,default:()=>h,frontMatter:()=>r,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"4","title":"Practice 4 \u2014 Data Cleaning & Transformation","description":"Skills: 2","source":"@site/practice/4.md","sourceDirName":".","slug":"/4","permalink":"/cs2000-public-resources/practice/4","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":4,"frontMatter":{"sidebar_position":4,"hide_table_of_contents":true,"title":"Practice 4 \u2014 Data Cleaning & Transformation"},"sidebar":"practiceSidebar","previous":{"title":"Practice 3 \u2014 Working with Tables","permalink":"/cs2000-public-resources/practice/3"},"next":{"title":"Practice 5 \u2014 Lists","permalink":"/cs2000-public-resources/practice/5"}}');var a=i(74848),t=i(28453);const r={sidebar_position:4,hide_table_of_contents:!0,title:"Practice 4 \u2014 Data Cleaning & Transformation"},l=void 0,d={},c=[{value:"Skills: 2",id:"skills-2",level:2},{value:"Reading: 4.2",id:"reading-42",level:2},{value:"Examining Messy Data",id:"examining-messy-data",level:2},{value:"Identifying Data Quality Issues",id:"identifying-data-quality-issues",level:3},{value:"Cleaning Text Data",id:"cleaning-text-data",level:2},{value:"Standardizing Case",id:"standardizing-case",level:3},{value:"Standardizing State Names",id:"standardizing-state-names",level:3},{value:"Handling Missing and Invalid Data",id:"handling-missing-and-invalid-data",level:2},{value:"Dealing with Missing Ages",id:"dealing-with-missing-ages",level:3},{value:"Converting Text Numbers to Actual Numbers",id:"converting-text-numbers-to-actual-numbers",level:3},{value:"Computing New Columns",id:"computing-new-columns",level:2},{value:"Wrap-Up",id:"wrap-up",level:2},{value:"Reflection Questions:",id:"reflection-questions",level:3}];function o(e){const n={a:"a",code:"code",h2:"h2",h3:"h3",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,t.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsxs)(n.h2,{id:"skills-2",children:["Skills: ",(0,a.jsx)(n.a,{href:"/skills/#2",children:"2"})]}),"\n",(0,a.jsxs)(n.h2,{id:"reading-42",children:["Reading: ",(0,a.jsx)(n.a,{href:"https://dcic.pdi.run/processing-tables.html",children:"4.2"})]}),"\n",(0,a.jsx)(n.h2,{id:"examining-messy-data",children:"Examining Messy Data"}),"\n",(0,a.jsx)(n.p,{children:"Here is an intentionally messy table:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-pyret",children:'SURVEY-RAW = table: name, age, favorite-food, rating, state\n    row: "Alice", "19", "pizza", "8", "MA"\n    row: "Bob", "20", "PIZZA", "9", "ma"\n    row: "Carol", "", "tacos", "7", "California"\n    row: "David", "18", "Pizza", "ten", "Mass"\n    row: "Eve", "21", "sushi", "", "MA"\n    row: "Frank", "19", "ice cream", "6", "massachusetts"\n    row: "Grace", "twenty", "Pizza", "8", "CA"\n    row: "Henry", "19", "ICE CREAM", "5", "calif"\nend\n'})}),"\n",(0,a.jsx)(n.h3,{id:"identifying-data-quality-issues",children:"Identifying Data Quality Issues"}),"\n",(0,a.jsxs)(n.p,{children:["First, let's look at the data and identify problems (",(0,a.jsx)(n.strong,{children:"Inconsistent capitalization"}),", ",(0,a.jsx)(n.strong,{children:"Missing values"}),", ",(0,a.jsx)(n.strong,{children:"Inconsistent formats"}),", ",(0,a.jsx)(n.strong,{children:"Inconsistent state abbreviations"}),")"]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Discussion:"}),' "What questions would be hard to answer?"']}),"\n",(0,a.jsx)(n.h2,{id:"cleaning-text-data",children:"Cleaning Text Data"}),"\n",(0,a.jsx)(n.h3,{id:"standardizing-case",children:"Standardizing Case"}),"\n",(0,a.jsxs)(n.p,{children:["Can normalize text using string functions -- can ",(0,a.jsx)(n.code,{children:"build-column"})," a new column using ",(0,a.jsx)(n.code,{children:"string-to-lower"})," of existing ",(0,a.jsx)(n.code,{children:"favorite-food"})," column."]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Check Understanding:"}),' "Why might we want all food names in lowercase?"']}),"\n",(0,a.jsx)(n.h3,{id:"standardizing-state-names",children:"Standardizing State Names"}),"\n",(0,a.jsxs)(n.p,{children:["Create a more complex transformation for states -- design a function that first uses ",(0,a.jsx)(n.code,{children:"string-to-lower"})," but then checks using ",(0,a.jsx)(n.code,{children:"if"})," if the lowercased version is one of the ",(0,a.jsx)(n.code,{children:"ma"})," variants or one of the ",(0,a.jsx)(n.code,{children:"ca"})," variants and replaces each with a standard version."]}),"\n",(0,a.jsx)(n.h2,{id:"handling-missing-and-invalid-data",children:"Handling Missing and Invalid Data"}),"\n",(0,a.jsx)(n.h3,{id:"dealing-with-missing-ages",children:"Dealing with Missing Ages"}),"\n",(0,a.jsx)(n.p,{children:"What different strategies can we use for missing ages?"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Can remove the data. In this case, create a new table that only includes data where the age is present."}),"\n",(0,a.jsx)(n.li,{children:"Can put a default age. In this case, add a new column with a cleaned age\ncolumn, where you fill in a default age (say, 20) in case it was missing."}),"\n"]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Discussion:"}),' Ask, "When might you choose between filtering vs. filling missing data?"']}),"\n",(0,a.jsx)(n.h3,{id:"converting-text-numbers-to-actual-numbers",children:"Converting Text Numbers to A

1ctual Numbers"}),"\n",(0,a.jsxs)(n.p,{children:["If people wrote words instead of numbers, we can convert them. If they were\narbitrary numbers, this could be complex, but if we expect numbers to only be\n1-10 (e.g., in the ",(0,a.jsx)(n.code,{children:"rating"})," column), then can write a large ",(0,a.jsx)(n.code,{children:"if ... else if... else..."}),' with cases for "", "ten", "nine", ..., and then the last case we can\njust convert from a number like ',(0,a.jsx)(n.code,{children:'"8"'})," to a number 8 with ",(0,a.jsx)(n.code,{children:"string-to-number"}),"."]}),"\n",(0,a.jsxs)(n.p,{children:["Design such a helper function, and use it to add a ",(0,a.jsx)(n.code,{children:"rating-clean"})," column."]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Check Understanding:"}),' Ask, "Why should we use -1 for missing ratings instead of 0?"']}),"\n",(0,a.jsx)(n.h2,{id:"computing-new-columns",children:"Computing New Columns"}),"\n",(0,a.jsxs)(n.p,{children:["Sometimes we might also want to create computed columns based on existing data.\nFor example, maybe we want a ",(0,a.jsx)(n.code,{children:"rating-category"}),' column that distinguishes between\n"no rating" (-1, based on previous cleaning), "high" (above 7), medium (above\n5), and low.']}),"\n",(0,a.jsx)(n.p,{children:"Add such a column."}),"\n",(0,a.jsx)(n.h2,{id:"wrap-up",children:"Wrap-Up"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Incremental Cleaning:"})," Building columns step by step rather than trying to fix everything at once"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Trade-offs:"})," Different strategies for missing data have different implications"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"reflection-questions",children:"Reflection Questions:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:'"What would happen if we tried to analyze the original messy data?"'}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(o,{...e})}):o(e)}}}]);