1"use strict";(self.webpackChunkegoexo_docs=self.webpackChunkegoexo_docs||[]).push([[7823],{3905:(e,n,t)=>{t.d(n,{Zo:()=>c,kt:()=>f});var a=t(7294);function o(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}function r(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?i(Object(t),!0).forEach((function(n){o(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):i(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function l(e,n){if(null==e)return{};var t,a,o=function(e,n){if(null==e)return{};var t,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=a.createContext({}),d=function(e){var n=a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):r(r({},n),e)),t},c=function(e){var n=d(e.components);return a.createElement(s.Provider,{value:n},e.children)},_="mdxType",p={inlineCode:"code",wrapper:function(e){var n=e.children;return a.createElement(a.Fragment,{},n)}},m=a.forwardRef((function(e,n){var t=e.components,o=e.mdxType,i=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),_=d(t),m=o,f=_["".concat(s,".").concat(m)]||_[m]||p[m]||i;return t?a.createElement(f,r(r({ref:n},c),{},{components:t})):a.createElement(f,r({ref:n},c))}));function f(e,n){var t=arguments,o=n&&n.mdxType;if("string"==typeof e||o){var i=t.length,r=new Array(i);r[0]=m;var l={};for(var s in n)hasOwnProperty.call(n,s)&&(l[s]=n[s]);l.originalType=e,l[_]="string"==typeof e?e:o,r[1]=l;for(var d=2;d<i;d++)r[d]=t[d];return a.createElement.apply(null,r)}return a.createElement.apply(null,t)}m.displayName="MDXCreateElement"},273:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>s,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>l,toc:()=>d});var a=t(7462),o=(t(7294),t(3905));const i={},r="Tutorial 3: Undistort frames and overlay annotations",l={unversionedId:"tutorials/undistort",id:"tutorials/undistort",title:"Tutorial 3: Undistort frames and overlay annotations",description:"Status",source:"@site/docs/tutorials/undistort.md",sourceDirName:"tutorials",slug:"/tutorials/undistort",permalink:"/tutorials/undistort",draft:!1,tags:[],version:"current",frontMatter:{},sidebar:"tutorialSidebar",previous:{title:"Tutorial 2: Hand and Body Pose in Ego-Exo4D Dataset",permalink:"/tutorials/pose"},next:{title:"EgoExo4D Challenge 2026",permalink:"/challenge"}},s={},d=[{value:"1. Prerequisites and Imports",id:"1-prerequisites-and-imports",level:3},{value:"2. Undistortion functions",id:"2-undistortion-functions",level:3},{value:"2.1 Aria Undistortion Utilities",id:"21-aria-undistortion-utilities",level:4},{value:"2.2 GoPro Undistortion Utilities",id:"22-gopro-undistortion-utilities",level:4},{value:"3. Overlay 2D annotations on undistorted frames",id:"3-overlay-2d-annotations-on-undistorted-frames",level:3},{value:"3.1 load 2D hand pose annotation and overlay it on undistorted egocentric frame",id:"31-load-2d-hand-pose-annotation-and-overlay-it-on-undistorted-egocentric-frame",level:4},{value:"3.2 load 2D body pose annotation and overlay it on undistorted exocentric frames",id:"32-load-2d-body-pose-annotation-and-overlay-it-on-undistorted-exocentric-frames",level:4},{value:"Conclusion",id:"conclusion",level:3}],c={toc:d},_="wrapper";function p(e){let{components:n,...i}=e;return(0,o.kt)(_,(0,a.Z)({},c,i,{components:n,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"tutorial-3-undistort-frames-and-overlay-annotations"},"Tutorial 3: Undistort frames and overlay annotations"),(0,o.kt)("p",null,(0,o.kt)("img",{parentName:"p",src:"https://img.shields.io/static/v1.svg?label=Status&message=Finished&color=green",alt:"Status"})),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Filled notebook:"),"\n",(0,o.kt)("a",{parentName:"p",href:"https://github.com/facebookresearch/Ego4d/tree/main/notebooks/egoexo"},(0,o.kt)("img",{parentName:"a",src:"https://img.shields.io/static/v1.svg?logo=github&label=Tutorial&message=View%20On%20Github&color=lightgrey",alt:"View on Github"})),"\n",(0,o.kt)("strong",{parentName:"p"},"Author:")," Xizi Wang"),(0,o.kt)("p",null,"Cameras on the Aria glasses and GoPro cameras might have lenses that can cause distortions. For example, cameras on the Aria glasses all have fisheye lenses, and spherical camera model are much better approximations for these glasses. Some of the annotations in the EgoExo4D dataset are done on the undistorted frames. Thus, we need to undistort the frames f
1irst, then we could overlay the 2D annotations on the frames. In this tutorial, we provide a step-by-step guide on undistorting the egocentric and exocentric frames, and overlay the annotations on them."),(0,o.kt)("h3",{id:"1-prerequisites-and-imports"},"1. Prerequisites and Imports"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'%matplotlib inline\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport math\nfrom matplotlib import rcParams\nfrom projectaria_tools.core import mps\nfrom projectaria_tools.core import data_provider\nfrom projectaria_tools.core import calibration\n#from projectaria_tools.core.calibration import CameraCalibration, KANNALA_BRANDT_K3\n\nrcParams["figure.figsize"] = 16, 32\n\nimport json\nimport os\nimport random\n\nimport av\nimport pandas as pd\nfrom PIL import Image, ImageDraw, ImageFont\n')),(0,o.kt)("p",null,"Next we define some necessary utility functions for retrieving pose metadata and drawing functions. These are the same as the utility functions in the pose tutorial."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'# Loads dataframe at target path to csv\ndef load_csv_to_df(filepath: str) -> pd.DataFrame:\n with open(filepath, "r") as csv_file:\n return pd.read_csv(csv_file)\n\n# color palette for drawing the keypoints\npalette = np.array(\n [\n [255, 128, 0],\n [255, 153, 51],\n [255, 178, 102],\n [230, 230, 0],\n [255, 153, 255],\n [153, 204, 255],\n [255, 102, 255],\n [255, 51, 255],\n [102, 178, 255],\n [51, 153, 255],\n [255, 153, 153],\n [255, 102, 102],\n [255, 51, 51],\n [153, 255, 153],\n [102, 255, 102],\n [51, 255, 51],\n [0, 255, 0],\n [0, 0, 255],\n [255, 0, 0],\n [255, 255, 255],\n ]\n)\n\n\n# retrieve the body keypoints and skeleton\ndef get_body_metadata():\n keypoints_map = [\n {"label": "Nose", "id": "fee3cbd2", "color": "#f77189"},\n {"label": "Left-eye", "id": "ab12de34", "color": "#d58c32"},\n {"label": "Right-eye", "id": "7f2g1h6k", "color": "#a4a031"},\n {"label": "Left-ear", "id": "mn0pqrst", "color": "#50b131"},\n {"label": "Right-ear", "id": "yz89wx76", "color": "#34ae91"},\n {"label": "Left-shoulder", "id": "5a4b3c2d", "color": "#37abb5"},\n {"label": "Right-shoulder", "id": "e1f2g3h4", "color": "#3ba3ec"},\n {"label": "Left-elbow", "id": "6i7j8k9l", "color": "#bb83f4"},\n {"label": "Right-elbow", "id": "uv0wxy12", "color": "#f564d4"},\n {"label": "Left-wrist", "id": "3z4ab5cd", "color": "#2fd4aa"},\n {"label": "Right-wrist", "id": "efgh6789", "color": "#94d14f"},\n {"label": "Left-hip", "id": "ijklmnop", "color": "#b3d32c"},\n {"label": "Right-hip", "id": "qrstuvwx", "color": "#f9b530"},\n {"label": "Left-knee", "id": "yz012345", "color": "#83f483"},\n {"label": "Right-knee", "id": "6bc7defg", "color": "#32d58c"},\n {"label": "Left-ankle", "id": "hijk8lmn", "color": "#3ba3ec"},\n {"label": "Right-ankle", "id": "opqrs1tu", "color": "#f564d4"},\n ]\n\n # pyre-ignore\n pose_kpt_color = palette[[16, 16, 16, 16, 16, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0]]\n\n skeleton = [\n [16, 14],\n [14, 12],\n [17, 15],\n [15, 13],\n [12, 13],\n [6, 12],\n [7, 13],\n [6, 7],\n [6, 8],\n [7, 9],\n [8, 10],\n [9, 11],\n [2, 3],\n [1, 2],\n [1, 3],\n [2, 4],\n [3, 5],\n [4, 6],\n [5, 7],\n ]\n return keypoints_map, skeleton, pose_kpt_color\n\n\n# retrieve the hand keypoints and skeleton\ndef get_hands_metadata():\n keypoints_map = [\n {"label": "Right_Wrist", "id": "fee3cbd2", "color": "#f77189"},\n {"label": "Right_Thumb_1", "id": "yz012345", "color": "#83f483"},\n {"label": "Right_Thumb_2", "id": "6bc7defg", "color": "#32d58c"},\n {"label": "Right_Thumb_3", "id": "hijk8lmn", "color": "#3ba3ec"},\n {"label": "Right_Thumb_4", "id": "opqrs1tu", "color": "#f564d4"},\n {"label": "Right_Index_1", "id": "ab12de34", "color": "#d58c32"},\n {"label": "Right_Index_2", "id": "7f2g1h6k", "color": "#a4a031"},\n {"label": "Right_Index_3", "id": "mn0pqrst", "color": "#50b131"},\n {"label": "Right_Index_4", "id": "9vwxyzab", "color": "#32d58c"},\n {"label": "Right_Middle_1", "id": "yz89wx76", "color": "#34ae91"},\n {"label": "Right_Middle_2", "id": "5a4b3c2d", "color": "#37abb5"},\n {"label": "Right_Middle_3", "id": "e1f2g3h4", "color": "#3ba3ec"},\n {"label": "Right_Middle_4", "id": "cdefgh23", "color": "#3ba3ec"},\n {"label": "Right_Ring_1", "id": "efgh6789", "color": "#94d14f"},\n {"label": "Right_Ring_2", "id": "ijklmnop", "color": "#b3d32c"},\n {"label": "Right_Ring_3", "id": "qrstuvwx", "color": "#f9b530"},\n {"label": "Right_Ring_4", "id": "ijkl4567", "color": "#bb83f4"},\n {"label": "Right_Pinky_1", "id": "6i7j8k9l", "color": "#bb83f4"},\n {"label": "Right_Pinky_2", "id": "uv0wxy12", "color": "#f564d4"},\n {"label": "Right_Pinky_3", "id": "3z4ab5cd", "color": "#2fd4aa"},\n {"label": "Right_Pinky_4", "id": "mnop8qrs", "color": "#f564d4"},\n {"label": "Left_Wrist", "id": "fee3cbd2_left", "color": "#f77189"},\n {"label": "Left_Thumb_1", "id": "yz012345_left", "color": "#83f483"},\n {"label": "Left_Thumb_2", "id": "6bc7defg_left", "color": "#32d58c"},\n {"label": "Left_Thumb_3", "id": "hijk8lmn_left", "color": "#3ba3ec"},\n {"label": "Left_Thumb_4", "id": "opqrs1tu_left", "color": "#f564d4"},\n {"label": "Left_Index_1", "id": "ab12de34_left", "color": "#d58c32"},\n {"label": "Left_Index_2", "id": "7f2g1h6k_left", "color": "#a4a031"},\n {"label": "Left_Index_3", "id": "mn0pqrst_left", "color": "#50b131"},\n {"label": "Left_Index_4", "id": "9vwxyzab_left", "color": "#32d58c"},\n {"label": "Left_Middle_1", "id": "yz89wx76_left", "color": "#34ae91"},\n {"label": "Left_Middle_2", "id": "5a4b3c2d_left", "color": "#37abb5"},\n {"label": "Left_Middle_3", "id": "e1f2g3h4_left", "color": "#3ba3ec"},\n {"label": "Left_Middle_4", "id": "cdefgh23_left", "color": "#3ba3ec"},\n {"label": "Left_Ring_1", "id": "efgh6789_left", "color": "#94d14f"},\n {"label": "Left_Ring_2", "id": "ijklmnop_left", "color": "#b3d32c"},\n {"label": "Left_Ring_3", "id": "qrstuvwx_left", "color": "#f9b530"},\n {"label": "Left_Ring_4", "id": "ijkl4567_left", "color": "#bb83f4"},\n {"label": "Left_Pinky_1", "id": "6i7j8k9l_left", "color": "#bb83f4"},\n {"label": "Left_Pinky_2", "id": "uv0wxy12_left", "color": "#f564d4"},\n {"label": "Left_Pinky_3", "id": "3z4ab5cd_left", "color": "#2fd4aa"},\n {"label": "Left_Pinky_4", "id": "mnop8qrs_left", "color": "#f564d4"},\n ]\n\n links = {\n "fee3cbd2": ["ab12de34", "yz89wx76", "6i7j8k9l", "efgh6789", "yz012345"],\n "ab12de34": ["7f2g1h6k"],\n "7f2g1h6k": ["mn0pqrst"],\n "mn0pqrst": ["9vwxyzab"],\n "yz89wx76": ["5a4b3c2d"],\n "5a4b3c2d": ["e1f2g3h4"],\n "e1f2g3h4": ["cdefgh23"],\n "6i7j8k9l": ["uv0wxy12"],\n "uv0wxy12": ["3z4ab5cd"],\n "3z4ab5cd": ["mnop8qrs"],\n "efgh6789": ["ijklmnop"],\n "ijklmnop": ["qrstuvwx"],\n "qrstuvwx": ["ijkl4567"],\n "yz012345": ["6bc7defg"],\n "6bc7defg": ["hijk8lmn"],\n "hijk8lmn": ["opqrs1tu"],\n "fee3cbd2_left": [\n "ab12de34_left",\n "yz89wx76_left",\n "6i7j8k9l_left",\n "efgh6789_left",\n "yz012345_left",\n ],\n "ab12de34_left": ["7f2g1h6k_left"],\n "7f2g1h6k_left": ["mn0pqrst_left"],\n "mn0pqrst_left": ["9vwxyzab_left"],\n "yz89wx76_left": ["5a4b3c2d_left"],\n "5a4b3c2d_left": ["e1f2g3h4_left"],\n "e1f2g3h4_left": ["cdefgh23_left"],\n "6i7j8k9l_left": ["uv0wxy12_left"],\n "uv0wxy12_left": ["3z4ab5cd_left"],\n "3z4ab5cd_left": ["mnop8qrs_left"],\n "efgh6789_left": ["ijklmnop_left"],\n "ijklmnop_left": ["qrstuvwx_left"],\n "qrstuvwx_left"
1: ["ijkl4567_left"],\n "yz012345_left": ["6bc7defg_left"],\n "6bc7defg_left": ["hijk8lmn_left"],\n "hijk8lmn_left": ["opqrs1tu_left"],\n }\n\n hand_dict = dict()\n for index, kpt in enumerate(keypoints_map):\n hand_dict[kpt["id"]] = index + 1\n\n skeleton = list()\n for start_point in links:\n end_points = links[start_point]\n for end_point in end_points:\n start_index = hand_dict[start_point]\n end_index = hand_dict[end_point]\n skeleton.append([start_index, end_index])\n\n klist = [0]\n klist.extend([2] * 4)\n klist.extend([4] * 4)\n klist.extend([6] * 4)\n klist.extend([8] * 4)\n klist.extend([10] * 4)\n klist.extend([0])\n klist.extend([2] * 4)\n klist.extend([4] * 4)\n klist.extend([6] * 4)\n klist.extend([8] * 4)\n klist.extend([10] * 4)\n\n pose_kpt_color = palette[klist]\n return keypoints_map, skeleton, pose_kpt_color\n\ndef get_coords(annot):\n pts = dict()\n for k in annot:\n atype = 1\n if annot[k]["placement"] == "auto":\n atype = 0\n pts[k] = [annot[k]["x"], annot[k]["y"], atype]\n return pts\n\n\ndef draw_skeleton(img, all_pts, skeleton):\n draw = ImageDraw.Draw(img)\n for item in skeleton:\n left_index = item[0] - 1\n right_index = item[1] - 1\n left_pt = all_pts[left_index]\n right_pt = all_pts[right_index]\n if len(left_pt) == 0 or len(right_pt) == 0:\n continue\n draw.line([left_pt, right_pt], fill="white", width=10)\n\n\ndef draw_cross(img, x, y, color):\n draw = ImageDraw.Draw(img)\n # Circle parameters\n center = (x, y) # Center of the cross\n cross_length = 10 # Half-length of the cross arms\n # Calculate the end points of the cross\n left_point = (center[0] - cross_length, center[1])\n right_point = (center[0] + cross_length, center[1])\n top_point = (center[0], center[1] - cross_length)\n bottom_point = (center[0], center[1] + cross_length)\n\n # Draw the horizontal line\n draw.line([left_point, right_point], fill=color, width=3)\n # Draw the vertical line\n draw.line([top_point, bottom_point], fill=color, width=3)\n\n\ndef draw_circle(img, x, y, color):\n draw = ImageDraw.Draw(img)\n # Circle parameters\n center = (x, y) # Center of the circle\n radius = 12 # Radius of the circle\n\n # Calculate the bounding box of the circle\n left_up_point = (center[0] - radius, center[1] - radius)\n right_down_point = (center[0] + radius, center[1] + radius)\n\n # Draw the circle with a black outline\n draw.ellipse(\n [left_up_point, right_down_point], outline=(255, 255, 255), fill=color, width=6\n )\n\n\ndef draw_label(img, x, y, color, label):\n draw = ImageDraw.Draw(img)\n font = ImageFont.load_default(size=40)\n # Circle parameters\n center = (x + 20, y - 20) # Center of the circle\n draw.text(center, label, fill=color, font=font)\n\n\ndef draw_skeleton_hands(img, all_pts, skeleton, ratio=1):\n draw = ImageDraw.Draw(img)\n for item in skeleton:\n left_index = item[0] - 1\n right_index = item[1] - 1\n left_pt = all_pts[left_index]\n right_pt = all_pts[right_index]\n if len(left_pt) == 0 or len(right_pt) == 0:\n continue\n draw.line([left_pt, right_pt], fill="white", width=int(ratio * 4))\n\n\ndef draw_circle_hands(img, x, y, color, ratio=1):\n draw = ImageDraw.Draw(img)\n # Circle parameters\n center = (x, y) # Center of the circle\n radius = int(ratio * 8) # Radius of the circle\n\n # Calculate the bounding box of the circle\n left_up_point = (center[0] - radius, center[1] - radius)\n right_down_point = (center[0] + radius, center[1] + radius)\n\n # Draw the circle with a black outline\n draw.ellipse(\n [left_up_point, right_down_point],\n outline=(255, 255, 255),\n fill=color,\n width=int(ratio * 4),\n )\n\n\ndef draw_cross_hands(img, x, y, color, ratio=1):\n draw = ImageDraw.Draw(img)\n # Circle parameters\n center = (x, y) # Center of the cross\n cross_length = int(ratio * 8) # Half-length of the cross arms\n # Calculate the end points of the cross\n left_point = (center[0] - cross_length, center[1])\n right_point = (center[0] + cross_length, center[1])\n top_point = (center[0], center[1] - cross_length)\n bottom_point = (center[0], center[1] + cross_length)\n\n # Draw the horizontal line\n draw.line([left_point, right_point], fill=color, width=int(ratio * 4))\n # Draw the vertical line\n draw.line([top_point, bottom_point], fill=color, width=int(ratio * 4))\n\n\ndef show_results(results):\n for cam_name in results:\n img = results[cam_name]\n plt.figure(figsize=(40, 20))\n plt.imshow(img)\n plt.axis("off") # Hide the axes ticks\n plt.title(f"{cam_name}", fontsize=20)\n plt.savefig(f"{cam_name}.png")\n #plt.show()\n\n\ndef get_viz(\n pil_img,\n keypoints_map,\n ann,\n skeleton,\n pose_kpt_color,\n annot_type="body",\n is_aria=False,\n):\n pts = get_coords(ann)\n ratio = 1\n if is_aria:\n ratio = 0.5\n\n all_pts = []\n for _, keypoints in enumerate(keypoints_map):\n kpname = keypoints["label"].lower()\n\n if kpname in pts:\n x, y = pts[kpname][0], pts[kpname][1]\n all_pts.append((x, y))\n else:\n all_pts.append(())\n\n if annot_type == "body":\n draw_skeleton(pil_img, all_pts, skeleton)\n else:\n draw_skeleton_hands(pil_img, all_pts, skeleton, ratio)\n\n for index, keypoints in enumerate(keypoints_map):\n kpname = keypoints["label"].lower()\n if kpname in pts:\n x, y, pt_type = pts[kpname][0], pts[kpname][1], pts[kpname][2]\n color = tuple(pose_kpt_color[index])\n if pt_type == 1:\n if annot_type == "body":\n draw_circle(pil_img, x, y, color)\n else:\n draw_circle_hands(pil_img, x, y, color, ratio)\n else:\n if annot_type == "body":\n draw_cross(pil_img, x, y, color)\n else:\n draw_cross_hands(pil_img, x, y, color, ratio)\n if annot_type == "body":\n draw_label(pil_img, x, y, color, kpname)\n\n else:\n pass\n return pil_img\n\n\n# Video Utilities\ndef get_frame(video_local_path, frame_idx):\n container = av.open(video_local_path)\n frame_count = 0\n for frame in container.decode(video=0):\n if frame_count == frame_idx:\n input_img = np.array(frame.to_image())\n pil_img = Image.fromarray(input_img)\n return pil_img\n frame_count += 1\n')),(0,o.kt)("h3",{id:"2-undistortion-functions"},"2. Undistortion functions"),(0,o.kt)("h4",{id:"21-aria-undistortion-utilities"}
1,"2.1 Aria Undistortion Utilities"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},"def undistort_aria(image_array, provider, sensor_name, focal_length, size):\n device_calib = provider.get_device_calibration()\n src_calib = device_calib.get_camera_calib(sensor_name)\n\n # create output calibration: a linear model of image size 512x512 and focal length 150\n # Invisible pixels are shown as black.\n dst_calib = calibration.get_linear_camera_calibration(\n size, size, focal_length, sensor_name\n )\n\n # distort image\n rectified_array = calibration.distort_by_calibration(\n image_array, dst_calib, src_calib\n )\n return (\n rectified_array,\n dst_calib.get_principal_point(),\n dst_calib.get_focal_lengths(),\n )\n\n")),(0,o.kt)("h4",{id:"22-gopro-undistortion-utilities"},"2.2 GoPro Undistortion Utilities"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'def undistort_exocam(image, intrinsics, distortion_coeffs, dimension=(3840, 2160)):\n DIM = dimension\n dim2 = None\n dim3 = None\n balance = 0.8\n # Load the distortion parameters\n distortion_coeffs = distortion_coeffs\n # Load the camera intrinsic parameters\n intrinsics = intrinsics\n\n dim1 = image.shape[:2][::-1] # dim1 is the dimension of input image to un-distort\n\n # Change the calibration dim dynamically (bouldering cam01 and cam04 are verticall for examples)\n if DIM[0] != dim1[0]:\n DIM = (DIM[1], DIM[0])\n\n assert (\n dim1[0] / dim1[1] == DIM[0] / DIM[1]\n ), "Image to undistort needs to have same aspect ratio as the ones used in calibration"\n if not dim2:\n dim2 = dim1\n if not dim3:\n dim3 = dim1\n scaled_K = (\n intrinsics * dim1[0] / DIM[0]\n ) # The values of K is to scale with image dimension.\n scaled_K[2][2] = 1.0 # Except that K[2][2] is always 1.0\n\n # This is how scaled_K, dim2 and balance are used to determine the final K used to un-distort image. OpenCV document failed to make this clear!\n new_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(\n scaled_K, distortion_coeffs, dim2, np.eye(3), balance=balance\n )\n map1, map2 = cv2.fisheye.initUndistortRectifyMap(\n scaled_K, distortion_coeffs, np.eye(3), new_K, dim3, cv2.CV_16SC2\n )\n undistorted_image = cv2.remap(\n image,\n map1,\n map2,\n interpolation=cv2.INTER_LINEAR,\n borderMode=cv2.BORDER_CONSTANT,\n )\n\n return undistorted_image, new_K\n\n\ndef get_distortion_and_intrinsics(_raw_camera):\n intrinsics = np.array(\n [\n [_raw_camera["intrinsics_0"], 0, _raw_camera["intrinsics_2"]],\n [0, _raw_camera["intrinsics_1"], _raw_camera["intrinsics_3"]],\n [0, 0, 1],\n ]\n )\n distortion_coeffs = np.array(\n [\n _raw_camera["intrinsics_4"],\n _raw_camera["intrinsics_5"],\n _raw_camera["intrinsics_6"],\n _raw_camera["intrinsics_7"],\n ]\n )\n return distortion_coeffs, intrinsics\n')),(0,o.kt)("h3",{id:"3-overlay-2d-annotations-on-undistorted-frames"},"3. Overlay 2D annotations on undistorted frames"),(0,o.kt)("p",null,"First, we locate the download folder of the EgoExo4D dataset and load necessary annotation files."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'release_dir = "/datasets01/egoexo4d/v2/" # replace this with your download folder\nannotation_dir = os.path.join(release_dir, "annotations/") # annotation folder\n\n# load necessary annotation files\negoexo = {\n "takes": os.path.join(release_dir, "takes.json"),\n "captures": os.path.join(release_dir, "captures.json")\n}\n\nfor k, v in egoexo.items():\n egoexo[k] = json.load(open(v))\n\ntakes = egoexo["takes"]\ncaptures = egoexo["captures"]\ntakes_by_uid = {x["take_uid"]: x for x in takes}\n')),(0,o.kt)("p",null,"We then randomly sample one example take of playing violin. We also provide the take uid of some other takes as examples."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'take_uid = "0bc47e29-e086-4726-b874-f89671366f06" # Violin\n#take_uid = "23ff1c48-01ea-4d34-a38b-bc96e767b9b9" #Piano\n#take_uid = "02715c86-e30c-4791-92b7-38b488e51aba" # Bike\n\ntake = [take for take in egoexo["takes"] if take["take_uid"] == take_uid]\ntake = take[0]\n')),(0,o.kt)("p",null,"And load the camera intrinsics and extrinsics of the take. ",(0,o.kt)("strong",{parentName:"p"},"exo_traj_df")," reads in the exocentric cameras calibrations in csv format. The camera calibrations can be used for frame undistortion."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'# Initialize exo cameras from calibration file\ntraj_dir = os.path.join(release_dir, take["root_dir"], "trajectory")\nexo_traj_path = os.path.join(traj_dir, "gopro_calibs.csv")\n\nexo_traj_df = load_csv_to_df(exo_traj_path)\nexo_cam_names = list(exo_traj_df["cam_uid"])\nego_cam_names = [x["cam_id"] for x in take["capture"]["cameras"] if x["is_ego"] and x["cam_id"].startswith("aria")]\nall_cams = ego_cam_names + exo_cam_names\nego_cam_name = ego_cam_names[0]\n')),(0,o.kt)("p",null,"We also configure the VRS data provider."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},"## Configure the VRSDataProvider (interface used to retrieve Trajectory data)\nego_exo_project_path = os.path.join(release_dir, 'takes', take['take_name'])\n\naria_dir = os.path.join(release_dir, take[\"root_dir\"])\naria_path = os.path.join(aria_dir, f\"{ego_cam_name}.vrs\")\nvrs_data_provider = data_provider.create_vrs_data_provider(aria_path)\n")),(0,o.kt)("h4",{id:"31-load-2d-hand-pose-annotation-and-overlay-it-on-undistorted-egocentric-frame"},"3.1 load 2D hand pose annotation and overlay it on undistorted egocentric frame"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'annotation_type = "hand" # annotation_type should be body or hand.\n\n# get hands meta data\nkeypoints_map, skeleton, pose_kpt_color = get_hands_metadata()\n\n# get hand pose annotation folder\negopose_ann_dir = os.path.join(\n annotation_dir, f"ego_pose/train/{annotation_type}/annotation"\n)\n# get the annotation file of the sampled take\nannotation_file_path = os.path.join(egopose_ann_dir, f"{take_uid}.json")\nall_annotations = json.load(open(annotation_file_path))\n# annotation is a dictionary with frame numbers as keys, we then randomly sample one frame.\nframe_idx = random.sample(list(all_annotations.keys()), 1)[0]\nannotation = all_annotations[frame_idx][0]\nframe_idx = int(frame_idx)\nprint(f"annotation at sampled frame {frame_idx} is {annotation.keys()}.")\n')),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"annotation at sampled frame 838 is dict_keys(['annotation3D', 'annotation2D']).\n")),(0,o.kt)("p",null,"Next we read the corresponding at the sampled frame index from exocentric videos and egocentric video. We store it in a dictionary ",(0,o.kt)("strong",{parentName:"p"},"videos")," with camera name as key."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},"base_directory = os.path.join(release_dir, take[\"root_dir\"])\nvideos = {}\nfor cam_name in all_cams:\n if cam_name in exo_cam_names:\n stream_name = '0'\n else:\n stream_name = 'rgb'\n\n local_path = os.path.join(base_directory, take['frame_aligned_videos'][cam_name][stream_name]['relative_path'])\n videos[cam_name] = get_frame(local_path, frame_idx)\n")),(0,o.kt)("p",null,"Now let's visualize the 2D annotations on the undistorted aria frame."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'ego_local_path = os.path.join(base_directory, take[\'frame_aligned_videos\'][ego_cam_name][\'rgb\'][\'relative_path\'])\nego_frame = videos[ego_cam_name]\n\ncam_name = ego_cam_name\nann = annotation["annotation2D"][ego_cam_name]\nimg = ego_frame\nimg = img.rotate(90)\nimage_array = np.asarray(img)\nrectified_array, principal_points, focal_lengths = undistort_aria(image_array, vrs_data_provider, "camera-rgb", 150, 512)\nundistorted_frame = Image.fromarray(rectified_array, "RGB")\n\nviz_img = get_viz(undistorted_frame, keypoints_map, ann, skeleton, pose_kpt_color, annot_type=annotation_type, is_aria=True)\n# uncomment the following lines to
1show the result\n# plt.figure(figsize=(8, 8))\n# plt.imshow(viz_img.rotate(270))\n# plt.axis("off") # Hide the axes ticks\n# plt.title(f"{cam_name}")\n# plt.show()\n')),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"png",src:t(7863).Z,width:"800",height:"800"})),(0,o.kt)("h4",{id:"32-load-2d-body-pose-annotation-and-overlay-it-on-undistorted-exocentric-frames"},"3.2 load 2D body pose annotation and overlay it on undistorted exocentric frames"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'annotation_type = "body"\nkeypoints_map, skeleton, pose_kpt_color = get_body_metadata()\n\n# get body pose annotation folder\negopose_ann_dir = os.path.join(\n annotation_dir, f"ego_pose/train/{annotation_type}/annotation"\n)\n# get the annotation file of the sampled take\nannotation_file_path = os.path.join(egopose_ann_dir, f"{take_uid}.json")\nall_annotations = json.load(open(annotation_file_path))\nannotation = all_annotations[str(frame_idx)][0]\nframe_idx = int(frame_idx)\nprint(f"annotation at sampled frame {frame_idx} is {annotation.keys()}.")\n')),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"annotation at sampled frame 838 is dict_keys(['annotation3D', 'annotation2D']).\n")),(0,o.kt)("p",null,"We load the frames from all exocentric cameras, and overlay the body pose annotation on them."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'results = {}\nfor cam_name in exo_cam_names:\n annot_2d = annotation["annotation2D"]\n if cam_name not in annot_2d:\n continue\n calib_df = exo_traj_df[exo_traj_df.cam_uid == cam_name]\n calib_df = calib_df.iloc[0].to_dict()\n D, I = get_distortion_and_intrinsics(calib_df)\n ann = annot_2d[cam_name]\n img = videos[cam_name]\n undistorted_frame, new_K_latest = undistort_exocam(np.array(img), I, D)\n viz_img = get_viz(\n Image.fromarray(undistorted_frame),\n keypoints_map,\n ann,\n skeleton,\n pose_kpt_color,\n annot_type=annotation_type,\n )\n results[cam_name] = viz_img\n# uncomment the following line to display the results\n# show_results(results)\n')),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"png",src:t(9113).Z,width:"4000",height:"2000"})),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"png",src:t(5886).Z,width:"4000",height:"2000"})),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"png",src:t(32).Z,width:"4000",height:"2000"})),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"png",src:t(9781).Z,width:"4000",height:"2000"})),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"png",src:t(8805).Z,width:"4000",height:"2000"})),(0,o.kt)("h3",{id:"conclusion"},"Conclusion"),(0,o.kt)("p",null,"In this notebook, we introduce how to undistort the frames from aria and gopro cameras. Instead of projecting 3D pose annotations on the camera frames, we load the 2D hand and body annotations, and overlay them on the undistorted frames."),(0,o.kt)("hr",null),(0,o.kt)("p",null,(0,o.kt)("a",{parentName:"p",href:"https://github.com/facebookresearch/Ego4d"},(0,o.kt)("img",{parentName:"a",src:"https://img.shields.io/static/v1.svg?logo=star&label=%E2%AD%90&message=Star%20Our%20Repository&color=yellow",alt:"Star our repository"}))," If you found this tutorial helpful, consider \u2b50-ing our repository."),(0,o.kt)("p",null,(0,o.kt)("a",{parentName:"p",href:"https://github.com/facebookresearch/Ego4d/issues"},(0,o.kt)("img",{parentName:"a",src:"https://img.shields.io/static/v1.svg?logo=star&label=%E2%9D%94&message=Ask%20Questions&color=9cf",alt:"Ask questions"}))," For any questions, typos, or bugs that you found, please raise an issue on GitHub."),(0,o.kt)("hr",null))}p.isMDXComponent=!0},7863:(e,n,t)=>{t.d(n,{Z:()=>a});const a=t.p+"assets/images/undistort-aria01-55fe517da8c19916b22d3dfb34cf5267.png"},9113:(e,n,t)=>{t.d(n,{Z:()=>a});const a=t.p+"assets/images/undistort-gp01-26196ff729eb58fe27795058a1e29499.png"},5886:(e,n,t)=>{t.d(n,{Z:()=>a});const a=t.p+"assets/images/undistort-gp02-9cf04febf67c4ad04ed981010141d9e3.png"},32:(e,n,t)=>{t.d(n,{Z:()=>a});const a=t.p+"assets/images/undistort-gp03-70a6aad3a8fd028a728a6a7f06ed347b.png"},9781:(e,n,t)=>{t.d(n,{Z:()=>a});const a=t.p+"assets/images/undistort-gp04-5f8ff87556d4be49c2f17c783462b4ba.png"},8805:(e,n,t)=>{t.d(n,{Z:()=>a});const a=t.p+"assets/images/undistort-gp06-31921e061199c58624a1cdf9410b66a7.png"}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.