1"use strict";(self.webpackChunkconjurekit_docs=self.webpackChunkconjurekit_docs||[]).push([[17262],{51589:(e,n,a)=>{a.r(n),a.d(n,{assets:()=>s,contentTitle:()=>o,default:()=>c,frontMatter:()=>i,metadata:()=>l,toc:()=>d});var t=a(87462),r=(a(67294),a(3905));a(61839);const i={sidebar_position:3},o="Handtracker in shared AR experience",l={unversionedId:"x-archived/handtracker_shared_ar_experience",id:"x-archived/handtracker_shared_ar_experience",title:"Handtracker in shared AR experience",description:"This section describes how to use Ur module and add simple hand interaction to the shared AR experience project.",source:"@site/docs/unity/x-archived/handtracker_shared_ar_experience.mdx",sourceDirName:"x-archived",slug:"/x-archived/handtracker_shared_ar_experience",permalink:"/unity/x-archived/handtracker_shared_ar_experience",draft:!1,tags:[],version:"current",sidebarPosition:3,frontMatter:{sidebar_position:3},sidebar:"tutorialSidebar",previous:{title:"Auki::Ceres::Solver",permalink:"/unity/com.aukilabs.unity.ceres/0.0.4/Classes/class_auki_1_1_ceres_1_1_solver"},next:{title:"Create a simple shared AR experience v0.5",permalink:"/unity/x-archived/create_shared_ar_experience"}},s={},d=[{value:"Before you begin",id:"before-you-begin",level:2},{value:"Initializing Handtracker",id:"initializing-handtracker",level:2},{value:"Hand Interaction",id:"hand-interaction",level:2},{value:"Hand Landmarks Visualization",id:"hand-landmarks-visualization",level:2},{value:"Calibration",id:"calibration",level:2},{value:"Occlusion Culling",id:"occlusion-culling",level:2},{value:"Complete code",id:"complete-code",level:2}],p={toc:d};function c(e){let{components:n,...a}=e;return(0,r.kt)("wrapper",(0,t.Z)({},p,a,{components:n,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"handtracker-in-shared-ar-experience"},"Handtracker in shared AR experience"),(0,r.kt)("p",null,"This section describes how to use Ur module and add simple hand interaction to the shared AR experience project."),(0,r.kt)("iframe",{style:{aspectRatio:"16/9",width:"100%",height:"100%"},src:"https://www.youtube.com/embed/z4jWUs9ikUA",title:"YouTube video player",frameborder:"0",allow:"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; fullscreen",allowfullscreen:!0}),(0,r.kt)("h2",{id:"before-you-begin"},"Before you begin"),(0,r.kt)("p",null,"Before getting started, complete the following steps:"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},"Check the ",(0,r.kt)("a",{parentName:"li",href:"/unity/prerequisites/"},"Prerequisites"),"."),(0,r.kt)("li",{parentName:"ol"},"Set up your development environment by following the steps in the ",(0,r.kt)("a",{parentName:"li",href:"/unity/quickstart"},"Quickstart"),"."),(0,r.kt)("li",{parentName:"ol"},"Follow the ",(0,r.kt)("a",{parentName:"li",href:"./unity/create_shared_ar_experience"},"Create a simple shared AR experience")," tutorial or clone the project from the repository.")),(0,r.kt)("h2",{id:"initializing-handtracker"},"Initializing Handtracker"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},"Install the Ur package."),(0,r.kt)("li",{parentName:"ol"},"Import ",(0,r.kt)("inlineCode",{parentName:"li"},"Ur")," in ",(0,r.kt)("inlineCode",{parentName:"li"},"ConjureKitManager"),".")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"using Auki.Ur;\n")),(0,r.kt)("ol",{start:3},(0,r.kt)("li",{parentName:"ol"},"Import ",(0,r.kt)("inlineCode",{parentName:"li"},"ARFoundation")," to be able to use the ",(0,r.kt)("inlineCode",{parentName:"li"},"ARRaycastManager"))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"using UnityEngine.XR.ARFoundation;\n")),(0,r.kt)("ol",{start:4},(0,r.kt)("li",{parentName:"ol"},"Create a private ",(0,r.kt)("inlineCode",{parentName:"li"},"HandTracker")," variable.")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"private HandTracker _handTracker;\n")),(0,r.kt)("ol",{start:5},(0,r.kt)("li",{parentName:"ol"},"Create serializable ",(0,r.kt)("inlineCode",{parentName:"li"},"ARSession")," and ",(0,r.kt)("inlineCode",{parentName:"li"},"ARRaycastManager")," variables.")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"[SerializeField] private ARSession arSession;\n[SerializeField] private ARRaycastManager arRaycastManager;\n")),(0,r.kt)("ol",{start:6},(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Attach the ",(0,r.kt)("inlineCode",{parentName:"p"},"ARRaycastManager")," component to ",(0,r.kt)("strong",{parentName:"p"},"AR Session Origin")," GameObject.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Drag the ",(0,r.kt)("inlineCode",{parentName:"p"},"ARSession")," and the ",(0,r.kt)("inlineCode",{parentName:"p"},"ARRaycastManager")," components to coressponding fields on the ",(0,r.kt)("inlineCode",{parentName:"p"},"ConjureKitManager")," GameObject.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Get ",(0,r.kt)("inlineCode",{parentName:"p"},"HandTracker")," instance and initialize the AR system in the ",(0,r.kt)("inlineCode",{parentName:"p"},"ConjureKitManager"),"'s ",(0,r.kt)("inlineCode",{parentName:"p"},"Start")," function."))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"_handTracker = HandTracker.GetInstance();\n_handTracker.SetARSystem(arSession, arCamera, arRaycastManager);\n")),(0,r.kt)("ol",{start:9},(0,r.kt)("li",{parentName:"ol"},"Start the ",(0,r.kt)("inlineCode",{parentName:"li"},"HandTracker")," by calling")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"_handTracker.Start();\n")),(0,r.kt)("ol",{start:10},(0,r.kt)("li",{parentName:"ol"},"Call the ",(0,r.kt)("inlineCode",{parentName:"li"},"_handTracker.Update")," every frame to continuously track the hand while moving.")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"}
1,"private void Update()\n{\n _handTracker.Update();\n}\n")),(0,r.kt)("h2",{id:"hand-interaction"},"Hand Interaction"),(0,r.kt)("p",null,"Now we want to position a sphere with a collider on our hand's index fingertip so it can interact with the cube."),(0,r.kt)("ol",{start:11},(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Create a 3d sphere, rename it to ",(0,r.kt)("em",{parentName:"p"},"FingertipLandmark"),", and scale it down to 0.3.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"On the collider component, tick the ",(0,r.kt)("inlineCode",{parentName:"p"},"isTrigger")," checkbox.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Create a new material, change its color to something more noticeable, and drag it to the sphere mesh renderer.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Add a new tag in ",(0,r.kt)("strong",{parentName:"p"},"Project Settings -> Tags and Layers")," named ",(0,r.kt)("inlineCode",{parentName:"p"},"hand")," or any other name you choose and add this tag to the ",(0,r.kt)("em",{parentName:"p"},"FingertipLandmark")," we just created")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Create a ",(0,r.kt)("inlineCode",{parentName:"p"},"Renderer")," variable for the ",(0,r.kt)("em",{parentName:"p"},"FingertipLandmark")))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"[SerializeField] private Renderer Fingertip Landmark;\n")),(0,r.kt)("ol",{start:16},(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Populate it with the sphere we just created.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"And in Start function, Set the fingertip landmark as a child of our camera transform."))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"fingertipLandmark.transform.SetParent(arCamera.transform);\n")),(0,r.kt)("ol",{start:18},(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"To get triggers from other colliders, the cube should have a ",(0,r.kt)("inlineCode",{parentName:"p"},"Rigidbody")," component.\nAdd it and tick the ",(0,r.kt)("inlineCode",{parentName:"p"},"Is Kinematic")," checkbox to make sure the cube doesn't fall.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Create a new C# script that will handle trigger events on the cube.\nIf the cube is triggered with an object tagged with ",(0,r.kt)("inlineCode",{parentName:"p"},"hand")," its color will change to a random color.\nIf the trigger exits the cube, it will return to white."))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},'public class TouchableByHand : MonoBehaviour\n{\n private void OnTriggerEnter(Collider other)\n {\n if (other.tag == "hand")\n {\n gameObject.GetComponent<Renderer>().material.color = Random.ColorHSV();\n }\n }\n\n private void OnTriggerExit(Collider other)\n {\n if (other.tag == "hand")\n {\n gameObject.GetComponent<Renderer>().material.color = Color.white;\n }\n }\n}\n')),(0,r.kt)("ol",{start:20},(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Add this script to the cube prefab.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},'To get our landmark positions in real-time, we will use the Ur callback - OnUpdate that is invoked when a new hand pose is received.\nThis callback will pass 4 data types"'))),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"landmarks"),": are landmark positions relative to the hand position"),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"translations"),": are the positions of each hand in camera space"),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"isRightHand"),": indicates whether the right or left hand was detected"),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"score"),": is the hand confidence score. If 0, no hand was detected"),(0,r.kt)("p",null,"The landmark and translation arrays contain consecutive floats representing the x, y & z-components of a 3D vector."),(0,r.kt)("p",null,"Once we get the landmarks and translations, we can place the fingertip landmark on landmark 8, the tip of the index finger (",(0,r.kt)("a",{parentName:"p",href:"../#landmark"},"see diagram"),").\nIf the hand tracker detects a hand, we should see the fingertip landmark. If not, meaning our hand is not in camera sight, we can disable the fingertip landmark renderer."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"_handTracker.OnUpdate += (landmarks, translations, isRightHand, score) =>\n{\n if (score[0] > 0)\n {\n var handPosition = new Vector3(\n translations[0],\n translations[1],\n translations[2]);\n\n var pointerLandmarkIndex = 8 * 3; // Index fingertip\n var pointerLandMarkPosition = new Vector3(\n landmarks[pointerLandmarkIndex + 0],\n landmarks[pointerLandmarkIndex + 1],\n landmarks[pointerLandmarkIndex + 2]);\n\n fingertipLandmark.enabled = true;\n fingertipLandmark.transform.localPosition = handPosition + pointerLandMarkPosition;\n }\n else\n {\n fingertipLandmark.enabled = false;\n }\n};\n")),(0,r.kt)("h2",{id:"hand-landmarks-visualization"},"Hand Landmarks Visualization"),(0,r.kt)("p",null,"Ur module allows us to visualize the hand landmarks with two simple methods."),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},"Start by creating a private boolean. ")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"landmarksVisualizeBool = true;\n")),(0,r.kt)("ol",{start:2},(0,r.kt)("li",{parentName:"ol"},"Create a toggle method that uses the hand tracker methods ",(0,r.kt)("inlineCode",{parentName:"li"},"ShowHandMesh")," and ",(0,r.kt)("inlineCode",{parentName:"li"},"HideHandMesh"),".")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"public void ToggleHandLandmarks()\n{\n landmarksVisualizeBool = !landmarksVisualizeBool;\n\n if (landmarksVisualizeBool)\n {\n _handTracker.ShowHandMesh();\n }\n else\n {\n _handTracker.HideHandMesh();\n }\n}\n")),(0,r.kt)("p",null,"Now it can be toggled using a UI toggle or any other method you choose."),(0,r.kt)("h2",{id:"calibration"},"Calibration"),(0,r.kt)("p",null,"The hand tracker can sometimes project the hand landmarks into the wrong depth range. A calibration process is required to fix this error.\nIt's a short process where we ask the user to place their hand on a flat surface, direct the camera on it and bring it closer and further for a few seconds while a plane ray-casting is performed to determine the distance from the camera to the plane on which the user has laid the hand.\nIn the demo scene from the sample provided with the Ur package, you can find a Calibrate Hand Tracker Button and code that can easily be used in any project. "),(0,r.kt)("p",null,"You will also need to add the ",(0,r.kt)("inlineCode",{parentName:"p"},"ARPlaneManager")," component on the ",(0,r.kt)("inlineCode",{parentName:"p"},"ARSessionOrigion")," GameObject."),(0,r.kt)("h2",{id:"occlusion-culling"},"Occlusion Culling"),(0,r.kt)("p",null,"In order to layer our 3D objects and real-world elements correctly, we use Occlusion culling."),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Add the ",(0,r.kt)("inlineCode",{parentName:"p"},"AROcclusionManager")," component to ",(0,r.kt)("inlineCode",{parentName:"p"},"AR Camera")," GameObject and adjust its properties (Fastest on all).")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"To use the ",(0,r.kt)("inlineCode",{parentName:"p"},"AROcclusionManager")," in the code import ",(0,r.kt)("inlineCode",{parentName:"p"},"ARSubsystems"),"."))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"using UnityEngine.XR.ARSubsystems;\n")),(0,r.kt)("ol",{start:3},(0,r.kt)("li",{parentName:"ol"},"Declare an ",(0,r.kt)("inlineCode",{parentName:"li"},"AROcclusionManager")," serializable variable and a boolean for toggling it on and off.")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"[SerializeField] private AROcclusionManager arOcclusionManager;\nprivate bool occlusionBool = true;\n")),(0,r.kt)("ol",{start:4},(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Drag the ",(0,r.kt)("inlineCode",{parentName:"p"},"ARCamera")," GameObject to the ",(0,r.kt)("inlineCode",{parentName:"p"},"AROcclusionManager")," field in the ",(0,r.kt)("inlineCode",{parentName:"p"},"ConureKitManager"),".")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Create a toggle method to toggle between the different ",(0,r.kt)("inlineCode",{parentName:"p"},"AROcclusionManager")," modes."))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},"public void ToggleOcclusion()\n{\n occlusionBool = !occlusionBool;\n\n arOcclusionManager.requestedHumanDepthMode = occlusionBool ? HumanSegmentationDepthMode.Fastest : HumanSegmentationDepthMode.Disabled;\n arOcclusionManager.requestedHumanStencilMode = occlusionBool ? HumanSegmentationStencilMode.Fastest : HumanSegmentationStencilMode.Disabled;\n arOcclusionManager.requestedEnvironmentDepthMode = occlusionBool ? EnvironmentDepthMode.Fastest : EnvironmentDepthMode.Disabled;\n}\n")),(0,r.kt)("p",null,"Use a UI toggle to switch between modes, but again you can use any method you choose."),(0,r.kt)("h2",{id:"complete-code"},"Complete code"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-csharp"},'using UnityEngine;\nusing Auki.ConjureKit;\nusing UnityEngine.UI;\nusing Auki.ConjureKit.Manna;\nusing Auki.Ur;\nusing UnityEngine.XR.ARFoundation;\nusing UnityEngine.XR.ARSubsystems;\n\npublic class ConjureKitManager : MonoBehaviour\n{\n [SerializeField] private Camera arCamera;\n [SerializeField] private ARSession arSession;\n [SerializeField] private ARRaycastManager arRaycastManager;\n\n [SerializeField] private Text sessionState;\n [SerializeField] private Text sessionID;\n \n [SerializeField] private GameObject cube;\n [SerializeField] private Button spawnButton;\n\n [SerializeField] Button qrCodeButton;\n private bool qrCodeBool;\n \n private IConjureKit _conjureKit;\n private Manna _manna;\n \n [SerializeField] private Renderer fingertipLandmark;\n private HandTracker _handTracker;\n private bool landmarksVisualizeBool = true;\n \n [SerializeField] private AROcclusionManager arOcclusionManager;\n private bool occlusionBool = true;\n\n void Start()\n {\n _conjureKit = new ConjureKit(\n ConjureKitConfiguration.Get(), arCamera.transform,\n "YOUR_APP_KEY",\n "YOUR_APP_SECRET");\n\n _manna = new Manna(_conjureKit);\n \n _conjureKit.OnStateChanged += state =>\n {\n if (state == State.JoinedSession)\n {\n Debug.Log("State.JoinedSession " + Time.realtimeSinceStartup);\n }\n\n if (state == State.Calibrated)\n {\n Debug.Log("State.Calibrated " + Time.realtimeSinceStartup);\n }\n\n sessionState.text = state.ToString();\n ToggleControlsState(state == State.Calibrated);\n };\n\n _conjureKit.OnJoined += session =>\n {\n Debug.Log("OnJoined " + Time.realtimeSinceStartup);\n sessionID.text = session.Id.ToString();\n };\n\n _conjureKit.OnLeft += () =>\n {\n sessionID.text = "";\n };\n\n _conjureKit.OnEntityAdded += CreateCube;\n _conjureKit.Connect();\n \n _handTracker = HandTracker.GetInstance();\n _handTracker.SetARSystem(arSession, arCamera, arRaycastManager);\n \n _handTracker.OnUpdate += (landmarks, translations, isRightHand, score) =>\n {\n if (score[0] > 0)\n {\n var handPosition = new Vector3(\n translations[0],\n translations[1],\n translations[2]);\n\n var pointerLandmarkIndex = 8 * 3; // Index fingertip\n var pointerLandMarkPosition = new Vector3(\n landmarks[pointerLandmarkIndex + 0],\n landmarks[pointerLandmarkIndex + 1],\n landmarks[pointerLandmarkIndex + 2]);\n\n fingertipLandmark.enabled = true;\n fingertipLandmark.transform.localPosition = handPosition + pointerLandMarkPosition;\n }\n else\n {\n fingertipLandmark.enabled = false;\n }\n };\n \n _handTracker.Start();\n _handTracker.ShowHandMesh();\n }
1\n \n private void Update()\n {\n _handTracker.Update();\n }\n\n private void ToggleControlsState(bool interactable)\n {\n if (spawnButton) spawnButton.interactable = interactable;\n if (qrCodeButton) qrCodeButton.interactable = interactable;\n }\n \n public void ToggleLighthouse()\n {\n qrCodeBool = !qrCodeBool;\n _manna.SetLighthouseVisible(qrCodeBool);\n }\n \n public void ToggleHandLandmarks()\n {\n landmarksVisualizeBool = !landmarksVisualizeBool;\n\n if (landmarksVisualizeBool)\n {\n _handTracker.ShowHandMesh();\n }\n else\n {\n _handTracker.HideHandMesh();\n }\n }\n \n public void ToggleOcclusion()\n {\n occlusionBool = !occlusionBool;\n\n arOcclusionManager.requestedHumanDepthMode = occlusionBool ? HumanSegmentationDepthMode.Fastest : HumanSegmentationDepthMode.Disabled;\n arOcclusionManager.requestedHumanStencilMode = occlusionBool ? HumanSegmentationStencilMode.Fastest : HumanSegmentationStencilMode.Disabled;\n arOcclusionManager.requestedEnvironmentDepthMode = occlusionBool ? EnvironmentDepthMode.Fastest : EnvironmentDepthMode.Disabled;\n }\n\n public void CreateCubeEntity()\n {\n if (_conjureKit.GetState() != State.Calibrated)\n return;\n\n Vector3 position = arCamera.transform.position + arCamera.transform.forward * 0.5f;\n Quaternion rotation = Quaternion.Euler(0, arCamera.transform.eulerAngles.y, 0);\n\n Pose entityPos = new Pose(position, rotation);\n\n _conjureKit.AddEntity(\n entityPos,\n onComplete: entity => CreateCube(entity),\n onError: error => Debug.Log(error));\n }\n\n private void CreateCube(Entity entity)\n {\n if (entity.Flag == EntityFlag.EntityFlagParticipantEntity) return;\n\n Instantiate(cube, entity.Pose.position, entity.Pose.rotation);\n }\n}\n')))}c.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.