From b25b9e983f5181492eae6182778534522114d69d Mon Sep 17 00:00:00 2001 From: Rik Cabanier Date: Fri, 15 Mar 2024 00:23:01 +0000 Subject: [PATCH 1/2] fix up hands example + add support for tracked sources --- immersive-hands.html | 150 +++++++++++++++++++++++------- js/render/nodes/input-renderer.js | 8 ++ 2 files changed, 122 insertions(+), 36 deletions(-) diff --git a/immersive-hands.html b/immersive-hands.html index 927193c5..c9ec6e74 100644 --- a/immersive-hands.html +++ b/immersive-hands.html @@ -62,6 +62,12 @@ import {vec3} from './js/render/math/gl-matrix.js'; import {Ray} from './js/render/math/ray.js'; + // This library matches XRInputSource profiles to available controller models for us. + import { fetchProfile } from 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/motion-controllers@1.0/dist/motion-controllers.module.js'; + + // The path of the CDN the sample will fetch controller models from. + const DEFAULT_PROFILES_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles'; + // XR globals. let xrButton = null; let xrRefSpace = null; @@ -72,8 +78,10 @@ // Boxes let boxes_left = []; let boxes_right = []; - let boxes = { left: boxes_left, right: boxes_right }; - let indexFingerBoxes = { left: null, right: null }; + let tracked_boxes_left = []; + let tracked_boxes_right = []; + let boxes = { input_left: boxes_left, input_right: boxes_right, tracked_left: tracked_boxes_left, tracked_right: tracked_boxes_right }; + let indexFingerBoxes = { input_left: null, input_right: null, tracked_left: null, tracked_right: null }; const defaultBoxColor = {r: 0.5, g: 0.5, b: 0.5}; const leftBoxColor = {r: 1, g: 0, b: 1}; const rightBoxColor = {r: 0, g: 1, b: 1}; @@ -115,7 +123,7 @@ } boxes_left = []; boxes_right = []; - boxes = { left: boxes_left, right: boxes_right }; + boxes = { input_left: boxes_left, input_right: boxes_right, tracked_left: tracked_boxes_left, tracked_right: tracked_boxes_right }; if (typeof XRHand !== 'undefined') { for (let i = 0; i <= 24; i++) { const r = .6 + Math.random() * .4; @@ -123,16 +131,26 @@ const b = .6 + Math.random() * .4; boxes_left.push(addBox(0, 0, 0, r, g, b)); boxes_right.push(addBox(0, 0, 0, r, g, b)); + tracked_boxes_left.push(addBox(0, 0, 0, r, g, b)); + tracked_boxes_right.push(addBox(0, 0, 0, r, g, b)); } } - if (indexFingerBoxes.left) { + if (indexFingerBoxes.input_left) { scene.removeNode(indexFingerBoxes.left); } - if (indexFingerBoxes.right) { - scene.removeNode(indexFingerBoxes.right); + if (indexFingerBoxes.input_right) { + scene.removeNode(indexFingerBoxes.input_right); + } + if (indexFingerBoxes.tracked_left) { + scene.removeNode(indexFingerBoxes.tracked_left); + } + if (indexFingerBoxes.tracked_right) { + scene.removeNode(indexFingerBoxes.tracked_right); } - indexFingerBoxes.left = addBox(0, 0, 0, leftBoxColor.r, leftBoxColor.g, leftBoxColor.b); - indexFingerBoxes.right = addBox(0, 0, 0, rightBoxColor.r, rightBoxColor.g, rightBoxColor.b); + indexFingerBoxes.input_left = addBox(0, 0, 0, leftBoxColor.r, leftBoxColor.g, leftBoxColor.b); + indexFingerBoxes.input_right = addBox(0, 0, 0, rightBoxColor.r, rightBoxColor.g, rightBoxColor.b); + indexFingerBoxes.tracked_left = addBox(0, 0, 0, leftBoxColor.r, leftBoxColor.g, leftBoxColor.b); + indexFingerBoxes.tracked_right = addBox(0, 0, 0, rightBoxColor.r, rightBoxColor.g, rightBoxColor.b); } // Checks to see if WebXR is available and, if so, queries a list of @@ -178,14 +196,22 @@ // Listen for the sessions 'end' event so we can respond if the user // or UA ends the session for any reason. session.addEventListener('end', onSessionEnded); + session.addEventListener('inputsourceschange', onInputSourcesChange); + session.addEventListener('trackedsourceschange', onInputSourcesChange); session.addEventListener('visibilitychange', e => { // remove hand controller while blurred if(e.session.visibilityState === 'visible-blurred') { - for (const box of boxes['left']) { + for (const box of boxes['input_left']) { + scene.removeNode(box); + } + for (const box of boxes['input_right']) { + scene.removeNode(box); + } + for (const box of boxes['tracked_left']) { scene.removeNode(box); } - for (const box of boxes['right']) { + for (const box of boxes['tracked_right']) { scene.removeNode(box); } } @@ -240,41 +266,92 @@ renderer = null; } + function onInputSourcesChange(event) { + onSourcesChange(event, "input_"); + } + + function onTrackedSourcesChange(event) { + onSourcesChange(event, "tracked_"); + } + + function onSourcesChange(event, type) { + // As input sources are connected if they are tracked-pointer devices + // look up which meshes should be associated with their profile and + // load as the controller model for that hand. + for (let inputSource of event.added) { + if (inputSource.targetRayMode == 'tracked-pointer') { + // Use the fetchProfile method from the motionControllers library + // to find the appropriate glTF mesh path for this controller. + fetchProfile(inputSource, DEFAULT_PROFILES_PATH).then(({profile, assetPath}) => { + // Typically if you wanted to animate the controllers in response + // to device inputs you'd create a new MotionController() instance + // here to handle the animation, but this sample will skip that + // and only display a static mesh for simplicity. + + scene.inputRenderer.setControllerMesh(new Gltf2Node({url: assetPath}), inputSource.handedness, inputSource.profiles[0]); + }); + } + } + } + function updateInputSources(session, frame, refSpace) { + updateSources(session, frame, refSpace, session.inputSources, "input_"); + } + + function updateTrackedSources(session, frame, refSpace) { + if (session.trackedSources) { + updateSources(session, frame, refSpace, session.trackedSources, "tracked_"); + } + } + + function updateSources(session, frame, refSpace, sources, type) { if(session.visibilityState === 'visible-blurred') { return; } - for (let inputSource of session.inputSources) { - let targetRayPose = frame.getPose(inputSource.targetRaySpace, refSpace); - if (targetRayPose) { - if (inputSource.targetRayMode == 'tracked-pointer') { - scene.inputRenderer.addLaserPointer(targetRayPose.transform); + for (let inputSource of sources) { + let hand_type = type + inputSource.handedness; + if (type == "input_") { + let targetRayPose = frame.getPose(inputSource.targetRaySpace, refSpace); + if (targetRayPose) { + if (inputSource.targetRayMode == 'tracked-pointer') { + scene.inputRenderer.addLaserPointer(targetRayPose.transform); + } + + let targetRay = new Ray(targetRayPose.transform); + let cursorDistance = 2.0; + let cursorPos = vec3.fromValues( + targetRay.origin.x, + targetRay.origin.y, + targetRay.origin.z + ); + vec3.add(cursorPos, cursorPos, [ + targetRay.direction.x * cursorDistance, + targetRay.direction.y * cursorDistance, + targetRay.direction.z * cursorDistance, + ]); + + scene.inputRenderer.addCursor(cursorPos); } + } - let targetRay = new Ray(targetRayPose.transform); - let cursorDistance = 2.0; - let cursorPos = vec3.fromValues( - targetRay.origin.x, - targetRay.origin.y, - targetRay.origin.z - ); - vec3.add(cursorPos, cursorPos, [ - targetRay.direction.x * cursorDistance, - targetRay.direction.y * cursorDistance, - targetRay.direction.z * cursorDistance, - ]); - - scene.inputRenderer.addCursor(cursorPos); + if (!inputSource.hand && inputSource.gripSpace) { + let gripPose = frame.getPose(inputSource.gripSpace, refSpace); + if (gripPose) { + scene.inputRenderer.addController(gripPose.transform.matrix, inputSource.handedness, inputSource.profiles[0]); + } else { + scene.inputRenderer.hideController(hand_type); + } } let offset = 0; if (!inputSource.hand) { - continue; - } else { - for (const box of boxes[inputSource.handedness]) { + for (const box of boxes[hand_type]) { scene.removeNode(box); } + scene.removeNode(indexFingerBoxes[hand_type]); + continue; + } else { let pose = frame.getPose(inputSource.targetRaySpace, refSpace); if (pose === undefined) { console.log("no pose"); @@ -288,7 +365,7 @@ console.log("no fillPoses"); continue; } - for (const box of boxes[inputSource.handedness]) { + for (const box of boxes[hand_type]) { scene.addNode(box); let matrix = positions.slice(offset * 16, (offset + 1) * 16); let jointRadius = radii[offset]; @@ -299,7 +376,7 @@ } // Render a special box for each index finger on each hand - const indexFingerBox = indexFingerBoxes[inputSource.handedness]; + const indexFingerBox = indexFingerBoxes[hand_type]; scene.addNode(indexFingerBox); let joint = inputSource.hand.get('index-finger-tip'); let jointPose = frame.getJointPose(joint, xrRefSpace); @@ -343,9 +420,9 @@ const interactionDistance = interactionBox.scale[0]; leftInteractionBox.visible = false; rightInteractionBox.visible = false; - if (Distance(indexFingerBoxes.left, interactionBox) < interactionDistance) { + if (Distance(indexFingerBoxes.input_left, interactionBox) < interactionDistance) { leftInteractionBox.visible = true; - } else if (Distance(indexFingerBoxes.right, interactionBox) < interactionDistance) { + } else if (Distance(indexFingerBoxes.input_right, interactionBox) < interactionDistance) { rightInteractionBox.visible = true; } interactionBox.visible = !(leftInteractionBox.visible || rightInteractionBox.visible); @@ -367,6 +444,7 @@ session.requestAnimationFrame(onXRFrame); updateInputSources(session, frame, xrRefSpace); + updateTrackedSources(session, frame, xrRefSpace); UpdateInteractables(t); // Get the XRDevice pose relative to the Frame of Reference we created diff --git a/js/render/nodes/input-renderer.js b/js/render/nodes/input-renderer.js index 79030874..beb6e69f 100644 --- a/js/render/nodes/input-renderer.js +++ b/js/render/nodes/input-renderer.js @@ -292,6 +292,14 @@ export class InputRenderer extends Node { controllerNode.visible = true; } + hideController(handedness = 'right', profile = '') { + if (!this._controllers) { return; } + let controller = this._controllers[profile + "_" + handedness]; + + if (!controller) { return; } + controllerNode.visible = true; + } + addLaserPointer(rigidTransform) { if (this._blurred) { return; } // Create the laser pointer mesh if needed. From b598bc05a0baa647c9d1b16c3dee3e727296fb53 Mon Sep 17 00:00:00 2001 From: Rik Cabanier Date: Fri, 15 Mar 2024 17:45:42 +0000 Subject: [PATCH 2/2] add comment to clarify that ttracked sources are experimental --- immersive-hands.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/immersive-hands.html b/immersive-hands.html index c9ec6e74..ff6f2a0d 100644 --- a/immersive-hands.html +++ b/immersive-hands.html @@ -197,6 +197,7 @@ // or UA ends the session for any reason. session.addEventListener('end', onSessionEnded); session.addEventListener('inputsourceschange', onInputSourcesChange); + // trackedSources are still experimental. Don't rely on this feature yet. session.addEventListener('trackedsourceschange', onInputSourcesChange); session.addEventListener('visibilitychange', e => { @@ -299,6 +300,7 @@ } function updateTrackedSources(session, frame, refSpace) { + // session.trackedSources are still experimental. Don't rely on this feature yet. if (session.trackedSources) { updateSources(session, frame, refSpace, session.trackedSources, "tracked_"); }