Add support for XR_MSFT_hand_tracking_mesh and XR_ANDROID_hand_mesh

UnityProjects/MRTKDevTemplate/Assets/Scripts/Editor/AndroidXRConfig.cs

@@ -24,16 +24,16 @@ public static void InstallPackages()
                 return;
             }
 
-            Debug.Log("Adding com.unity.xr.androidxr-openxr and com.google.xr.extensions...");
-            request = Client.AddAndRemove(new[] { "com.unity.xr.androidxr-openxr", "https://github.com/android/android-xr-unity-package.git" });
+            Debug.Log("Adding the Unity OpenXR Android XR package...");
+            request = Client.AddAndRemove(new[] { "com.unity.xr.androidxr-openxr" });
             EditorApplication.update += Progress;
         }
 
         private static void Progress()
         {
             if (request.IsCompleted)
             {
-                Debug.Log($"Package install request complete ({request.Status})");
+                Debug.Log($"Package install request complete ({request.Status}).");
                 EditorApplication.update -= Progress;
                 request = null;
             }

UnityProjects/MRTKDevTemplate/Packages/packages-lock.json

@@ -264,12 +264,13 @@
       "url": "https://packages.unity.com"
     },
     "com.unity.xr.hands": {
-      "version": "1.3.0",
+      "version": "1.6.0",
       "depth": 1,
       "source": "registry",
       "dependencies": {
         "com.unity.modules.xr": "1.0.0",
         "com.unity.inputsystem": "1.3.0",
+        "com.unity.mathematics": "1.2.6",
         "com.unity.xr.core-utils": "2.2.0",
         "com.unity.xr.management": "4.0.1"
       },
@@ -389,7 +390,7 @@
         "com.unity.inputsystem": "1.6.1",
         "com.unity.xr.arfoundation": "5.0.5",
         "com.unity.xr.core-utils": "2.1.0",
-        "com.unity.xr.hands": "1.3.0",
+        "com.unity.xr.hands": "1.6.0",
         "com.unity.xr.interaction.toolkit": "3.0.4",
         "org.mixedrealitytoolkit.core": "4.0.0"
       }

org.mixedrealitytoolkit.input/CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 * Added toggle for frame rate independent smoothing in camera simulation. [PR #1011](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1011)
 * Added implementation for the synthesized TriggerButton, accounting for animation smoothing. [PR #1043](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1043)
 * Added a "squeeze" alias for the grip states, to account for broader input action mapping support. [PR #1043](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1043)
+* Added support for XR_MSFT_hand_tracking_mesh and XR_ANDROID_hand_mesh on compatible runtimes. [PR #993](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/993)
 
 ### Fixed
 

org.mixedrealitytoolkit.input/Controllers/HandModel.cs

@@ -57,7 +57,7 @@ public Transform ModelPrefab
         /// the hand model prefab when implementing <see cref="ISelectInputVisualizer"/>.
         /// </summary>
         public XRInputButtonReader SelectInput => selectInput;
-        
+
         #endregion Associated hand select values
 
         /// <summary>
@@ -73,9 +73,13 @@ protected virtual void Start()
                 Debug.Assert(selectInput != null, $"The Select Input reader for {name} is not set and will not be used with the instantiated hand model.");
 
                 // Set the select input reader for the model if it implements ISelectInputVisualizer
-                if (selectInput != null && model != null && model.TryGetComponent(out ISelectInputVisualizer selectInputVisualizer))
+                if (selectInput != null && model != null)
                 {
-                    selectInputVisualizer.SelectInput = selectInput;
+                    ISelectInputVisualizer[] selectInputVisualizers = model.GetComponentsInChildren<ISelectInputVisualizer>();
+                    foreach (ISelectInputVisualizer selectInputVisualizer in selectInputVisualizers)
+                    {
+                        selectInputVisualizer.SelectInput = selectInput;
+                    }
                 }
             }
         }

org.mixedrealitytoolkit.input/Subsystems/Hands/UnityHandsSubsystem.cs

@@ -263,8 +263,7 @@ private XRHand GetTrackedHand()
                         }
                     }
 
-                    XRHand hand = HandNode == XRNode.LeftHand ? xrHandSubsystem.leftHand : xrHandSubsystem.rightHand;
-                    return hand;
+                    return HandNode == XRNode.LeftHand ? xrHandSubsystem.leftHand : xrHandSubsystem.rightHand;
                 }
             }
 

org.mixedrealitytoolkit.input/Tracking/HandPoseDriver.cs

@@ -33,21 +33,23 @@ public class HandPoseDriver : TrackedPoseDriver
         private bool m_firstUpdate = true;
         private InputAction m_boundTrackingAction = null;
         private InputTrackingState m_trackingState = InputTrackingState.None;
+        private const InputTrackingState m_polyfillTrackingState = InputTrackingState.Position | InputTrackingState.Rotation;
 
         /// <summary>
         /// Expose the tracking state for the hand pose driver, to allow <see cref="TrackedPoseDriverExtensions"/> to query it.
         /// </summary>
         /// <remarks>
         /// Avoid exposing this publicly as this <see cref="HandPoseDriver"/> is a workaround solution to support hand tracking on devices without interaction profiles.
         /// </remarks>
-        internal InputTrackingState CachedTrackingState => m_trackingState;
+        internal InputTrackingState CachedTrackingState => IsPolyfillDevicePose ? m_polyfillTrackingState : m_trackingState;
 
         /// <summary>
         /// Get if the last pose set was from a polyfill device pose. That is, if the last pose originated from the <see cref="XRSubsystemHelpers.HandsAggregator "/>.
         /// </summary>
         internal bool IsPolyfillDevicePose { get; private set; }
 
         #region Serialized Fields
+
         [Header("Hand Pose Driver Settings")]
 
         [SerializeField, Tooltip("The XRNode associated with this Hand Controller. Expected to be XRNode.LeftHand or XRNode.RightHand.")]
@@ -58,9 +60,11 @@ public class HandPoseDriver : TrackedPoseDriver
         /// </summary>
         /// <remarks>Expected to be XRNode.LeftHand or XRNode.RightHand.</remarks>
         public XRNode HandNode => handNode;
+
         #endregion Serialized Fields
 
         #region TrackedPoseDriver Overrides
+
         /// <inheritdoc />
         protected override void PerformUpdate()
         {
@@ -94,7 +98,6 @@ protected override void PerformUpdate()
             if ((missingPositionController || missingRotationController || IsTrackingNone()) &&
                 TryGetPolyfillDevicePose(out Pose devicePose))
             {
-                m_trackingState = InputTrackingState.Position | InputTrackingState.Rotation;
                 IsPolyfillDevicePose = true;
                 ForceSetLocalTransform(devicePose.position, devicePose.rotation);
             }
@@ -103,9 +106,11 @@ protected override void PerformUpdate()
                 IsPolyfillDevicePose = false;
             }
         }
+
         #endregion TrackedPoseDriver Overrides
 
         #region Private Functions
+
         /// <summary>
         /// Check the tracking state here to account for a bound but untracked interaction profile.
         /// This could show up on runtimes where a controller is disconnected, hand tracking spins up,
@@ -278,6 +283,7 @@ private void OnTrackingStateInputCanceled(InputAction.CallbackContext context)
                 m_trackingState = InputTrackingState.None;
             }
         }
+
         #endregion Private Functions
     }
 }

org.mixedrealitytoolkit.input/Visualizers/HandMeshVisualizer.cs

@@ -0,0 +1,232 @@
+// Copyright (c) Mixed Reality Toolkit Contributors
+// Licensed under the BSD 3-Clause
+
+using System;
+using System.Collections.Generic;
+using UnityEngine;
+using UnityEngine.XR;
+using UnityEngine.XR.Interaction.Toolkit;
+using UnityEngine.XR.Interaction.Toolkit.Inputs.Readers;
+
+namespace MixedReality.Toolkit.Input
+{
+    public abstract class HandMeshVisualizer : MonoBehaviour, ISelectInputVisualizer
+    {
+        [SerializeField]
+        [Tooltip("The XRNode on which this hand is located.")]
+        private XRNode handNode = XRNode.LeftHand;
+
+        /// <summary> The XRNode on which this hand is located. </summary>
+        public XRNode HandNode { get => handNode; set => handNode = value; }
+
+        [SerializeField]
+        [Tooltip("When true, this visualizer will render rigged hands even on XR devices " +
+                 "with transparent displays or with passthrough enabled. When false, the rigged hands will only render " +
+                 "on devices with opaque displays. This behavior uses XRDisplaySubsystem.displayOpaque.")]
+        private bool showHandsOnTransparentDisplays;
+
+        /// <summary>
+        /// When true, this visualizer will render rigged hands even on XR devices with transparent displays or with passthrough enabled.
+        /// When false, the rigged hands will only render on devices with opaque displays.
+        /// </summary>
+        /// <remarks>
+        /// This behavior uses <see cref="XRDisplaySubsystem.displayOpaque"/>.
+        /// Usually, it's recommended not to show hand visualization on transparent displays as it can
+        /// distract from the user's real hands, and cause a "double image" effect that can be disconcerting.
+        /// </remarks>
+        public bool ShowHandsOnTransparentDisplays
+        {
+            get => showHandsOnTransparentDisplays;
+            set => showHandsOnTransparentDisplays = value;
+        }
+
+        [SerializeField]
+        [Tooltip("Name of the shader property used to drive pinch-amount-based visual effects. " +
+                 "Generally, maps to something like a glow or an outline color!")]
+        private string pinchAmountMaterialProperty = "_PinchAmount";
+
+        [SerializeField]
+        [Tooltip("The input reader used when pinch selecting an interactable.")]
+        private XRInputButtonReader selectInput = new XRInputButtonReader("Select");
+
+        #region ISelectInputVisualizer implementation
+
+        /// <summary>
+        /// Input reader used when pinch selecting an interactable.
+        /// </summary>
+        public XRInputButtonReader SelectInput
+        {
+            get => selectInput;
+            set => SetInputProperty(ref selectInput, value);
+        }
+
+        #endregion ISelectInputVisualizer implementation
+
+        // The property block used to modify the pinch amount property on the material
+        private MaterialPropertyBlock propertyBlock = null;
+
+        // Scratch list for checking for the presence of display subsystems.
+        private readonly List<XRDisplaySubsystem> displaySubsystems = new List<XRDisplaySubsystem>();
+
+        // The XRController that is used to determine the pinch strength (i.e., select value!)
+        [Obsolete("This field has been deprecated in version 4.0.0 and will be removed in a future version. Use the SelectInput property instead.")]
+        private XRBaseController controller;
+
+        /// <summary>
+        /// The list of button input readers used by this interactor. This interactor will automatically enable or disable direct actions
+        /// if that mode is used during <see cref="OnEnable"/> and <see cref="OnDisable"/>.
+        /// </summary>
+        /// <seealso cref="XRInputButtonReader.EnableDirectActionIfModeUsed"/>
+        /// <seealso cref="XRInputButtonReader.DisableDirectActionIfModeUsed"/>
+        private readonly List<XRInputButtonReader> buttonReaders = new List<XRInputButtonReader>();
+
+        /// <summary>
+        /// Whether this visualizer currently has a loaded and visible hand mesh or not.
+        /// </summary>
+        protected internal bool IsRendering => HandRenderer != null && HandRenderer.enabled;
+
+        /// <summary>
+        /// The renderer for this visualizer, to use to visualize the pinch amount.
+        /// </summary>
+        protected abstract Renderer HandRenderer { get; }
+
+        /// <summary>
+        /// A Unity event function that is called when an enabled script instance is being loaded.
+        /// </summary>
+        protected virtual void Awake()
+        {
+            propertyBlock = new MaterialPropertyBlock();
+            buttonReaders.Add(selectInput);
+        }
+
+        /// <summary>
+        /// A Unity event function that is called when the script component has been enabled.
+        /// </summary>
+        protected virtual void OnEnable()
+        {
+            buttonReaders.ForEach(reader => reader?.EnableDirectActionIfModeUsed());
+
+            // Ensure hand is not visible until we can update position first time.
+            HandRenderer.enabled = false;
+
+            Debug.Assert(handNode == XRNode.LeftHand || handNode == XRNode.RightHand,
+                $"HandVisualizer has an invalid XRNode ({handNode})!");
+        }
+
+        /// <summary>
+        /// A Unity event function that is called when the script component has been disabled.
+        /// </summary>
+        protected virtual void OnDisable()
+        {
+            buttonReaders.ForEach(reader => reader?.DisableDirectActionIfModeUsed());
+
+            // Disable the rigged hand renderer when this component is disabled
+            HandRenderer.enabled = false;
+        }
+
+        /// <summary>
+        /// Helper method for setting an input property.
+        /// </summary>
+        /// <param name="property">The <see langword="ref"/> to the field.</param>
+        /// <param name="value">The new value being set.</param>
+        /// <remarks>
+        /// If the application is playing, this method will also enable or disable directly embedded input actions
+        /// serialized by the input if that mode is used. It will also add or remove the input from the list of button inputs
+        /// to automatically manage enabling and disabling direct actions with this behavior.
+        /// </remarks>
+        /// <seealso cref="buttonReaders"/>
+        protected void SetInputProperty(ref XRInputButtonReader property, XRInputButtonReader value)
+        {
+            if (value == null)
+            {
+                Debug.LogError("Setting XRInputButtonReader property to null is disallowed and has therefore been ignored.");
+                return;
+            }
+
+            if (Application.isPlaying && property != null)
+            {
+                buttonReaders?.Remove(property);
+                property.DisableDirectActionIfModeUsed();
+            }
+
+            property = value;
+
+            if (Application.isPlaying)
+            {
+                buttonReaders?.Add(property);
+                if (isActiveAndEnabled)
+                {
+                    property.EnableDirectActionIfModeUsed();
+                }
+            }
+        }
+
+        protected virtual bool ShouldRenderHand()
+        {
+            if (displaySubsystems.Count == 0)
+            {
+                SubsystemManager.GetSubsystems(displaySubsystems);
+            }
+
+            // Are we running on an XR display and it happens to be transparent?
+            // Probably shouldn't be showing rigged hands! (Users can
+            // specify showHandsOnTransparentDisplays if they disagree.)
+            if (displaySubsystems.Count > 0 &&
+                displaySubsystems[0].running &&
+                !displaySubsystems[0].displayOpaque &&
+                !showHandsOnTransparentDisplays)
+            {
+                return false;
+            }
+
+            // All checks out!
+            return true;
+        }
+
+        protected virtual void UpdateHandMaterial()
+        {
+            if (HandRenderer == null)
+            {
+                return;
+            }
+
+            // Update the hand material
+            float pinchAmount = TryGetSelectionValue(out float selectionValue) ? Mathf.Pow(selectionValue, 2.0f) : 0;
+            HandRenderer.GetPropertyBlock(propertyBlock);
+            propertyBlock.SetFloat(pinchAmountMaterialProperty, pinchAmount);
+            HandRenderer.SetPropertyBlock(propertyBlock);
+        }
+
+        /// <summary>
+        /// Try to obtain the tracked devices selection value from the provided input reader.
+        /// </summary>
+        /// <remarks>
+        /// For backwards compatibility, this method will also attempt to get the selection amount from a
+        /// legacy XRI controller if the input reader is not set.
+        /// </remarks>
+        private bool TryGetSelectionValue(out float value)
+        {
+            if (selectInput != null && selectInput.TryReadValue(out value))
+            {
+                return true;
+            }
+
+            bool success = false;
+            value = 0.0f;
+
+#pragma warning disable CS0618 // XRBaseController is obsolete
+            if (controller == null)
+            {
+                controller = GetComponentInParent<XRBaseController>();
+            }
+            if (controller != null)
+            {
+                value = controller.selectInteractionState.value;
+                success = true;
+            }
+#pragma warning restore CS0618 // XRBaseController is obsolete
+
+            return success;
+        }
+    }
+}