I am trying to set the hand into one of a series of poses. I have created a script that captures a frame, serializes the left hand which is then stored as an xml for loading. I currently have the system setting the hand to the correct pose by calculating the offset between the palm position in the live data and that of the stored pose. The problem I am having is getting the hand to rotate.
The 'hand' parameter is the current hand from the live data and the 'pose' parameter is the hand that has been loaded from xml.
This is the method I have created:
public static Hand SetHandInPose(Hand hand, Hand pose)
{
if(hand == null)
{
Debug.Log("Hand is null, so returning that");
return hand;
}
if(pose == null)
{
Debug.Log("The loaded pose is null, so let's just return the original hand");
return hand;
}
Hand h = pose;
Quaternion handRotOffset = pose.Rotation.ToQuaternion() * Quaternion.Inverse(hand.Rotation.ToQuaternion());
//Debug.Log("The rotational offset is: " + handRotOffset.eulerAngles);
Vector offset = hand.PalmPosition - pose.PalmPosition;
h.Rotation = hand.Rotation;//(h.Rotation.ToQuaternion() * handRotOffset).ToLeapQuaternion();
h.PalmPosition += (offset.ToVector3()).ToVector();
h.WristPosition += (offset.ToVector3()).ToVector();
for(int f = 0; f< h.Fingers.Count; f++)
{
for(int i = 0; i < h.Fingers[f].bones.Length; i++)
{
//offset = hand.Fingers[f].bones[i].Center - pose.Fingers[f].bones[i].Center;
//if (h.Fingers[f].bones[i].Type == Bone.BoneType.TYPE_METACARPAL) continue;
h.Fingers[f].bones[i].Center += (offset.ToVector3()).ToVector();
h.Fingers[f].bones[i].NextJoint += (offset.ToVector3()).ToVector();
h.Fingers[f].bones[i].PrevJoint += (offset.ToVector3()).ToVector();
h.Fingers[f].bones[i].Rotation = hand.Fingers[f].bones[i].Rotation;
//h.Fingers[f].bones[i].Rotation = (h.Fingers[f].bones[i].Rotation.ToQuaternion() * handRotOffset).ToLeapQuaternion();
h.Fingers[f].bones[i].Direction = (h.Fingers[f].bones[i].NextJoint - h.Fingers[f].bones[i].PrevJoint);
h.Fingers[f].bones[i].Center = (h.Fingers[f].bones[i].PrevJoint + h.Fingers[f].bones[i].NextJoint) / 2f;
}
h.Fingers[f].Direction = h.Fingers[f].GetBone(Bone.BoneType.TYPE_INTERMEDIATE).Direction;
}
return h;
}
Any suggestions/help would be much appreciated.
UPDATE:
Here's the new code based on the suggestion recieved.
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Leap;
using Leap.Unity;
public class MimicHandModelDriver : MonoBehaviour
{
public Chirality whichHand = Chirality.Left;
public enum RotationMode { Inherit, Overwrite }
public RotationMode rotationMode = RotationMode.Inherit;
public MultiLeap_CapsuleHand handModelToDrive;
private bool _handModelInitialized = false;
private bool _handModelBegun = false;
private Hand mimicHand = null;
Hand sourceHand; //The current hand from the live data
Hand poseHand; //Reference to the hand we load from XML that is in the pose we want
public Hand SourceHand
{
set
{
sourceHand = value;
}
}
public Hand PoseHand
{
set
{
poseHand = value;
}
}
private void Update()
{
if (sourceHand != null)
{
// Copy data from the tracked hand into the mimic hand.
if (mimicHand == null) { mimicHand = new Hand(); }
mimicHand.CopyFrom(poseHand); //copy the stored pose in the mimic hand
mimicHand.Arm.CopyFrom(poseHand.Arm); // copy the stored pose's arm into the mimic hand
// Use the rotation from the live data
var handRotation = sourceHand.Rotation.ToQuaternion();
// Transform the copied hand so that it's centered on the current hands position and matches it's rotation.
mimicHand.SetTransform(sourceHand.PalmPosition.ToVector3(), handRotation);
}
// Drive the attached HandModel.
if (mimicHand != null && handModelToDrive != null)
{
// Initialize the handModel if it hasn't already been initialized.
if (!_handModelInitialized)
{
handModelToDrive.SetLeapHand(mimicHand); //Prevents an error with null reference exception when creating the spheres from
//the init hand call
handModelToDrive.InitHand();
_handModelInitialized = true;
}
// Set the HandModel's hand data.
handModelToDrive.SetLeapHand(mimicHand);
// "Begin" the HandModel to represent a 'newly tracked' hand.
if (!_handModelBegun)
{
handModelToDrive.BeginHand();
_handModelBegun = true;
}
Debug.Log("Updating the mimic hand");
handModelToDrive.UpdateTheHand(); //This method contains the update code, with update code commented out
//so i control when a hand is updated. I have used this throughout the rest of my project so i know this works.
}
}
}
We've got some convenience/extension methods built into our UnityModules that make copying hand data around easier:
hand.Transform(LeapTransform transform) -- this will apply a transform to the Hand.
hand.SetTransform(Vector3 position, Quaternion rotation) -- this will transform a Hand to center the PalmPosition on position and transform the whole hand to align with rotation.
hand.CopyFrom(Hand other) -- this one will handle all the gorey details of copying the data (read: pose) from one Hand to another.
In this case, CopyFrom and SetTransform will get you the data you're looking for.
EDIT: So, the new aspect of your question involves using this data to drive a HandModel -- in this case, a CapsuleHand. For the current version of our Unity assets, the hand model pipeline is pretty shut-down. You're either using the standard Leap rig to drive real tracked hands, or you have a bit of a rough time.
Fortunately, you can just drive a HandModel manually, but you have to be careful to call the right methods in the right order. Check out this example script, which can drive a HandModel that it has a reference to. Important: because you're driving the HandModel independently of the normal Provider/HandPool/HandGroup pipeline, you need to create a new HandModel -- e.g. a new, duplicate CapsuleHand object -- that isn't in your Leap Rig and isn't being driven by that rig's HandPool. This script can then drive it:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Leap;
using Leap.Unity;
public class MimicHandModelDriver : MonoBehaviour {
public Chirality whichHand = Chirality.Left;
public enum RotationMode { Inherit, Overwrite }
public RotationMode rotationMode = RotationMode.Inherit;
public HandModelBase handModelToDrive;
private bool _handModelInitialized = false;
private bool _handModelBegun = false;
[Header("Debug")]
public bool drawEditorGizmos = false;
private Hand mimicHand = null;
private void Update() {
// Get a hand from the standard tracking pipeline.
var sourceHand = Hands.Get(whichHand);
if (sourceHand != null) {
// Copy data from the tracked hand into the mimic hand.
if (mimicHand == null) { mimicHand = new Hand(); }
mimicHand.CopyFrom(sourceHand);
mimicHand.Arm.CopyFrom(sourceHand.Arm); // Capsule Hands like to have Arm data too.
// Figure out what rotation to use for the mimic hand.
var handRotation = this.transform.rotation;
if (rotationMode == RotationMode.Inherit) {
handRotation = mimicHand.Rotation.ToQuaternion();
}
// Transform the copied hand so that it's centered on this object's transform.
mimicHand.SetTransform(this.transform.position, handRotation);
}
// Drive the attached HandModel.
if (mimicHand != null && handModelToDrive != null) {
// Initialize the handModel if it hasn't already been initialized.
if (!_handModelInitialized) {
handModelToDrive.InitHand();
_handModelInitialized = true;
}
// Set the HandModel's hand data.
handModelToDrive.SetLeapHand(mimicHand);
// "Begin" the HandModel to represent a 'newly tracked' hand.
if (!_handModelBegun) {
handModelToDrive.BeginHand();
_handModelBegun = true;
}
// Every Update, we call UpdateHand. It's necessary to call this every update
// specifically for CapsuleHands, which uses manual GL calls to render rather than
// a standard MeshRenderer.
handModelToDrive.UpdateHand();
}
}
// Draw some gizmos in case there's no HandModel attached.
private void OnDrawGizmos() {
if (!drawEditorGizmos) return;
Gizmos.color = Color.red;
if (mimicHand != null) {
draw(mimicHand.PalmPosition.ToVector3());
for (int f = 0; f < 5; f++) {
for (int b = 0; b < 4; b++) {
draw(mimicHand.Fingers[f].bones[b].NextJoint.ToVector3());
}
}
}
}
private void draw(Vector3 pos) {
Gizmos.DrawWireSphere(pos, 0.01f);
}
}