From 7d1a1abe7680f75b83034cbd6438690b75c83718 Mon Sep 17 00:00:00 2001 From: Malcolm Nixon Date: Sun, 11 Feb 2024 20:23:52 -0500 Subject: [PATCH] Initial XRFaceTrackingProvider and XRFaceTracker work. Updated to XRFaceModifier3D. --- doc/classes/XRFaceModifier3D.xml | 22 ++ doc/classes/XRFaceTracker.xml | 469 +++++++++++++++++++++++ doc/classes/XRServer.xml | 48 +++ scene/3d/xr_face_modifier_3d.cpp | 615 ++++++++++++++++++++++++++++++ scene/3d/xr_face_modifier_3d.h | 73 ++++ scene/register_scene_types.cpp | 2 + servers/register_server_types.cpp | 2 + servers/xr/xr_face_tracker.cpp | 222 +++++++++++ servers/xr/xr_face_tracker.h | 213 +++++++++++ servers/xr_server.cpp | 48 +++ servers/xr_server.h | 11 + 11 files changed, 1725 insertions(+) create mode 100644 doc/classes/XRFaceModifier3D.xml create mode 100644 doc/classes/XRFaceTracker.xml create mode 100644 scene/3d/xr_face_modifier_3d.cpp create mode 100644 scene/3d/xr_face_modifier_3d.h create mode 100644 servers/xr/xr_face_tracker.cpp create mode 100644 servers/xr/xr_face_tracker.h diff --git a/doc/classes/XRFaceModifier3D.xml b/doc/classes/XRFaceModifier3D.xml new file mode 100644 index 00000000000..7a60e6db346 --- /dev/null +++ b/doc/classes/XRFaceModifier3D.xml @@ -0,0 +1,22 @@ + + + + A node for driving standard face meshes from [XRFaceTracker] weights. + + + This node applies weights from a [XRFaceTracker] to a mesh with supporting face blend shapes. + The [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/unified-blendshapes]Unified Expressions[/url] blend shapes are supported, as well as ARKit and SRanipal blend shapes. + The node attempts to identify blend shapes based on name matching. Blend shapes should match the names listed in the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/compatibility/overview]Unified Expressions Compatibility[/url] chart. + + + $DOCS_URL/tutorials/xr/index.html + + + + The [XRFaceTracker] path. + + + The [NodePath] of the face [MeshInstance3D]. + + + diff --git a/doc/classes/XRFaceTracker.xml b/doc/classes/XRFaceTracker.xml new file mode 100644 index 00000000000..0212cc8fffe --- /dev/null +++ b/doc/classes/XRFaceTracker.xml @@ -0,0 +1,469 @@ + + + + A tracked face. + + + An instance of this object represents a tracked face and its corresponding blend shapes. The blend shapes come from the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/unified-blendshapes]Unified Expressions[/url] standard, and contain extended details and visuals for each blend shape. Additionally the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/compatibility/overview]Tracking Standard Comparison[/url] page documents the relationship between Unified Expressions and other standards. + As face trackers are turned on they are registered with the [XRServer]. + + + $DOCS_URL/tutorials/xr/index.html + + + + + + + Returns the requested face blend shape weight. + + + + + + + + Sets a face blend shape weight. + + + + + + The array of face blend shape weights with indices corresponding to the [enum BlendShapeEntry] enum. + + + + + Right eye looks outwards. + + + Right eye looks inwards. + + + Right eye looks upwards. + + + Right eye looks downwards. + + + Left eye looks outwards. + + + Left eye looks inwards. + + + Left eye looks upwards. + + + Left eye looks downwards. + + + Closes the right eyelid. + + + Closes the left eyelid. + + + Squeezes the right eye socket muscles. + + + Squeezes the left eye socket muscles. + + + Right eyelid widens beyond relaxed. + + + Left eyelid widens beyond relaxed. + + + Dilates the right eye pupil. + + + Dilates the left eye pupil. + + + Constricts the right eye pupil. + + + Constricts the left eye pupil. + + + Right eyebrow pinches in. + + + Left eyebrow pinches in. + + + Outer right eyebrow pulls down. + + + Outer left eyebrow pulls down. + + + Inner right eyebrow pulls up. + + + Inner left eyebrow pulls up. + + + Outer right eyebrow pulls up. + + + Outer left eyebrow pulls up. + + + Right side face sneers. + + + Left side face sneers. + + + Right side nose canal dilates. + + + Left side nose canal dilates. + + + Right side nose canal constricts. + + + Left side nose canal constricts. + + + Raises the right side cheek. + + + Raises the left side cheek. + + + Puffs the right side cheek. + + + Puffs the left side cheek. + + + Sucks in the right side cheek. + + + Sucks in the left side cheek. + + + Opens jawbone. + + + Closes the mouth. + + + Pushes jawbone right. + + + Pushes jawbone left. + + + Pushes jawbone forward. + + + Pushes jawbone backward. + + + Flexes jaw muscles. + + + Raises the jawbone. + + + Upper right lip part tucks in the mouth. + + + Upper left lip part tucks in the mouth. + + + Lower right lip part tucks in the mouth. + + + Lower left lip part tucks in the mouth. + + + Right lip corner folds into the mouth. + + + Left lip corner folds into the mouth. + + + Upper right lip part pushes into a funnel. + + + Upper left lip part pushes into a funnel. + + + Lower right lip part pushes into a funnel. + + + Lower left lip part pushes into a funnel. + + + Upper right lip part pushes outwards. + + + Upper left lip part pushes outwards. + + + Lower right lip part pushes outwards. + + + Lower left lip part pushes outwards. + + + Upper right part of the lip pulls up. + + + Upper left part of the lip pulls up. + + + Lower right part of the lip pulls up. + + + Lower left part of the lip pulls up. + + + Upper right lip part pushes in the cheek. + + + Upper left lip part pushes in the cheek. + + + Moves upper lip right. + + + Moves upper lip left. + + + Moves lower lip right. + + + Moves lower lip left. + + + Right lip corner pulls diagonally up and out. + + + Left lip corner pulls diagonally up and out. + + + Right corner lip slants up. + + + Left corner lip slants up. + + + Right corner lip pulls down. + + + Left corner lip pulls down. + + + Mouth corner lip pulls out and down. + + + Mouth corner lip pulls out and down. + + + Right lip corner is pushed backwards. + + + Left lip corner is pushed backwards. + + + Raises and slightly pushes out the upper mouth. + + + Raises and slightly pushes out the lower mouth. + + + Right side lips press and flatten together vertically. + + + Left side lips press and flatten together vertically. + + + Right side lips squeeze together horizontally. + + + Left side lips squeeze together horizontally. + + + Tongue visibly sticks out of the mouth. + + + Tongue points upwards. + + + Tongue points downwards. + + + Tongue points right. + + + Tongue points left. + + + Sides of the tongue funnel, creating a roll. + + + Tongue arches up then down inside the mouth. + + + Tongue arches down then up inside the mouth. + + + Tongue squishes together and thickens. + + + Tongue flattens and thins out. + + + Tongue tip rotates clockwise, with the rest following gradually. + + + Tongue tip rotates counter-clockwise, with the rest following gradually. + + + Inner mouth throat closes. + + + The Adam's apple visibly swallows. + + + Right side neck visibly flexes. + + + Left side neck visibly flexes. + + + Closes both eye lids. + + + Widens both eye lids. + + + Squints both eye lids. + + + Dilates both pupils. + + + Constricts both pupils. + + + Pulls the right eyebrow down and in. + + + Pulls the left eyebrow down and in. + + + Pulls both eyebrows down and in. + + + Right brow appears worried. + + + Left brow appears worried. + + + Both brows appear worried. + + + Entire face sneers. + + + Both nose canals dilate. + + + Both nose canals constrict. + + + Puffs both cheeks. + + + Sucks in both cheeks. + + + Raises both cheeks. + + + Tucks in the upper lips. + + + Tucks in the lower lips. + + + Tucks in both lips. + + + Funnels in the upper lips. + + + Funnels in the lower lips. + + + Funnels in both lips. + + + Upper lip part pushes outwards. + + + Lower lip part pushes outwards. + + + Lips push outwards. + + + Raises the upper lips. + + + Lowers the lower lips. + + + Mouth opens, revealing teeth. + + + Moves mouth right. + + + Moves mouth left. + + + Right side of the mouth smiles. + + + Left side of the mouth smiles. + + + Mouth expresses a smile. + + + Right side of the mouth expresses sadness. + + + Left side of the mouth expresses sadness. + + + Mouth expresses sadness. + + + Mouth stretches. + + + Lip corners dimple. + + + Mouth tightens. + + + Mouth presses together. + + + Represents the size of the [enum BlendShapeEntry] enum. + + + diff --git a/doc/classes/XRServer.xml b/doc/classes/XRServer.xml index 2f106af340a..f98c1d66a48 100644 --- a/doc/classes/XRServer.xml +++ b/doc/classes/XRServer.xml @@ -10,6 +10,14 @@ $DOCS_URL/tutorials/xr/index.html + + + + + + Registers a new [XRFaceTracker] that tracks the blend shapes of a face. + + @@ -50,6 +58,19 @@ Finds an interface by its [param name]. For example, if your project uses capabilities of an AR/VR platform, you can find the interface for that platform by name and initialize it. + + + + + Returns the [XRFaceTracker] with the given tracker name. + + + + + + Returns a dictionary of the registered face trackers. Each element of the dictionary is a tracker name mapping to the [XRFaceTracker] instance. + + @@ -95,6 +116,13 @@ Returns a dictionary of trackers for [param tracker_types]. + + + + + Removes a registered [XRFaceTracker]. + + @@ -123,6 +151,26 @@ + + + + + Emitted when a new face tracker is added. + + + + + + Emitted when a face tracker is removed. + + + + + + + Emitted when an existing face tracker is updated. + + diff --git a/scene/3d/xr_face_modifier_3d.cpp b/scene/3d/xr_face_modifier_3d.cpp new file mode 100644 index 00000000000..be92a587b0d --- /dev/null +++ b/scene/3d/xr_face_modifier_3d.cpp @@ -0,0 +1,615 @@ +/**************************************************************************/ +/* xr_face_modifier_3d.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "xr_face_modifier_3d.h" + +#include "servers/xr/xr_face_tracker.h" +#include "servers/xr_server.h" + +// This method takes the name of a mesh blend shape and returns the +// corresponding XRFaceTracker blend shape. If no match is +// found then the function returns -1. +static int find_face_blend_shape(const StringName &p_name) { + // Entry for blend shape name table. + struct blend_map_entry { + int blend; + const char *name[4]; + }; + + // Table of blend shape names. + // + // This table consists of the XRFaceTracker blend shape and + // the corresponding names (lowercase and no underscore) of: + // - The Unified Expression blend shape name. + // - The ARKit blend shape name (if present and different). + // - The SRanipal blend shape name (if present and different). + // - The Meta blend shape name (if present and different). + static constexpr blend_map_entry blend_map[] = { + { XRFaceTracker::FT_EYE_LOOK_OUT_RIGHT, + { "eyelookoutright", "eyerightright", "eyeslookoutr" } }, + { XRFaceTracker::FT_EYE_LOOK_IN_RIGHT, + { "eyelookinright", "eyerightleft", "eyeslookinr" } }, + { XRFaceTracker::FT_EYE_LOOK_UP_RIGHT, + { "eyelookupright", "eyerightlookup", "eyeslookupr" } }, + { XRFaceTracker::FT_EYE_LOOK_DOWN_RIGHT, + { "eyelookdownright", "eyerightlookdown", "eyeslookdownr" } }, + { XRFaceTracker::FT_EYE_LOOK_OUT_LEFT, + { "eyelookoutleft", "eyeleftleft", "eyeslookoutl" } }, + { XRFaceTracker::FT_EYE_LOOK_IN_LEFT, + { "eyelookinleft", "eyeleftright", "eyeslookinl" } }, + { XRFaceTracker::FT_EYE_LOOK_UP_LEFT, + { "eyelookupleft", "eyeleftlookup", "eyeslookupl" } }, + { XRFaceTracker::FT_EYE_LOOK_DOWN_LEFT, + { "eyelookdownleft", "eyeleftlookdown", "eyeslookdownl" } }, + { XRFaceTracker::FT_EYE_CLOSED_RIGHT, + { "eyeclosedright", "eyeblinkright", "eyerightblink", "eyesclosedr" } }, + { XRFaceTracker::FT_EYE_CLOSED_LEFT, + { "eyeclosedleft", "eyeblinkleft", "eyeleftblink", "eyesclosedl" } }, + { XRFaceTracker::FT_EYE_SQUINT_RIGHT, + { "eyesquintright", "eyessquintr" } }, + { XRFaceTracker::FT_EYE_SQUINT_LEFT, + { "eyesquintleft", "eyessquintl" } }, + { XRFaceTracker::FT_EYE_WIDE_RIGHT, + { "eyewideright", "eyerightwide", "eyeswidenr" } }, + { XRFaceTracker::FT_EYE_WIDE_LEFT, + { "eyewideleft", "eyeleftwide", "eyeswidenl" } }, + { XRFaceTracker::FT_EYE_DILATION_RIGHT, + { "eyedilationright", "eyerightdilation" } }, + { XRFaceTracker::FT_EYE_DILATION_LEFT, + { "eyedilationleft", "eyeleftdilation" } }, + { XRFaceTracker::FT_EYE_CONSTRICT_RIGHT, + { "eyeconstrictright", "eyerightconstrict" } }, + { XRFaceTracker::FT_EYE_CONSTRICT_LEFT, + { "eyeconstrictleft", "eyeleftconstrict" } }, + { XRFaceTracker::FT_BROW_PINCH_RIGHT, + { "browpinchright" } }, + { XRFaceTracker::FT_BROW_PINCH_LEFT, + { "browpinchleft" } }, + { XRFaceTracker::FT_BROW_LOWERER_RIGHT, + { "browlowererright" } }, + { XRFaceTracker::FT_BROW_LOWERER_LEFT, + { "browlowererleft" } }, + { XRFaceTracker::FT_BROW_INNER_UP_RIGHT, + { "browinnerupright", "innerbrowraiserr" } }, + { XRFaceTracker::FT_BROW_INNER_UP_LEFT, + { "browinnerupleft", "innerbrowraiserl" } }, + { XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, + { "browouterupright", "outerbrowraiserr" } }, + { XRFaceTracker::FT_BROW_OUTER_UP_LEFT, + { "browouterupleft", "outerbrowraiserl" } }, + { XRFaceTracker::FT_NOSE_SNEER_RIGHT, + { "nosesneerright", "nosewrinklerr" } }, + { XRFaceTracker::FT_NOSE_SNEER_LEFT, + { "nosesneerleft", "nosewrinklerl" } }, + { XRFaceTracker::FT_NASAL_DILATION_RIGHT, + { "nasaldilationright" } }, + { XRFaceTracker::FT_NASAL_DILATION_LEFT, + { "nasaldilationleft" } }, + { XRFaceTracker::FT_NASAL_CONSTRICT_RIGHT, + { "nasalconstrictright" } }, + { XRFaceTracker::FT_NASAL_CONSTRICT_LEFT, + { "nasalconstrictleft" } }, + { XRFaceTracker::FT_CHEEK_SQUINT_RIGHT, + { "cheeksquintright", "cheekraiserr" } }, + { XRFaceTracker::FT_CHEEK_SQUINT_LEFT, + { "cheeksquintleft", "cheekraiserl" } }, + { XRFaceTracker::FT_CHEEK_PUFF_RIGHT, + { "cheekpuffright", "cheekpuffr" } }, + { XRFaceTracker::FT_CHEEK_PUFF_LEFT, + { "cheekpuffleft", "cheekpuffl" } }, + { XRFaceTracker::FT_CHEEK_SUCK_RIGHT, + { "cheeksuckright", "cheeksuckr" } }, + { XRFaceTracker::FT_CHEEK_SUCK_LEFT, + { "cheeksuckleft", "cheeksuckl" } }, + { XRFaceTracker::FT_JAW_OPEN, + { "jawopen", "jawdrop" } }, + { XRFaceTracker::FT_MOUTH_CLOSED, + { "mouthclosed", "mouthclose", "mouthapeshape", "lipstoward" } }, + { XRFaceTracker::FT_JAW_RIGHT, + { "jawright", "jawsidewaysright" } }, + { XRFaceTracker::FT_JAW_LEFT, + { "jawleft", "jawsidewaysleft" } }, + { XRFaceTracker::FT_JAW_FORWARD, + { "jawforward", "jawthrust" } }, + { XRFaceTracker::FT_JAW_BACKWARD, + { "jawbackward" } }, + { XRFaceTracker::FT_JAW_CLENCH, + { "jawclench" } }, + { XRFaceTracker::FT_JAW_MANDIBLE_RAISE, + { "jawmandibleraise" } }, + { XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, + { "lipsuckupperright", "lipsuckrt" } }, + { XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, + { "lipsuckupperleft", "lipsucklt" } }, + { XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, + { "lipsucklowerright", "lipsuckrb" } }, + { XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT, + { "lipsucklowerleft", "lipsucklb" } }, + { XRFaceTracker::FT_LIP_SUCK_CORNER_RIGHT, + { "lipsuckcornerright" } }, + { XRFaceTracker::FT_LIP_SUCK_CORNER_LEFT, + { "lipsuckcornerleft" } }, + { XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, + { "lipfunnelupperright", "lipfunnelerrt" } }, + { XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, + { "lipfunnelupperleft", "lipfunnelerlt" } }, + { XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, + { "lipfunnellowerright", "lipsuckrb" } }, + { XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT, + { "lipfunnellowerleft", "lipsucklb" } }, + { XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, + { "lippuckerupperright" } }, + { XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, + { "lippuckerupperleft" } }, + { XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, + { "lippuckerlowerright" } }, + { XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT, + { "lippuckerlowerleft" } }, + { XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, + { "mouthupperupright", "upperlipraiserr" } }, + { XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, + { "mouthupperupleft", "upperlipraiserl" } }, + { XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, + { "mouthlowerdownright", "mouthlowerupright", "lowerlipdepressorr" } }, + { XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT, + { "mouthlowerdownleft", "mouthlowerupleft", "lowerlipdepressorl" } }, + { XRFaceTracker::FT_MOUTH_UPPER_DEEPEN_RIGHT, + { "mouthupperdeepenright" } }, + { XRFaceTracker::FT_MOUTH_UPPER_DEEPEN_LEFT, + { "mouthupperdeepenleft" } }, + { XRFaceTracker::FT_MOUTH_UPPER_RIGHT, + { "mouthupperright" } }, + { XRFaceTracker::FT_MOUTH_UPPER_LEFT, + { "mouthupperleft" } }, + { XRFaceTracker::FT_MOUTH_LOWER_RIGHT, + { "mouthlowerright" } }, + { XRFaceTracker::FT_MOUTH_LOWER_LEFT, + { "mouthlowerleft" } }, + { XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, + { "mouthcornerpullright" } }, + { XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, + { "mouthcornerpullleft" } }, + { XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, + { "mouthcornerslantright" } }, + { XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT, + { "mouthcornerslantleft" } }, + { XRFaceTracker::FT_MOUTH_FROWN_RIGHT, + { "mouthfrownright", "lipcornerdepressorr" } }, + { XRFaceTracker::FT_MOUTH_FROWN_LEFT, + { "mouthfrownleft", "lipcornerdepressorl" } }, + { XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, + { "mouthstretchright", "lipstretcherr" } }, + { XRFaceTracker::FT_MOUTH_STRETCH_LEFT, + { "mouthstretchleft", "lipstretcherl" } }, + { XRFaceTracker::FT_MOUTH_DIMPLE_RIGHT, + { "mouthdimplerright", "mouthdimpleright", "dimplerr" } }, + { XRFaceTracker::FT_MOUTH_DIMPLE_LEFT, + { "mouthdimplerleft", "mouthdimpleleft", "dimplerl" } }, + { XRFaceTracker::FT_MOUTH_RAISER_UPPER, + { "mouthraiserupper", "mouthshrugupper", "chinraisert" } }, + { XRFaceTracker::FT_MOUTH_RAISER_LOWER, + { "mouthraiserlower", "mouthshruglower", "mouthloweroverlay", "chinraiserb" } }, + { XRFaceTracker::FT_MOUTH_PRESS_RIGHT, + { "mouthpressright", "lippressorr" } }, + { XRFaceTracker::FT_MOUTH_PRESS_LEFT, + { "mouthpressleft", "lippressorl" } }, + { XRFaceTracker::FT_MOUTH_TIGHTENER_RIGHT, + { "mouthtightenerright", "liptightenerr" } }, + { XRFaceTracker::FT_MOUTH_TIGHTENER_LEFT, + { "mouthtightenerleft", "liptightenerl" } }, + { XRFaceTracker::FT_TONGUE_OUT, + { "tongueout", "tonguelongstep2" } }, + { XRFaceTracker::FT_TONGUE_UP, + { "tongueup" } }, + { XRFaceTracker::FT_TONGUE_DOWN, + { "tonguedown" } }, + { XRFaceTracker::FT_TONGUE_RIGHT, + { "tongueright" } }, + { XRFaceTracker::FT_TONGUE_LEFT, + { "tongueleft" } }, + { XRFaceTracker::FT_TONGUE_ROLL, + { "tongueroll" } }, + { XRFaceTracker::FT_TONGUE_BLEND_DOWN, + { "tongueblenddown" } }, + { XRFaceTracker::FT_TONGUE_CURL_UP, + { "tonguecurlup" } }, + { XRFaceTracker::FT_TONGUE_SQUISH, + { "tonguesquish" } }, + { XRFaceTracker::FT_TONGUE_FLAT, + { "tongueflat" } }, + { XRFaceTracker::FT_TONGUE_TWIST_RIGHT, + { "tonguetwistright" } }, + { XRFaceTracker::FT_TONGUE_TWIST_LEFT, + { "tonguetwistleft" } }, + { XRFaceTracker::FT_SOFT_PALATE_CLOSE, + { "softpalateclose" } }, + { XRFaceTracker::FT_THROAT_SWALLOW, + { "throatswallow" } }, + { XRFaceTracker::FT_NECK_FLEX_RIGHT, + { "neckflexright" } }, + { XRFaceTracker::FT_NECK_FLEX_LEFT, + { "neckflexleft" } }, + { XRFaceTracker::FT_EYE_CLOSED, + { "eyeclosed" } }, + { XRFaceTracker::FT_EYE_WIDE, + { "eyewide" } }, + { XRFaceTracker::FT_EYE_SQUINT, + { "eyesquint" } }, + { XRFaceTracker::FT_EYE_DILATION, + { "eyedilation" } }, + { XRFaceTracker::FT_EYE_CONSTRICT, + { "eyeconstrict" } }, + { XRFaceTracker::FT_BROW_DOWN_RIGHT, + { "browdownright", "browlowererr" } }, + { XRFaceTracker::FT_BROW_DOWN_LEFT, + { "browdownleft", "browlowererl" } }, + { XRFaceTracker::FT_BROW_DOWN, + { "browdown" } }, + { XRFaceTracker::FT_BROW_UP_RIGHT, + { "browupright" } }, + { XRFaceTracker::FT_BROW_UP_LEFT, + { "browupleft" } }, + { XRFaceTracker::FT_BROW_UP, + { "browup" } }, + { XRFaceTracker::FT_NOSE_SNEER, + { "nosesneer" } }, + { XRFaceTracker::FT_NASAL_DILATION, + { "nasaldilation" } }, + { XRFaceTracker::FT_NASAL_CONSTRICT, + { "nasalconstrict" } }, + { XRFaceTracker::FT_CHEEK_PUFF, + { "cheekpuff" } }, + { XRFaceTracker::FT_CHEEK_SUCK, + { "cheeksuck" } }, + { XRFaceTracker::FT_CHEEK_SQUINT, + { "cheeksquint" } }, + { XRFaceTracker::FT_LIP_SUCK_UPPER, + { "lipsuckupper", "mouthrollupper", "mouthupperinside" } }, + { XRFaceTracker::FT_LIP_SUCK_LOWER, + { "lipsucklower", "mouthrolllower", "mouthlowerinside" } }, + { XRFaceTracker::FT_LIP_SUCK, + { "lipsuck" } }, + { XRFaceTracker::FT_LIP_FUNNEL_UPPER, + { "lipfunnelupper", "mouthupperoverturn" } }, + { XRFaceTracker::FT_LIP_FUNNEL_LOWER, + { "lipfunnellower", "mouthloweroverturn" } }, + { XRFaceTracker::FT_LIP_FUNNEL, + { "lipfunnel", "mouthfunnel" } }, + { XRFaceTracker::FT_LIP_PUCKER_UPPER, + { "lippuckerupper" } }, + { XRFaceTracker::FT_LIP_PUCKER_LOWER, + { "lippuckerlower" } }, + { XRFaceTracker::FT_LIP_PUCKER, + { "lippucker", "mouthpucker", "mouthpout" } }, + { XRFaceTracker::FT_MOUTH_UPPER_UP, + { "mouthupperup" } }, + { XRFaceTracker::FT_MOUTH_LOWER_DOWN, + { "mouthlowerdown" } }, + { XRFaceTracker::FT_MOUTH_OPEN, + { "mouthopen" } }, + { XRFaceTracker::FT_MOUTH_RIGHT, + { "mouthright" } }, + { XRFaceTracker::FT_MOUTH_LEFT, + { "mouthleft" } }, + { XRFaceTracker::FT_MOUTH_SMILE_RIGHT, + { "mouthsmileright", "lipcornerpullerr" } }, + { XRFaceTracker::FT_MOUTH_SMILE_LEFT, + { "mouthsmileleft", "lipcornerpullerl" } }, + { XRFaceTracker::FT_MOUTH_SMILE, + { "mouthsmile" } }, + { XRFaceTracker::FT_MOUTH_SAD_RIGHT, + { "mouthsadright" } }, + { XRFaceTracker::FT_MOUTH_SAD_LEFT, + { "mouthsadleft" } }, + { XRFaceTracker::FT_MOUTH_SAD, + { "mouthsad" } }, + { XRFaceTracker::FT_MOUTH_STRETCH, + { "mouthstretch" } }, + { XRFaceTracker::FT_MOUTH_DIMPLE, + { "mouthdimple" } }, + { XRFaceTracker::FT_MOUTH_TIGHTENER, + { "mouthtightener" } }, + { XRFaceTracker::FT_MOUTH_PRESS, + { "mouthpress" } } + }; + + // Convert the name to lower-case and strip non-alphanumeric characters. + const String name = String(p_name).to_lower().replace("_", ""); + + // Iterate through the blend map. + for (const blend_map_entry &entry : blend_map) { + for (const char *n : entry.name) { + if (n == nullptr) { + break; + } + + if (name == n) { + return entry.blend; + } + } + } + + // Blend shape not found. + return -1; +} + +// This method adds all the identified XRFaceTracker blend shapes of +// the mesh to the p_blend_mapping map. The map is indexed by the +// XRFaceTracker blend shape, and the value is the index of the mesh +// blend shape. +static void identify_face_blend_shapes(RBMap &p_blend_mapping, const Ref &mesh) { + // Find all blend shapes. + const int count = mesh->get_blend_shape_count(); + for (int i = 0; i < count; i++) { + const int blend = find_face_blend_shape(mesh->get_blend_shape_name(i)); + if (blend >= 0) { + p_blend_mapping[blend] = i; + } + } +} + +// This method removes any unified blend shapes from the p_blend_mapping map +// if all the individual blend shapes are found and going to be driven. +static void remove_driven_unified_blend_shapes(RBMap &p_blend_mapping) { + // Entry for unified blend table. + struct unified_blend_entry { + int unified; + int individual[4]; + }; + + // Table of unified blend shapes. + // + // This table consists of: + // - The XRFaceTracker unified blend shape. + // - The individual blend shapes that make up the unified blend shape. + static constexpr unified_blend_entry unified_blends[] = { + { XRFaceTracker::FT_EYE_CLOSED, + { XRFaceTracker::FT_EYE_CLOSED_RIGHT, XRFaceTracker::FT_EYE_CLOSED_LEFT, -1, -1 } }, + { XRFaceTracker::FT_EYE_WIDE, + { XRFaceTracker::FT_EYE_WIDE_RIGHT, XRFaceTracker::FT_EYE_WIDE_LEFT, -1, -1 } }, + { XRFaceTracker::FT_EYE_SQUINT, + { XRFaceTracker::FT_EYE_SQUINT_RIGHT, XRFaceTracker::FT_EYE_SQUINT_LEFT, -1, -1 } }, + { XRFaceTracker::FT_EYE_DILATION, + { XRFaceTracker::FT_EYE_DILATION_RIGHT, XRFaceTracker::FT_EYE_DILATION_LEFT, -1, -1 } }, + { XRFaceTracker::FT_EYE_CONSTRICT, + { XRFaceTracker::FT_EYE_CONSTRICT_RIGHT, XRFaceTracker::FT_EYE_CONSTRICT_LEFT, -1, -1 } }, + { XRFaceTracker::FT_BROW_DOWN_RIGHT, + { XRFaceTracker::FT_BROW_LOWERER_RIGHT, XRFaceTracker::FT_BROW_PINCH_RIGHT, -1, -1 } }, + { XRFaceTracker::FT_BROW_DOWN_LEFT, + { XRFaceTracker::FT_BROW_LOWERER_LEFT, XRFaceTracker::FT_BROW_PINCH_LEFT, -1, -1 } }, + { XRFaceTracker::FT_BROW_DOWN, + { XRFaceTracker::FT_BROW_LOWERER_RIGHT, XRFaceTracker::FT_BROW_PINCH_RIGHT, XRFaceTracker::FT_BROW_LOWERER_LEFT, XRFaceTracker::FT_BROW_PINCH_LEFT } }, + { XRFaceTracker::FT_BROW_UP_RIGHT, + { XRFaceTracker::FT_BROW_INNER_UP_RIGHT, XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, -1, -1 } }, + { XRFaceTracker::FT_BROW_UP_LEFT, + { XRFaceTracker::FT_BROW_INNER_UP_LEFT, XRFaceTracker::FT_BROW_OUTER_UP_LEFT, -1, -1 } }, + { XRFaceTracker::FT_BROW_UP, + { XRFaceTracker::FT_BROW_INNER_UP_RIGHT, XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, XRFaceTracker::FT_BROW_INNER_UP_LEFT, XRFaceTracker::FT_BROW_OUTER_UP_LEFT } }, + { XRFaceTracker::FT_NOSE_SNEER, + { XRFaceTracker::FT_NOSE_SNEER_RIGHT, XRFaceTracker::FT_NOSE_SNEER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_NASAL_DILATION, + { XRFaceTracker::FT_NASAL_DILATION_RIGHT, XRFaceTracker::FT_NASAL_DILATION_LEFT, -1, -1 } }, + { XRFaceTracker::FT_NASAL_CONSTRICT, + { XRFaceTracker::FT_NASAL_CONSTRICT_RIGHT, XRFaceTracker::FT_NASAL_CONSTRICT_LEFT, -1, -1 } }, + { XRFaceTracker::FT_CHEEK_PUFF, + { XRFaceTracker::FT_CHEEK_PUFF_RIGHT, XRFaceTracker::FT_CHEEK_PUFF_LEFT, -1, -1 } }, + { XRFaceTracker::FT_CHEEK_SUCK, + { XRFaceTracker::FT_CHEEK_SUCK_RIGHT, XRFaceTracker::FT_CHEEK_SUCK_LEFT, -1, -1 } }, + { XRFaceTracker::FT_CHEEK_SQUINT, + { XRFaceTracker::FT_CHEEK_SQUINT_RIGHT, XRFaceTracker::FT_CHEEK_SQUINT_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_SUCK_UPPER, + { XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_SUCK_LOWER, + { XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_SUCK, + { XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT } }, + { XRFaceTracker::FT_LIP_FUNNEL_UPPER, + { XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_FUNNEL_LOWER, + { XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_FUNNEL, + { XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT } }, + { XRFaceTracker::FT_LIP_PUCKER_UPPER, + { XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_PUCKER_LOWER, + { XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_LIP_PUCKER, + { XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT } }, + { XRFaceTracker::FT_MOUTH_UPPER_UP, + { XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_LOWER_DOWN, + { XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_OPEN, + { XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT } }, + { XRFaceTracker::FT_MOUTH_RIGHT, + { XRFaceTracker::FT_MOUTH_UPPER_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_RIGHT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_LEFT, + { XRFaceTracker::FT_MOUTH_UPPER_LEFT, XRFaceTracker::FT_MOUTH_LOWER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_SMILE_RIGHT, + { XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_SMILE_LEFT, + { XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_SMILE, + { XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT } }, + { XRFaceTracker::FT_MOUTH_SAD_RIGHT, + { XRFaceTracker::FT_MOUTH_FROWN_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_SAD_LEFT, + { XRFaceTracker::FT_MOUTH_FROWN_LEFT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_SAD, + { XRFaceTracker::FT_MOUTH_FROWN_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, XRFaceTracker::FT_MOUTH_FROWN_LEFT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT } }, + { XRFaceTracker::FT_MOUTH_STRETCH, + { XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_DIMPLE, + { XRFaceTracker::FT_MOUTH_DIMPLE_RIGHT, XRFaceTracker::FT_MOUTH_DIMPLE_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_TIGHTENER, + { XRFaceTracker::FT_MOUTH_TIGHTENER_RIGHT, XRFaceTracker::FT_MOUTH_TIGHTENER_LEFT, -1, -1 } }, + { XRFaceTracker::FT_MOUTH_PRESS, + { XRFaceTracker::FT_MOUTH_PRESS_RIGHT, XRFaceTracker::FT_MOUTH_PRESS_LEFT, -1, -1 } } + }; + + // Remove unified blend shapes if individual blend shapes are found. + for (const unified_blend_entry &entry : unified_blends) { + // Check if all individual blend shapes are found. + bool found = true; + for (const int i : entry.individual) { + if (i >= 0 && !p_blend_mapping.find(i)) { + found = false; + break; + } + } + + // If all individual blend shapes are found then remove the unified blend shape. + if (found) { + p_blend_mapping.erase(entry.unified); + } + } +} + +void XRFaceModifier3D::_bind_methods() { + ClassDB::bind_method(D_METHOD("set_face_tracker", "tracker_name"), &XRFaceModifier3D::set_face_tracker); + ClassDB::bind_method(D_METHOD("get_face_tracker"), &XRFaceModifier3D::get_face_tracker); + ADD_PROPERTY(PropertyInfo(Variant::STRING, "face_tracker", PROPERTY_HINT_ENUM_SUGGESTION, "/user/head"), "set_face_tracker", "get_face_tracker"); + + ClassDB::bind_method(D_METHOD("set_target", "target"), &XRFaceModifier3D::set_target); + ClassDB::bind_method(D_METHOD("get_target"), &XRFaceModifier3D::get_target); + ADD_PROPERTY(PropertyInfo(Variant::NODE_PATH, "target", PROPERTY_HINT_NODE_PATH_VALID_TYPES, "MeshInstance3D"), "set_target", "get_target"); +} + +void XRFaceModifier3D::set_face_tracker(const StringName &p_tracker_name) { + tracker_name = p_tracker_name; +} + +StringName XRFaceModifier3D::get_face_tracker() const { + return tracker_name; +} + +void XRFaceModifier3D::set_target(const NodePath &p_target) { + target = p_target; + + if (is_inside_tree()) { + _get_blend_data(); + } +} + +NodePath XRFaceModifier3D::get_target() const { + return target; +} + +MeshInstance3D *XRFaceModifier3D::get_mesh_instance() const { + if (!has_node(target)) { + return nullptr; + } + + Node *node = get_node(target); + if (!node) { + return nullptr; + } + + return Object::cast_to(node); +} + +void XRFaceModifier3D::_get_blend_data() { + // This method constructs the blend mapping from the XRFaceTracker + // blend shapes to the available blend shapes of the target mesh. It does this + // by: + // + // 1. Identifying the blend shapes of the target mesh and identifying what + // XRFaceTracker blend shape they correspond to. The results are + // placed in the blend_mapping map. + // 2. Prevent over-driving facial blend-shapes by removing any unified blend + // shapes from the map if all the individual blend shapes are already + // found and going to be driven. + + blend_mapping.clear(); + + // Get the target MeshInstance3D. + const MeshInstance3D *mesh_instance = get_mesh_instance(); + if (!mesh_instance) { + return; + } + + // Get the mesh. + const Ref mesh = mesh_instance->get_mesh(); + if (mesh.is_null()) { + return; + } + + // Identify all face blend shapes and populate the map. + identify_face_blend_shapes(blend_mapping, mesh); + + // Remove the unified blend shapes if all the individual blend shapes are found. + remove_driven_unified_blend_shapes(blend_mapping); +} + +void XRFaceModifier3D::_update_face_blends() const { + // Get the XR Server. + const XRServer *xr_server = XRServer::get_singleton(); + if (!xr_server) { + return; + } + + // Get the face tracker. + const Ref p = xr_server->get_face_tracker(tracker_name); + if (!p.is_valid()) { + return; + } + + // Get the face mesh. + MeshInstance3D *mesh_instance = get_mesh_instance(); + if (!mesh_instance) { + return; + } + + // Get the blend weights. + const PackedFloat32Array weights = p->get_blend_shapes(); + + // Apply all the face blend weights to the mesh. + for (const KeyValue &it : blend_mapping) { + mesh_instance->set_blend_shape_value(it.value, weights[it.key]); + } +} + +void XRFaceModifier3D::_notification(int p_what) { + switch (p_what) { + case NOTIFICATION_ENTER_TREE: { + _get_blend_data(); + set_process_internal(true); + } break; + case NOTIFICATION_EXIT_TREE: { + set_process_internal(false); + blend_mapping.clear(); + } break; + case NOTIFICATION_INTERNAL_PROCESS: { + _update_face_blends(); + } break; + default: { + } break; + } +} diff --git a/scene/3d/xr_face_modifier_3d.h b/scene/3d/xr_face_modifier_3d.h new file mode 100644 index 00000000000..147c374e95e --- /dev/null +++ b/scene/3d/xr_face_modifier_3d.h @@ -0,0 +1,73 @@ +/**************************************************************************/ +/* xr_face_modifier_3d.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef XR_FACE_MODIFIER_3D_H +#define XR_FACE_MODIFIER_3D_H + +#include "mesh_instance_3d.h" +#include "scene/3d/node_3d.h" + +/** + The XRFaceModifier3D node drives the blend shapes of a MeshInstance3D + with facial expressions from an XRFaceTracking instance. + + The blend shapes provided by the mesh are interrogated, and used to + deduce an optimal mapping from the Unified Expressions blend shapes + provided by the XRFaceTracking instance to drive the face. + */ + +class XRFaceModifier3D : public Node3D { + GDCLASS(XRFaceModifier3D, Node3D); + +private: + StringName tracker_name = "/user/head"; + NodePath target; + + // Map from XRFaceTracker blend shape index to mesh blend shape index. + RBMap blend_mapping; + + MeshInstance3D *get_mesh_instance() const; + void _get_blend_data(); + void _update_face_blends() const; + +protected: + static void _bind_methods(); + +public: + void set_face_tracker(const StringName &p_tracker_name); + StringName get_face_tracker() const; + + void set_target(const NodePath &p_target); + NodePath get_target() const; + + void _notification(int p_what); +}; + +#endif // XR_FACE_MODIFIER_3D_H diff --git a/scene/register_scene_types.cpp b/scene/register_scene_types.cpp index 64a1c72f9d9..2a5e32f0256 100644 --- a/scene/register_scene_types.cpp +++ b/scene/register_scene_types.cpp @@ -271,6 +271,7 @@ #include "scene/3d/visible_on_screen_notifier_3d.h" #include "scene/3d/voxel_gi.h" #include "scene/3d/world_environment.h" +#include "scene/3d/xr_face_modifier_3d.h" #include "scene/3d/xr_nodes.h" #include "scene/animation/root_motion_view.h" #include "scene/resources/environment.h" @@ -516,6 +517,7 @@ void register_scene_types() { GDREGISTER_CLASS(XRController3D); GDREGISTER_CLASS(XRAnchor3D); GDREGISTER_CLASS(XROrigin3D); + GDREGISTER_CLASS(XRFaceModifier3D); GDREGISTER_CLASS(MeshInstance3D); GDREGISTER_CLASS(OccluderInstance3D); GDREGISTER_ABSTRACT_CLASS(Occluder3D); diff --git a/servers/register_server_types.cpp b/servers/register_server_types.cpp index 0ee2984a8ca..b42e7bf9bb6 100644 --- a/servers/register_server_types.cpp +++ b/servers/register_server_types.cpp @@ -80,6 +80,7 @@ #include "text/text_server_dummy.h" #include "text/text_server_extension.h" #include "text_server.h" +#include "xr/xr_face_tracker.h" #include "xr/xr_interface.h" #include "xr/xr_interface_extension.h" #include "xr/xr_positional_tracker.h" @@ -189,6 +190,7 @@ void register_server_types() { GDREGISTER_CLASS(XRInterfaceExtension); // can't register this as virtual because we need a creation function for our extensions. GDREGISTER_CLASS(XRPose); GDREGISTER_CLASS(XRPositionalTracker); + GDREGISTER_CLASS(XRFaceTracker); GDREGISTER_CLASS(AudioStream); GDREGISTER_CLASS(AudioStreamPlayback); diff --git a/servers/xr/xr_face_tracker.cpp b/servers/xr/xr_face_tracker.cpp new file mode 100644 index 00000000000..a38ccfd527a --- /dev/null +++ b/servers/xr/xr_face_tracker.cpp @@ -0,0 +1,222 @@ +/**************************************************************************/ +/* xr_face_tracker.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "xr_face_tracker.h" + +void XRFaceTracker::_bind_methods() { + // Base Shapes + BIND_ENUM_CONSTANT(FT_EYE_LOOK_OUT_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_IN_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_UP_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_DOWN_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_OUT_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_IN_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_UP_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_LOOK_DOWN_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_CLOSED_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_CLOSED_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_SQUINT_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_SQUINT_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_WIDE_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_WIDE_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_DILATION_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_DILATION_LEFT); + BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT_RIGHT); + BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT_LEFT); + BIND_ENUM_CONSTANT(FT_BROW_PINCH_RIGHT); + BIND_ENUM_CONSTANT(FT_BROW_PINCH_LEFT); + BIND_ENUM_CONSTANT(FT_BROW_LOWERER_RIGHT); + BIND_ENUM_CONSTANT(FT_BROW_LOWERER_LEFT); + BIND_ENUM_CONSTANT(FT_BROW_INNER_UP_RIGHT); + BIND_ENUM_CONSTANT(FT_BROW_INNER_UP_LEFT); + BIND_ENUM_CONSTANT(FT_BROW_OUTER_UP_RIGHT); + BIND_ENUM_CONSTANT(FT_BROW_OUTER_UP_LEFT); + BIND_ENUM_CONSTANT(FT_NOSE_SNEER_RIGHT); + BIND_ENUM_CONSTANT(FT_NOSE_SNEER_LEFT); + BIND_ENUM_CONSTANT(FT_NASAL_DILATION_RIGHT); + BIND_ENUM_CONSTANT(FT_NASAL_DILATION_LEFT); + BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT_RIGHT); + BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT_LEFT); + BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT_RIGHT); + BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT_LEFT); + BIND_ENUM_CONSTANT(FT_CHEEK_PUFF_RIGHT); + BIND_ENUM_CONSTANT(FT_CHEEK_PUFF_LEFT); + BIND_ENUM_CONSTANT(FT_CHEEK_SUCK_RIGHT); + BIND_ENUM_CONSTANT(FT_CHEEK_SUCK_LEFT); + BIND_ENUM_CONSTANT(FT_JAW_OPEN); + BIND_ENUM_CONSTANT(FT_MOUTH_CLOSED); + BIND_ENUM_CONSTANT(FT_JAW_RIGHT); + BIND_ENUM_CONSTANT(FT_JAW_LEFT); + BIND_ENUM_CONSTANT(FT_JAW_FORWARD); + BIND_ENUM_CONSTANT(FT_JAW_BACKWARD); + BIND_ENUM_CONSTANT(FT_JAW_CLENCH); + BIND_ENUM_CONSTANT(FT_JAW_MANDIBLE_RAISE); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER_LEFT); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER_LEFT); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_CORNER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_CORNER_LEFT); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER_LEFT); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER_LEFT); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER_LEFT); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER_RIGHT); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_DEEPEN_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_DEEPEN_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_PULL_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_PULL_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_SLANT_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_SLANT_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_FROWN_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_FROWN_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_RAISER_UPPER); + BIND_ENUM_CONSTANT(FT_MOUTH_RAISER_LOWER); + BIND_ENUM_CONSTANT(FT_MOUTH_PRESS_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_PRESS_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER_LEFT); + BIND_ENUM_CONSTANT(FT_TONGUE_OUT); + BIND_ENUM_CONSTANT(FT_TONGUE_UP); + BIND_ENUM_CONSTANT(FT_TONGUE_DOWN); + BIND_ENUM_CONSTANT(FT_TONGUE_RIGHT); + BIND_ENUM_CONSTANT(FT_TONGUE_LEFT); + BIND_ENUM_CONSTANT(FT_TONGUE_ROLL); + BIND_ENUM_CONSTANT(FT_TONGUE_BLEND_DOWN); + BIND_ENUM_CONSTANT(FT_TONGUE_CURL_UP); + BIND_ENUM_CONSTANT(FT_TONGUE_SQUISH); + BIND_ENUM_CONSTANT(FT_TONGUE_FLAT); + BIND_ENUM_CONSTANT(FT_TONGUE_TWIST_RIGHT); + BIND_ENUM_CONSTANT(FT_TONGUE_TWIST_LEFT); + BIND_ENUM_CONSTANT(FT_SOFT_PALATE_CLOSE); + BIND_ENUM_CONSTANT(FT_THROAT_SWALLOW); + BIND_ENUM_CONSTANT(FT_NECK_FLEX_RIGHT); + BIND_ENUM_CONSTANT(FT_NECK_FLEX_LEFT); + // Blended Shapes + BIND_ENUM_CONSTANT(FT_EYE_CLOSED); + BIND_ENUM_CONSTANT(FT_EYE_WIDE); + BIND_ENUM_CONSTANT(FT_EYE_SQUINT); + BIND_ENUM_CONSTANT(FT_EYE_DILATION); + BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT); + BIND_ENUM_CONSTANT(FT_BROW_DOWN_RIGHT); + BIND_ENUM_CONSTANT(FT_BROW_DOWN_LEFT); + BIND_ENUM_CONSTANT(FT_BROW_DOWN); + BIND_ENUM_CONSTANT(FT_BROW_UP_RIGHT); + BIND_ENUM_CONSTANT(FT_BROW_UP_LEFT); + BIND_ENUM_CONSTANT(FT_BROW_UP); + BIND_ENUM_CONSTANT(FT_NOSE_SNEER); + BIND_ENUM_CONSTANT(FT_NASAL_DILATION); + BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT); + BIND_ENUM_CONSTANT(FT_CHEEK_PUFF); + BIND_ENUM_CONSTANT(FT_CHEEK_SUCK); + BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER); + BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER); + BIND_ENUM_CONSTANT(FT_LIP_SUCK); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER); + BIND_ENUM_CONSTANT(FT_LIP_FUNNEL); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER); + BIND_ENUM_CONSTANT(FT_LIP_PUCKER); + BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP); + BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN); + BIND_ENUM_CONSTANT(FT_MOUTH_OPEN); + BIND_ENUM_CONSTANT(FT_MOUTH_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_SMILE_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_SMILE_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_SMILE); + BIND_ENUM_CONSTANT(FT_MOUTH_SAD_RIGHT); + BIND_ENUM_CONSTANT(FT_MOUTH_SAD_LEFT); + BIND_ENUM_CONSTANT(FT_MOUTH_SAD); + BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH); + BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE); + BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER); + BIND_ENUM_CONSTANT(FT_MOUTH_PRESS); + BIND_ENUM_CONSTANT(FT_MAX); + + ClassDB::bind_method(D_METHOD("get_blend_shape", "blend_shape"), &XRFaceTracker::get_blend_shape); + ClassDB::bind_method(D_METHOD("set_blend_shape", "blend_shape", "weight"), &XRFaceTracker::set_blend_shape); + + ClassDB::bind_method(D_METHOD("get_blend_shapes"), &XRFaceTracker::get_blend_shapes); + ClassDB::bind_method(D_METHOD("set_blend_shapes", "weights"), &XRFaceTracker::set_blend_shapes); + ADD_PROPERTY(PropertyInfo(Variant::PACKED_FLOAT32_ARRAY, "blend_shapes"), "set_blend_shapes", "get_blend_shapes"); + ADD_PROPERTY_DEFAULT("blend_shapes", PackedFloat32Array()); // To prevent ludicrously large default values. +} + +float XRFaceTracker::get_blend_shape(BlendShapeEntry p_blend_shape) const { + // Fail if the blend shape index is out of range. + ERR_FAIL_INDEX_V(p_blend_shape, FT_MAX, 0.0f); + + // Return the blend shape value. + return blend_shape_values[p_blend_shape]; +} + +void XRFaceTracker::set_blend_shape(BlendShapeEntry p_blend_shape, float p_value) { + // Fail if the blend shape index is out of range. + ERR_FAIL_INDEX(p_blend_shape, FT_MAX); + + // Save the new blend shape value. + blend_shape_values[p_blend_shape] = p_value; +} + +PackedFloat32Array XRFaceTracker::get_blend_shapes() const { + // Create a packed float32 array and copy the blend shape values into it. + PackedFloat32Array data; + data.resize(FT_MAX); + memcpy(data.ptrw(), blend_shape_values, sizeof(blend_shape_values)); + + // Return the blend shape array. + return data; +} + +void XRFaceTracker::set_blend_shapes(const PackedFloat32Array &p_blend_shapes) { + // Fail if the blend shape array is not the correct size. + ERR_FAIL_COND(p_blend_shapes.size() != FT_MAX); + + // Copy the blend shape values into the blend shape array. + memcpy(blend_shape_values, p_blend_shapes.ptr(), sizeof(blend_shape_values)); +} diff --git a/servers/xr/xr_face_tracker.h b/servers/xr/xr_face_tracker.h new file mode 100644 index 00000000000..b9f553cba69 --- /dev/null +++ b/servers/xr/xr_face_tracker.h @@ -0,0 +1,213 @@ +/**************************************************************************/ +/* xr_face_tracker.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef XR_FACE_TRACKER_H +#define XR_FACE_TRACKER_H + +#include "core/object/ref_counted.h" + +/** + The XRFaceTracker class provides face blend shape weights. + + The supported blend shapes are based on the Unified Expressions + standard, and as such have a well defined mapping to ARKit, SRanipal, + and Meta Movement standards. + */ + +class XRFaceTracker : public RefCounted { + GDCLASS(XRFaceTracker, RefCounted); + _THREAD_SAFE_CLASS_ + +public: + enum BlendShapeEntry { + // Base Shapes + FT_EYE_LOOK_OUT_RIGHT, // Right eye looks outwards. + FT_EYE_LOOK_IN_RIGHT, // Right eye looks inwards. + FT_EYE_LOOK_UP_RIGHT, // Right eye looks upwards. + FT_EYE_LOOK_DOWN_RIGHT, // Right eye looks downwards. + FT_EYE_LOOK_OUT_LEFT, // Left eye looks outwards. + FT_EYE_LOOK_IN_LEFT, // Left eye looks inwards. + FT_EYE_LOOK_UP_LEFT, // Left eye looks upwards. + FT_EYE_LOOK_DOWN_LEFT, // Left eye looks downwards. + FT_EYE_CLOSED_RIGHT, // Closes the right eyelid. + FT_EYE_CLOSED_LEFT, // Closes the left eyelid. + FT_EYE_SQUINT_RIGHT, // Squeezes the right eye socket muscles. + FT_EYE_SQUINT_LEFT, // Squeezes the left eye socket muscles. + FT_EYE_WIDE_RIGHT, // Right eyelid widens beyond relaxed. + FT_EYE_WIDE_LEFT, // Left eyelid widens beyond relaxed. + FT_EYE_DILATION_RIGHT, // Dilates the right eye pupil. + FT_EYE_DILATION_LEFT, // Dilates the left eye pupil. + FT_EYE_CONSTRICT_RIGHT, // Constricts the right eye pupil. + FT_EYE_CONSTRICT_LEFT, // Constricts the left eye pupil. + FT_BROW_PINCH_RIGHT, // Right eyebrow pinches in. + FT_BROW_PINCH_LEFT, // Left eyebrow pinches in. + FT_BROW_LOWERER_RIGHT, // Outer right eyebrow pulls down. + FT_BROW_LOWERER_LEFT, // Outer left eyebrow pulls down. + FT_BROW_INNER_UP_RIGHT, // Inner right eyebrow pulls up. + FT_BROW_INNER_UP_LEFT, // Inner left eyebrow pulls up. + FT_BROW_OUTER_UP_RIGHT, // Outer right eyebrow pulls up. + FT_BROW_OUTER_UP_LEFT, // Outer left eyebrow pulls up. + FT_NOSE_SNEER_RIGHT, // Right side face sneers. + FT_NOSE_SNEER_LEFT, // Left side face sneers. + FT_NASAL_DILATION_RIGHT, // Right side nose canal dilates. + FT_NASAL_DILATION_LEFT, // Left side nose canal dilates. + FT_NASAL_CONSTRICT_RIGHT, // Right side nose canal constricts. + FT_NASAL_CONSTRICT_LEFT, // Left side nose canal constricts. + FT_CHEEK_SQUINT_RIGHT, // Raises the right side cheek. + FT_CHEEK_SQUINT_LEFT, // Raises the left side cheek. + FT_CHEEK_PUFF_RIGHT, // Puffs the right side cheek. + FT_CHEEK_PUFF_LEFT, // Puffs the left side cheek. + FT_CHEEK_SUCK_RIGHT, // Sucks in the right side cheek. + FT_CHEEK_SUCK_LEFT, // Sucks in the left side cheek. + FT_JAW_OPEN, // Opens jawbone. + FT_MOUTH_CLOSED, // Closes the mouth. + FT_JAW_RIGHT, // Pushes jawbone right. + FT_JAW_LEFT, // Pushes jawbone left. + FT_JAW_FORWARD, // Pushes jawbone forward. + FT_JAW_BACKWARD, // Pushes jawbone backward. + FT_JAW_CLENCH, // Flexes jaw muscles. + FT_JAW_MANDIBLE_RAISE, // Raises the jawbone. + FT_LIP_SUCK_UPPER_RIGHT, // Upper right lip part tucks in the mouth. + FT_LIP_SUCK_UPPER_LEFT, // Upper left lip part tucks in the mouth. + FT_LIP_SUCK_LOWER_RIGHT, // Lower right lip part tucks in the mouth. + FT_LIP_SUCK_LOWER_LEFT, // Lower left lip part tucks in the mouth. + FT_LIP_SUCK_CORNER_RIGHT, // Right lip corner folds into the mouth. + FT_LIP_SUCK_CORNER_LEFT, // Left lip corner folds into the mouth. + FT_LIP_FUNNEL_UPPER_RIGHT, // Upper right lip part pushes into a funnel. + FT_LIP_FUNNEL_UPPER_LEFT, // Upper left lip part pushes into a funnel. + FT_LIP_FUNNEL_LOWER_RIGHT, // Lower right lip part pushes into a funnel. + FT_LIP_FUNNEL_LOWER_LEFT, // Lower left lip part pushes into a funnel. + FT_LIP_PUCKER_UPPER_RIGHT, // Upper right lip part pushes outwards. + FT_LIP_PUCKER_UPPER_LEFT, // Upper left lip part pushes outwards. + FT_LIP_PUCKER_LOWER_RIGHT, // Lower right lip part pushes outwards. + FT_LIP_PUCKER_LOWER_LEFT, // Lower left lip part pushes outwards. + FT_MOUTH_UPPER_UP_RIGHT, // Upper right part of the lip pulls up. + FT_MOUTH_UPPER_UP_LEFT, // Upper left part of the lip pulls up. + FT_MOUTH_LOWER_DOWN_RIGHT, // Lower right part of the lip pulls up. + FT_MOUTH_LOWER_DOWN_LEFT, // Lower left part of the lip pulls up. + FT_MOUTH_UPPER_DEEPEN_RIGHT, // Upper right lip part pushes in the cheek. + FT_MOUTH_UPPER_DEEPEN_LEFT, // Upper left lip part pushes in the cheek. + FT_MOUTH_UPPER_RIGHT, // Moves upper lip right. + FT_MOUTH_UPPER_LEFT, // Moves upper lip left. + FT_MOUTH_LOWER_RIGHT, // Moves lower lip right. + FT_MOUTH_LOWER_LEFT, // Moves lower lip left. + FT_MOUTH_CORNER_PULL_RIGHT, // Right lip corner pulls diagonally up and out. + FT_MOUTH_CORNER_PULL_LEFT, // Left lip corner pulls diagonally up and out. + FT_MOUTH_CORNER_SLANT_RIGHT, // Right corner lip slants up. + FT_MOUTH_CORNER_SLANT_LEFT, // Left corner lip slants up. + FT_MOUTH_FROWN_RIGHT, // Right corner lip pulls down. + FT_MOUTH_FROWN_LEFT, // Left corner lip pulls down. + FT_MOUTH_STRETCH_RIGHT, // Mouth corner lip pulls out and down. + FT_MOUTH_STRETCH_LEFT, // Mouth corner lip pulls out and down. + FT_MOUTH_DIMPLE_RIGHT, // Right lip corner is pushed backwards. + FT_MOUTH_DIMPLE_LEFT, // Left lip corner is pushed backwards. + FT_MOUTH_RAISER_UPPER, // Raises and slightly pushes out the upper mouth. + FT_MOUTH_RAISER_LOWER, // Raises and slightly pushes out the lower mouth. + FT_MOUTH_PRESS_RIGHT, // Right side lips press and flatten together vertically. + FT_MOUTH_PRESS_LEFT, // Left side lips press and flatten together vertically. + FT_MOUTH_TIGHTENER_RIGHT, // Right side lips squeeze together horizontally. + FT_MOUTH_TIGHTENER_LEFT, // Left side lips squeeze together horizontally. + FT_TONGUE_OUT, // Tongue visibly sticks out of the mouth. + FT_TONGUE_UP, // Tongue points upwards. + FT_TONGUE_DOWN, // Tongue points downwards. + FT_TONGUE_RIGHT, // Tongue points right. + FT_TONGUE_LEFT, // Tongue points left. + FT_TONGUE_ROLL, // Sides of the tongue funnel, creating a roll. + FT_TONGUE_BLEND_DOWN, // Tongue arches up then down inside the mouth. + FT_TONGUE_CURL_UP, // Tongue arches down then up inside the mouth. + FT_TONGUE_SQUISH, // Tongue squishes together and thickens. + FT_TONGUE_FLAT, // Tongue flattens and thins out. + FT_TONGUE_TWIST_RIGHT, // Tongue tip rotates clockwise, with the rest following gradually. + FT_TONGUE_TWIST_LEFT, // Tongue tip rotates counter-clockwise, with the rest following gradually. + FT_SOFT_PALATE_CLOSE, // Inner mouth throat closes. + FT_THROAT_SWALLOW, // The Adam's apple visibly swallows. + FT_NECK_FLEX_RIGHT, // Right side neck visibly flexes. + FT_NECK_FLEX_LEFT, // Left side neck visibly flexes. + // Blended Shapes + FT_EYE_CLOSED, // Closes both eye lids. + FT_EYE_WIDE, // Widens both eye lids. + FT_EYE_SQUINT, // Squints both eye lids. + FT_EYE_DILATION, // Dilates both pupils. + FT_EYE_CONSTRICT, // Constricts both pupils. + FT_BROW_DOWN_RIGHT, // Pulls the right eyebrow down and in. + FT_BROW_DOWN_LEFT, // Pulls the left eyebrow down and in. + FT_BROW_DOWN, // Pulls both eyebrows down and in. + FT_BROW_UP_RIGHT, // Right brow appears worried. + FT_BROW_UP_LEFT, // Left brow appears worried. + FT_BROW_UP, // Both brows appear worried. + FT_NOSE_SNEER, // Entire face sneers. + FT_NASAL_DILATION, // Both nose canals dilate. + FT_NASAL_CONSTRICT, // Both nose canals constrict. + FT_CHEEK_PUFF, // Puffs both cheeks. + FT_CHEEK_SUCK, // Sucks in both cheeks. + FT_CHEEK_SQUINT, // Raises both cheeks. + FT_LIP_SUCK_UPPER, // Tucks in the upper lips. + FT_LIP_SUCK_LOWER, // Tucks in the lower lips. + FT_LIP_SUCK, // Tucks in both lips. + FT_LIP_FUNNEL_UPPER, // Funnels in the upper lips. + FT_LIP_FUNNEL_LOWER, // Funnels in the lower lips. + FT_LIP_FUNNEL, // Funnels in both lips. + FT_LIP_PUCKER_UPPER, // Upper lip part pushes outwards. + FT_LIP_PUCKER_LOWER, // Lower lip part pushes outwards. + FT_LIP_PUCKER, // Lips push outwards. + FT_MOUTH_UPPER_UP, // Raises the upper lips. + FT_MOUTH_LOWER_DOWN, // Lowers the lower lips. + FT_MOUTH_OPEN, // Mouth opens, revealing teeth. + FT_MOUTH_RIGHT, // Moves mouth right. + FT_MOUTH_LEFT, // Moves mouth left. + FT_MOUTH_SMILE_RIGHT, // Right side of the mouth smiles. + FT_MOUTH_SMILE_LEFT, // Left side of the mouth smiles. + FT_MOUTH_SMILE, // Mouth expresses a smile. + FT_MOUTH_SAD_RIGHT, // Right side of the mouth expresses sadness. + FT_MOUTH_SAD_LEFT, // Left side of the mouth expresses sadness. + FT_MOUTH_SAD, // Mouth expresses sadness. + FT_MOUTH_STRETCH, // Mouth stretches. + FT_MOUTH_DIMPLE, // Lip corners dimple. + FT_MOUTH_TIGHTENER, // Mouth tightens. + FT_MOUTH_PRESS, // Mouth presses together. + FT_MAX // Maximum blend shape. + }; + + float get_blend_shape(BlendShapeEntry p_blend_shape) const; + void set_blend_shape(BlendShapeEntry p_blend_shape, float p_value); + + PackedFloat32Array get_blend_shapes() const; + void set_blend_shapes(const PackedFloat32Array &p_blend_shapes); + +protected: + static void _bind_methods(); + +private: + float blend_shape_values[FT_MAX] = {}; +}; + +VARIANT_ENUM_CAST(XRFaceTracker::BlendShapeEntry); + +#endif // XR_FACE_TRACKER_H diff --git a/servers/xr_server.cpp b/servers/xr_server.cpp index e7f644d53fb..b3bb0a3702a 100644 --- a/servers/xr_server.cpp +++ b/servers/xr_server.cpp @@ -30,6 +30,7 @@ #include "xr_server.h" #include "core/config/project_settings.h" +#include "xr/xr_face_tracker.h" #include "xr/xr_interface.h" #include "xr/xr_positional_tracker.h" @@ -74,6 +75,11 @@ void XRServer::_bind_methods() { ClassDB::bind_method(D_METHOD("get_trackers", "tracker_types"), &XRServer::get_trackers); ClassDB::bind_method(D_METHOD("get_tracker", "tracker_name"), &XRServer::get_tracker); + ClassDB::bind_method(D_METHOD("add_face_tracker", "tracker_name", "face_tracker"), &XRServer::add_face_tracker); + ClassDB::bind_method(D_METHOD("remove_face_tracker", "tracker_name"), &XRServer::remove_face_tracker); + ClassDB::bind_method(D_METHOD("get_face_trackers"), &XRServer::get_face_trackers); + ClassDB::bind_method(D_METHOD("get_face_tracker", "tracker_name"), &XRServer::get_face_tracker); + ClassDB::bind_method(D_METHOD("get_primary_interface"), &XRServer::get_primary_interface); ClassDB::bind_method(D_METHOD("set_primary_interface", "interface"), &XRServer::set_primary_interface); @@ -97,6 +103,10 @@ void XRServer::_bind_methods() { ADD_SIGNAL(MethodInfo("tracker_added", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type"))); ADD_SIGNAL(MethodInfo("tracker_updated", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type"))); ADD_SIGNAL(MethodInfo("tracker_removed", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type"))); + + ADD_SIGNAL(MethodInfo("face_tracker_added", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::OBJECT, "face_tracker", PROPERTY_HINT_RESOURCE_TYPE, "XRFaceTracker"))); + ADD_SIGNAL(MethodInfo("face_tracker_updated", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::OBJECT, "face_tracker", PROPERTY_HINT_RESOURCE_TYPE, "XRFaceTracker"))); + ADD_SIGNAL(MethodInfo("face_tracker_removed", PropertyInfo(Variant::STRING_NAME, "tracker_name"))); }; double XRServer::get_world_scale() const { @@ -352,6 +362,44 @@ PackedStringArray XRServer::get_suggested_pose_names(const StringName &p_tracker return arr; } +void XRServer::add_face_tracker(const StringName &p_tracker_name, Ref p_face_tracker) { + ERR_FAIL_COND(p_face_tracker.is_null()); + + if (!face_trackers.has(p_tracker_name)) { + // We don't have a tracker with this name, we're going to add it. + face_trackers[p_tracker_name] = p_face_tracker; + emit_signal(SNAME("face_tracker_added"), p_tracker_name, p_face_tracker); + } else if (face_trackers[p_tracker_name] != p_face_tracker) { + // We already have a tracker with this name, we're going to replace it. + face_trackers[p_tracker_name] = p_face_tracker; + emit_signal(SNAME("face_tracker_updated"), p_tracker_name, p_face_tracker); + } +} + +void XRServer::remove_face_tracker(const StringName &p_tracker_name) { + // Skip if no face tracker is found. + if (!face_trackers.has(p_tracker_name)) { + return; + } + + // Send the removed signal, then remove the face tracker. + emit_signal(SNAME("face_tracker_removed"), p_tracker_name); + face_trackers.erase(p_tracker_name); +} + +Dictionary XRServer::get_face_trackers() const { + return face_trackers; +} + +Ref XRServer::get_face_tracker(const StringName &p_tracker_name) const { + // Skip if no tracker is found. + if (!face_trackers.has(p_tracker_name)) { + return Ref(); + } + + return face_trackers[p_tracker_name]; +} + void XRServer::_process() { // called from our main game loop before we handle physics and game logic // note that we can have multiple interfaces active if we have interfaces that purely handle tracking diff --git a/servers/xr_server.h b/servers/xr_server.h index fe59fc22cb7..0a4e020a1f6 100644 --- a/servers/xr_server.h +++ b/servers/xr_server.h @@ -39,6 +39,7 @@ class XRInterface; class XRPositionalTracker; +class XRFaceTracker; /** The XR server is a singleton object that gives access to the various @@ -86,6 +87,8 @@ private: Vector> interfaces; Dictionary trackers; + Dictionary face_trackers; + Ref primary_interface; /* we'll identify one interface as primary, this will be used by our viewports */ double world_scale; /* scale by which we multiply our tracker positions */ @@ -183,6 +186,14 @@ public: PackedStringArray get_suggested_pose_names(const StringName &p_tracker_name) const; // Q: Should we add get_suggested_input_names and get_suggested_haptic_names even though we don't use them for the IDE? + /* + Face trackers are objects that expose the tracked blend shapes of a face. + */ + void add_face_tracker(const StringName &p_tracker_name, Ref p_face_tracker); + void remove_face_tracker(const StringName &p_tracker_name); + Dictionary get_face_trackers() const; + Ref get_face_tracker(const StringName &p_tracker_name) const; + // Process is called before we handle our physics process and game process. This is where our interfaces will update controller data and such. void _process();