diff --git a/doc/classes/XRFaceModifier3D.xml b/doc/classes/XRFaceModifier3D.xml
new file mode 100644
index 00000000000..7a60e6db346
--- /dev/null
+++ b/doc/classes/XRFaceModifier3D.xml
@@ -0,0 +1,22 @@
+
+
+
+ A node for driving standard face meshes from [XRFaceTracker] weights.
+
+
+ This node applies weights from a [XRFaceTracker] to a mesh with supporting face blend shapes.
+ The [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/unified-blendshapes]Unified Expressions[/url] blend shapes are supported, as well as ARKit and SRanipal blend shapes.
+ The node attempts to identify blend shapes based on name matching. Blend shapes should match the names listed in the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/compatibility/overview]Unified Expressions Compatibility[/url] chart.
+
+
+ $DOCS_URL/tutorials/xr/index.html
+
+
+
+ The [XRFaceTracker] path.
+
+
+ The [NodePath] of the face [MeshInstance3D].
+
+
+
diff --git a/doc/classes/XRFaceTracker.xml b/doc/classes/XRFaceTracker.xml
new file mode 100644
index 00000000000..0212cc8fffe
--- /dev/null
+++ b/doc/classes/XRFaceTracker.xml
@@ -0,0 +1,469 @@
+
+
+
+ A tracked face.
+
+
+ An instance of this object represents a tracked face and its corresponding blend shapes. The blend shapes come from the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/unified-blendshapes]Unified Expressions[/url] standard, and contain extended details and visuals for each blend shape. Additionally the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/compatibility/overview]Tracking Standard Comparison[/url] page documents the relationship between Unified Expressions and other standards.
+ As face trackers are turned on they are registered with the [XRServer].
+
+
+ $DOCS_URL/tutorials/xr/index.html
+
+
+
+
+
+
+ Returns the requested face blend shape weight.
+
+
+
+
+
+
+
+ Sets a face blend shape weight.
+
+
+
+
+
+ The array of face blend shape weights with indices corresponding to the [enum BlendShapeEntry] enum.
+
+
+
+
+ Right eye looks outwards.
+
+
+ Right eye looks inwards.
+
+
+ Right eye looks upwards.
+
+
+ Right eye looks downwards.
+
+
+ Left eye looks outwards.
+
+
+ Left eye looks inwards.
+
+
+ Left eye looks upwards.
+
+
+ Left eye looks downwards.
+
+
+ Closes the right eyelid.
+
+
+ Closes the left eyelid.
+
+
+ Squeezes the right eye socket muscles.
+
+
+ Squeezes the left eye socket muscles.
+
+
+ Right eyelid widens beyond relaxed.
+
+
+ Left eyelid widens beyond relaxed.
+
+
+ Dilates the right eye pupil.
+
+
+ Dilates the left eye pupil.
+
+
+ Constricts the right eye pupil.
+
+
+ Constricts the left eye pupil.
+
+
+ Right eyebrow pinches in.
+
+
+ Left eyebrow pinches in.
+
+
+ Outer right eyebrow pulls down.
+
+
+ Outer left eyebrow pulls down.
+
+
+ Inner right eyebrow pulls up.
+
+
+ Inner left eyebrow pulls up.
+
+
+ Outer right eyebrow pulls up.
+
+
+ Outer left eyebrow pulls up.
+
+
+ Right side face sneers.
+
+
+ Left side face sneers.
+
+
+ Right side nose canal dilates.
+
+
+ Left side nose canal dilates.
+
+
+ Right side nose canal constricts.
+
+
+ Left side nose canal constricts.
+
+
+ Raises the right side cheek.
+
+
+ Raises the left side cheek.
+
+
+ Puffs the right side cheek.
+
+
+ Puffs the left side cheek.
+
+
+ Sucks in the right side cheek.
+
+
+ Sucks in the left side cheek.
+
+
+ Opens jawbone.
+
+
+ Closes the mouth.
+
+
+ Pushes jawbone right.
+
+
+ Pushes jawbone left.
+
+
+ Pushes jawbone forward.
+
+
+ Pushes jawbone backward.
+
+
+ Flexes jaw muscles.
+
+
+ Raises the jawbone.
+
+
+ Upper right lip part tucks in the mouth.
+
+
+ Upper left lip part tucks in the mouth.
+
+
+ Lower right lip part tucks in the mouth.
+
+
+ Lower left lip part tucks in the mouth.
+
+
+ Right lip corner folds into the mouth.
+
+
+ Left lip corner folds into the mouth.
+
+
+ Upper right lip part pushes into a funnel.
+
+
+ Upper left lip part pushes into a funnel.
+
+
+ Lower right lip part pushes into a funnel.
+
+
+ Lower left lip part pushes into a funnel.
+
+
+ Upper right lip part pushes outwards.
+
+
+ Upper left lip part pushes outwards.
+
+
+ Lower right lip part pushes outwards.
+
+
+ Lower left lip part pushes outwards.
+
+
+ Upper right part of the lip pulls up.
+
+
+ Upper left part of the lip pulls up.
+
+
+ Lower right part of the lip pulls up.
+
+
+ Lower left part of the lip pulls up.
+
+
+ Upper right lip part pushes in the cheek.
+
+
+ Upper left lip part pushes in the cheek.
+
+
+ Moves upper lip right.
+
+
+ Moves upper lip left.
+
+
+ Moves lower lip right.
+
+
+ Moves lower lip left.
+
+
+ Right lip corner pulls diagonally up and out.
+
+
+ Left lip corner pulls diagonally up and out.
+
+
+ Right corner lip slants up.
+
+
+ Left corner lip slants up.
+
+
+ Right corner lip pulls down.
+
+
+ Left corner lip pulls down.
+
+
+ Mouth corner lip pulls out and down.
+
+
+ Mouth corner lip pulls out and down.
+
+
+ Right lip corner is pushed backwards.
+
+
+ Left lip corner is pushed backwards.
+
+
+ Raises and slightly pushes out the upper mouth.
+
+
+ Raises and slightly pushes out the lower mouth.
+
+
+ Right side lips press and flatten together vertically.
+
+
+ Left side lips press and flatten together vertically.
+
+
+ Right side lips squeeze together horizontally.
+
+
+ Left side lips squeeze together horizontally.
+
+
+ Tongue visibly sticks out of the mouth.
+
+
+ Tongue points upwards.
+
+
+ Tongue points downwards.
+
+
+ Tongue points right.
+
+
+ Tongue points left.
+
+
+ Sides of the tongue funnel, creating a roll.
+
+
+ Tongue arches up then down inside the mouth.
+
+
+ Tongue arches down then up inside the mouth.
+
+
+ Tongue squishes together and thickens.
+
+
+ Tongue flattens and thins out.
+
+
+ Tongue tip rotates clockwise, with the rest following gradually.
+
+
+ Tongue tip rotates counter-clockwise, with the rest following gradually.
+
+
+ Inner mouth throat closes.
+
+
+ The Adam's apple visibly swallows.
+
+
+ Right side neck visibly flexes.
+
+
+ Left side neck visibly flexes.
+
+
+ Closes both eye lids.
+
+
+ Widens both eye lids.
+
+
+ Squints both eye lids.
+
+
+ Dilates both pupils.
+
+
+ Constricts both pupils.
+
+
+ Pulls the right eyebrow down and in.
+
+
+ Pulls the left eyebrow down and in.
+
+
+ Pulls both eyebrows down and in.
+
+
+ Right brow appears worried.
+
+
+ Left brow appears worried.
+
+
+ Both brows appear worried.
+
+
+ Entire face sneers.
+
+
+ Both nose canals dilate.
+
+
+ Both nose canals constrict.
+
+
+ Puffs both cheeks.
+
+
+ Sucks in both cheeks.
+
+
+ Raises both cheeks.
+
+
+ Tucks in the upper lips.
+
+
+ Tucks in the lower lips.
+
+
+ Tucks in both lips.
+
+
+ Funnels in the upper lips.
+
+
+ Funnels in the lower lips.
+
+
+ Funnels in both lips.
+
+
+ Upper lip part pushes outwards.
+
+
+ Lower lip part pushes outwards.
+
+
+ Lips push outwards.
+
+
+ Raises the upper lips.
+
+
+ Lowers the lower lips.
+
+
+ Mouth opens, revealing teeth.
+
+
+ Moves mouth right.
+
+
+ Moves mouth left.
+
+
+ Right side of the mouth smiles.
+
+
+ Left side of the mouth smiles.
+
+
+ Mouth expresses a smile.
+
+
+ Right side of the mouth expresses sadness.
+
+
+ Left side of the mouth expresses sadness.
+
+
+ Mouth expresses sadness.
+
+
+ Mouth stretches.
+
+
+ Lip corners dimple.
+
+
+ Mouth tightens.
+
+
+ Mouth presses together.
+
+
+ Represents the size of the [enum BlendShapeEntry] enum.
+
+
+
diff --git a/doc/classes/XRServer.xml b/doc/classes/XRServer.xml
index 2f106af340a..f98c1d66a48 100644
--- a/doc/classes/XRServer.xml
+++ b/doc/classes/XRServer.xml
@@ -10,6 +10,14 @@
$DOCS_URL/tutorials/xr/index.html
+
+
+
+
+
+ Registers a new [XRFaceTracker] that tracks the blend shapes of a face.
+
+
@@ -50,6 +58,19 @@
Finds an interface by its [param name]. For example, if your project uses capabilities of an AR/VR platform, you can find the interface for that platform by name and initialize it.
+
+
+
+
+ Returns the [XRFaceTracker] with the given tracker name.
+
+
+
+
+
+ Returns a dictionary of the registered face trackers. Each element of the dictionary is a tracker name mapping to the [XRFaceTracker] instance.
+
+
@@ -95,6 +116,13 @@
Returns a dictionary of trackers for [param tracker_types].
+
+
+
+
+ Removes a registered [XRFaceTracker].
+
+
@@ -123,6 +151,26 @@
+
+
+
+
+ Emitted when a new face tracker is added.
+
+
+
+
+
+ Emitted when a face tracker is removed.
+
+
+
+
+
+
+ Emitted when an existing face tracker is updated.
+
+
diff --git a/scene/3d/xr_face_modifier_3d.cpp b/scene/3d/xr_face_modifier_3d.cpp
new file mode 100644
index 00000000000..be92a587b0d
--- /dev/null
+++ b/scene/3d/xr_face_modifier_3d.cpp
@@ -0,0 +1,615 @@
+/**************************************************************************/
+/* xr_face_modifier_3d.cpp */
+/**************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* https://godotengine.org */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/**************************************************************************/
+
+#include "xr_face_modifier_3d.h"
+
+#include "servers/xr/xr_face_tracker.h"
+#include "servers/xr_server.h"
+
+// This method takes the name of a mesh blend shape and returns the
+// corresponding XRFaceTracker blend shape. If no match is
+// found then the function returns -1.
+static int find_face_blend_shape(const StringName &p_name) {
+ // Entry for blend shape name table.
+ struct blend_map_entry {
+ int blend;
+ const char *name[4];
+ };
+
+ // Table of blend shape names.
+ //
+ // This table consists of the XRFaceTracker blend shape and
+ // the corresponding names (lowercase and no underscore) of:
+ // - The Unified Expression blend shape name.
+ // - The ARKit blend shape name (if present and different).
+ // - The SRanipal blend shape name (if present and different).
+ // - The Meta blend shape name (if present and different).
+ static constexpr blend_map_entry blend_map[] = {
+ { XRFaceTracker::FT_EYE_LOOK_OUT_RIGHT,
+ { "eyelookoutright", "eyerightright", "eyeslookoutr" } },
+ { XRFaceTracker::FT_EYE_LOOK_IN_RIGHT,
+ { "eyelookinright", "eyerightleft", "eyeslookinr" } },
+ { XRFaceTracker::FT_EYE_LOOK_UP_RIGHT,
+ { "eyelookupright", "eyerightlookup", "eyeslookupr" } },
+ { XRFaceTracker::FT_EYE_LOOK_DOWN_RIGHT,
+ { "eyelookdownright", "eyerightlookdown", "eyeslookdownr" } },
+ { XRFaceTracker::FT_EYE_LOOK_OUT_LEFT,
+ { "eyelookoutleft", "eyeleftleft", "eyeslookoutl" } },
+ { XRFaceTracker::FT_EYE_LOOK_IN_LEFT,
+ { "eyelookinleft", "eyeleftright", "eyeslookinl" } },
+ { XRFaceTracker::FT_EYE_LOOK_UP_LEFT,
+ { "eyelookupleft", "eyeleftlookup", "eyeslookupl" } },
+ { XRFaceTracker::FT_EYE_LOOK_DOWN_LEFT,
+ { "eyelookdownleft", "eyeleftlookdown", "eyeslookdownl" } },
+ { XRFaceTracker::FT_EYE_CLOSED_RIGHT,
+ { "eyeclosedright", "eyeblinkright", "eyerightblink", "eyesclosedr" } },
+ { XRFaceTracker::FT_EYE_CLOSED_LEFT,
+ { "eyeclosedleft", "eyeblinkleft", "eyeleftblink", "eyesclosedl" } },
+ { XRFaceTracker::FT_EYE_SQUINT_RIGHT,
+ { "eyesquintright", "eyessquintr" } },
+ { XRFaceTracker::FT_EYE_SQUINT_LEFT,
+ { "eyesquintleft", "eyessquintl" } },
+ { XRFaceTracker::FT_EYE_WIDE_RIGHT,
+ { "eyewideright", "eyerightwide", "eyeswidenr" } },
+ { XRFaceTracker::FT_EYE_WIDE_LEFT,
+ { "eyewideleft", "eyeleftwide", "eyeswidenl" } },
+ { XRFaceTracker::FT_EYE_DILATION_RIGHT,
+ { "eyedilationright", "eyerightdilation" } },
+ { XRFaceTracker::FT_EYE_DILATION_LEFT,
+ { "eyedilationleft", "eyeleftdilation" } },
+ { XRFaceTracker::FT_EYE_CONSTRICT_RIGHT,
+ { "eyeconstrictright", "eyerightconstrict" } },
+ { XRFaceTracker::FT_EYE_CONSTRICT_LEFT,
+ { "eyeconstrictleft", "eyeleftconstrict" } },
+ { XRFaceTracker::FT_BROW_PINCH_RIGHT,
+ { "browpinchright" } },
+ { XRFaceTracker::FT_BROW_PINCH_LEFT,
+ { "browpinchleft" } },
+ { XRFaceTracker::FT_BROW_LOWERER_RIGHT,
+ { "browlowererright" } },
+ { XRFaceTracker::FT_BROW_LOWERER_LEFT,
+ { "browlowererleft" } },
+ { XRFaceTracker::FT_BROW_INNER_UP_RIGHT,
+ { "browinnerupright", "innerbrowraiserr" } },
+ { XRFaceTracker::FT_BROW_INNER_UP_LEFT,
+ { "browinnerupleft", "innerbrowraiserl" } },
+ { XRFaceTracker::FT_BROW_OUTER_UP_RIGHT,
+ { "browouterupright", "outerbrowraiserr" } },
+ { XRFaceTracker::FT_BROW_OUTER_UP_LEFT,
+ { "browouterupleft", "outerbrowraiserl" } },
+ { XRFaceTracker::FT_NOSE_SNEER_RIGHT,
+ { "nosesneerright", "nosewrinklerr" } },
+ { XRFaceTracker::FT_NOSE_SNEER_LEFT,
+ { "nosesneerleft", "nosewrinklerl" } },
+ { XRFaceTracker::FT_NASAL_DILATION_RIGHT,
+ { "nasaldilationright" } },
+ { XRFaceTracker::FT_NASAL_DILATION_LEFT,
+ { "nasaldilationleft" } },
+ { XRFaceTracker::FT_NASAL_CONSTRICT_RIGHT,
+ { "nasalconstrictright" } },
+ { XRFaceTracker::FT_NASAL_CONSTRICT_LEFT,
+ { "nasalconstrictleft" } },
+ { XRFaceTracker::FT_CHEEK_SQUINT_RIGHT,
+ { "cheeksquintright", "cheekraiserr" } },
+ { XRFaceTracker::FT_CHEEK_SQUINT_LEFT,
+ { "cheeksquintleft", "cheekraiserl" } },
+ { XRFaceTracker::FT_CHEEK_PUFF_RIGHT,
+ { "cheekpuffright", "cheekpuffr" } },
+ { XRFaceTracker::FT_CHEEK_PUFF_LEFT,
+ { "cheekpuffleft", "cheekpuffl" } },
+ { XRFaceTracker::FT_CHEEK_SUCK_RIGHT,
+ { "cheeksuckright", "cheeksuckr" } },
+ { XRFaceTracker::FT_CHEEK_SUCK_LEFT,
+ { "cheeksuckleft", "cheeksuckl" } },
+ { XRFaceTracker::FT_JAW_OPEN,
+ { "jawopen", "jawdrop" } },
+ { XRFaceTracker::FT_MOUTH_CLOSED,
+ { "mouthclosed", "mouthclose", "mouthapeshape", "lipstoward" } },
+ { XRFaceTracker::FT_JAW_RIGHT,
+ { "jawright", "jawsidewaysright" } },
+ { XRFaceTracker::FT_JAW_LEFT,
+ { "jawleft", "jawsidewaysleft" } },
+ { XRFaceTracker::FT_JAW_FORWARD,
+ { "jawforward", "jawthrust" } },
+ { XRFaceTracker::FT_JAW_BACKWARD,
+ { "jawbackward" } },
+ { XRFaceTracker::FT_JAW_CLENCH,
+ { "jawclench" } },
+ { XRFaceTracker::FT_JAW_MANDIBLE_RAISE,
+ { "jawmandibleraise" } },
+ { XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT,
+ { "lipsuckupperright", "lipsuckrt" } },
+ { XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT,
+ { "lipsuckupperleft", "lipsucklt" } },
+ { XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT,
+ { "lipsucklowerright", "lipsuckrb" } },
+ { XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT,
+ { "lipsucklowerleft", "lipsucklb" } },
+ { XRFaceTracker::FT_LIP_SUCK_CORNER_RIGHT,
+ { "lipsuckcornerright" } },
+ { XRFaceTracker::FT_LIP_SUCK_CORNER_LEFT,
+ { "lipsuckcornerleft" } },
+ { XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT,
+ { "lipfunnelupperright", "lipfunnelerrt" } },
+ { XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT,
+ { "lipfunnelupperleft", "lipfunnelerlt" } },
+ { XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT,
+ { "lipfunnellowerright", "lipsuckrb" } },
+ { XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT,
+ { "lipfunnellowerleft", "lipsucklb" } },
+ { XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT,
+ { "lippuckerupperright" } },
+ { XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT,
+ { "lippuckerupperleft" } },
+ { XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT,
+ { "lippuckerlowerright" } },
+ { XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT,
+ { "lippuckerlowerleft" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT,
+ { "mouthupperupright", "upperlipraiserr" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT,
+ { "mouthupperupleft", "upperlipraiserl" } },
+ { XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT,
+ { "mouthlowerdownright", "mouthlowerupright", "lowerlipdepressorr" } },
+ { XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT,
+ { "mouthlowerdownleft", "mouthlowerupleft", "lowerlipdepressorl" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_DEEPEN_RIGHT,
+ { "mouthupperdeepenright" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_DEEPEN_LEFT,
+ { "mouthupperdeepenleft" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_RIGHT,
+ { "mouthupperright" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_LEFT,
+ { "mouthupperleft" } },
+ { XRFaceTracker::FT_MOUTH_LOWER_RIGHT,
+ { "mouthlowerright" } },
+ { XRFaceTracker::FT_MOUTH_LOWER_LEFT,
+ { "mouthlowerleft" } },
+ { XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT,
+ { "mouthcornerpullright" } },
+ { XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT,
+ { "mouthcornerpullleft" } },
+ { XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT,
+ { "mouthcornerslantright" } },
+ { XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT,
+ { "mouthcornerslantleft" } },
+ { XRFaceTracker::FT_MOUTH_FROWN_RIGHT,
+ { "mouthfrownright", "lipcornerdepressorr" } },
+ { XRFaceTracker::FT_MOUTH_FROWN_LEFT,
+ { "mouthfrownleft", "lipcornerdepressorl" } },
+ { XRFaceTracker::FT_MOUTH_STRETCH_RIGHT,
+ { "mouthstretchright", "lipstretcherr" } },
+ { XRFaceTracker::FT_MOUTH_STRETCH_LEFT,
+ { "mouthstretchleft", "lipstretcherl" } },
+ { XRFaceTracker::FT_MOUTH_DIMPLE_RIGHT,
+ { "mouthdimplerright", "mouthdimpleright", "dimplerr" } },
+ { XRFaceTracker::FT_MOUTH_DIMPLE_LEFT,
+ { "mouthdimplerleft", "mouthdimpleleft", "dimplerl" } },
+ { XRFaceTracker::FT_MOUTH_RAISER_UPPER,
+ { "mouthraiserupper", "mouthshrugupper", "chinraisert" } },
+ { XRFaceTracker::FT_MOUTH_RAISER_LOWER,
+ { "mouthraiserlower", "mouthshruglower", "mouthloweroverlay", "chinraiserb" } },
+ { XRFaceTracker::FT_MOUTH_PRESS_RIGHT,
+ { "mouthpressright", "lippressorr" } },
+ { XRFaceTracker::FT_MOUTH_PRESS_LEFT,
+ { "mouthpressleft", "lippressorl" } },
+ { XRFaceTracker::FT_MOUTH_TIGHTENER_RIGHT,
+ { "mouthtightenerright", "liptightenerr" } },
+ { XRFaceTracker::FT_MOUTH_TIGHTENER_LEFT,
+ { "mouthtightenerleft", "liptightenerl" } },
+ { XRFaceTracker::FT_TONGUE_OUT,
+ { "tongueout", "tonguelongstep2" } },
+ { XRFaceTracker::FT_TONGUE_UP,
+ { "tongueup" } },
+ { XRFaceTracker::FT_TONGUE_DOWN,
+ { "tonguedown" } },
+ { XRFaceTracker::FT_TONGUE_RIGHT,
+ { "tongueright" } },
+ { XRFaceTracker::FT_TONGUE_LEFT,
+ { "tongueleft" } },
+ { XRFaceTracker::FT_TONGUE_ROLL,
+ { "tongueroll" } },
+ { XRFaceTracker::FT_TONGUE_BLEND_DOWN,
+ { "tongueblenddown" } },
+ { XRFaceTracker::FT_TONGUE_CURL_UP,
+ { "tonguecurlup" } },
+ { XRFaceTracker::FT_TONGUE_SQUISH,
+ { "tonguesquish" } },
+ { XRFaceTracker::FT_TONGUE_FLAT,
+ { "tongueflat" } },
+ { XRFaceTracker::FT_TONGUE_TWIST_RIGHT,
+ { "tonguetwistright" } },
+ { XRFaceTracker::FT_TONGUE_TWIST_LEFT,
+ { "tonguetwistleft" } },
+ { XRFaceTracker::FT_SOFT_PALATE_CLOSE,
+ { "softpalateclose" } },
+ { XRFaceTracker::FT_THROAT_SWALLOW,
+ { "throatswallow" } },
+ { XRFaceTracker::FT_NECK_FLEX_RIGHT,
+ { "neckflexright" } },
+ { XRFaceTracker::FT_NECK_FLEX_LEFT,
+ { "neckflexleft" } },
+ { XRFaceTracker::FT_EYE_CLOSED,
+ { "eyeclosed" } },
+ { XRFaceTracker::FT_EYE_WIDE,
+ { "eyewide" } },
+ { XRFaceTracker::FT_EYE_SQUINT,
+ { "eyesquint" } },
+ { XRFaceTracker::FT_EYE_DILATION,
+ { "eyedilation" } },
+ { XRFaceTracker::FT_EYE_CONSTRICT,
+ { "eyeconstrict" } },
+ { XRFaceTracker::FT_BROW_DOWN_RIGHT,
+ { "browdownright", "browlowererr" } },
+ { XRFaceTracker::FT_BROW_DOWN_LEFT,
+ { "browdownleft", "browlowererl" } },
+ { XRFaceTracker::FT_BROW_DOWN,
+ { "browdown" } },
+ { XRFaceTracker::FT_BROW_UP_RIGHT,
+ { "browupright" } },
+ { XRFaceTracker::FT_BROW_UP_LEFT,
+ { "browupleft" } },
+ { XRFaceTracker::FT_BROW_UP,
+ { "browup" } },
+ { XRFaceTracker::FT_NOSE_SNEER,
+ { "nosesneer" } },
+ { XRFaceTracker::FT_NASAL_DILATION,
+ { "nasaldilation" } },
+ { XRFaceTracker::FT_NASAL_CONSTRICT,
+ { "nasalconstrict" } },
+ { XRFaceTracker::FT_CHEEK_PUFF,
+ { "cheekpuff" } },
+ { XRFaceTracker::FT_CHEEK_SUCK,
+ { "cheeksuck" } },
+ { XRFaceTracker::FT_CHEEK_SQUINT,
+ { "cheeksquint" } },
+ { XRFaceTracker::FT_LIP_SUCK_UPPER,
+ { "lipsuckupper", "mouthrollupper", "mouthupperinside" } },
+ { XRFaceTracker::FT_LIP_SUCK_LOWER,
+ { "lipsucklower", "mouthrolllower", "mouthlowerinside" } },
+ { XRFaceTracker::FT_LIP_SUCK,
+ { "lipsuck" } },
+ { XRFaceTracker::FT_LIP_FUNNEL_UPPER,
+ { "lipfunnelupper", "mouthupperoverturn" } },
+ { XRFaceTracker::FT_LIP_FUNNEL_LOWER,
+ { "lipfunnellower", "mouthloweroverturn" } },
+ { XRFaceTracker::FT_LIP_FUNNEL,
+ { "lipfunnel", "mouthfunnel" } },
+ { XRFaceTracker::FT_LIP_PUCKER_UPPER,
+ { "lippuckerupper" } },
+ { XRFaceTracker::FT_LIP_PUCKER_LOWER,
+ { "lippuckerlower" } },
+ { XRFaceTracker::FT_LIP_PUCKER,
+ { "lippucker", "mouthpucker", "mouthpout" } },
+ { XRFaceTracker::FT_MOUTH_UPPER_UP,
+ { "mouthupperup" } },
+ { XRFaceTracker::FT_MOUTH_LOWER_DOWN,
+ { "mouthlowerdown" } },
+ { XRFaceTracker::FT_MOUTH_OPEN,
+ { "mouthopen" } },
+ { XRFaceTracker::FT_MOUTH_RIGHT,
+ { "mouthright" } },
+ { XRFaceTracker::FT_MOUTH_LEFT,
+ { "mouthleft" } },
+ { XRFaceTracker::FT_MOUTH_SMILE_RIGHT,
+ { "mouthsmileright", "lipcornerpullerr" } },
+ { XRFaceTracker::FT_MOUTH_SMILE_LEFT,
+ { "mouthsmileleft", "lipcornerpullerl" } },
+ { XRFaceTracker::FT_MOUTH_SMILE,
+ { "mouthsmile" } },
+ { XRFaceTracker::FT_MOUTH_SAD_RIGHT,
+ { "mouthsadright" } },
+ { XRFaceTracker::FT_MOUTH_SAD_LEFT,
+ { "mouthsadleft" } },
+ { XRFaceTracker::FT_MOUTH_SAD,
+ { "mouthsad" } },
+ { XRFaceTracker::FT_MOUTH_STRETCH,
+ { "mouthstretch" } },
+ { XRFaceTracker::FT_MOUTH_DIMPLE,
+ { "mouthdimple" } },
+ { XRFaceTracker::FT_MOUTH_TIGHTENER,
+ { "mouthtightener" } },
+ { XRFaceTracker::FT_MOUTH_PRESS,
+ { "mouthpress" } }
+ };
+
+ // Convert the name to lower-case and strip non-alphanumeric characters.
+ const String name = String(p_name).to_lower().replace("_", "");
+
+ // Iterate through the blend map.
+ for (const blend_map_entry &entry : blend_map) {
+ for (const char *n : entry.name) {
+ if (n == nullptr) {
+ break;
+ }
+
+ if (name == n) {
+ return entry.blend;
+ }
+ }
+ }
+
+ // Blend shape not found.
+ return -1;
+}
+
+// This method adds all the identified XRFaceTracker blend shapes of
+// the mesh to the p_blend_mapping map. The map is indexed by the
+// XRFaceTracker blend shape, and the value is the index of the mesh
+// blend shape.
+static void identify_face_blend_shapes(RBMap &p_blend_mapping, const Ref &mesh) {
+ // Find all blend shapes.
+ const int count = mesh->get_blend_shape_count();
+ for (int i = 0; i < count; i++) {
+ const int blend = find_face_blend_shape(mesh->get_blend_shape_name(i));
+ if (blend >= 0) {
+ p_blend_mapping[blend] = i;
+ }
+ }
+}
+
+// This method removes any unified blend shapes from the p_blend_mapping map
+// if all the individual blend shapes are found and going to be driven.
+static void remove_driven_unified_blend_shapes(RBMap &p_blend_mapping) {
+ // Entry for unified blend table.
+ struct unified_blend_entry {
+ int unified;
+ int individual[4];
+ };
+
+ // Table of unified blend shapes.
+ //
+ // This table consists of:
+ // - The XRFaceTracker unified blend shape.
+ // - The individual blend shapes that make up the unified blend shape.
+ static constexpr unified_blend_entry unified_blends[] = {
+ { XRFaceTracker::FT_EYE_CLOSED,
+ { XRFaceTracker::FT_EYE_CLOSED_RIGHT, XRFaceTracker::FT_EYE_CLOSED_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_EYE_WIDE,
+ { XRFaceTracker::FT_EYE_WIDE_RIGHT, XRFaceTracker::FT_EYE_WIDE_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_EYE_SQUINT,
+ { XRFaceTracker::FT_EYE_SQUINT_RIGHT, XRFaceTracker::FT_EYE_SQUINT_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_EYE_DILATION,
+ { XRFaceTracker::FT_EYE_DILATION_RIGHT, XRFaceTracker::FT_EYE_DILATION_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_EYE_CONSTRICT,
+ { XRFaceTracker::FT_EYE_CONSTRICT_RIGHT, XRFaceTracker::FT_EYE_CONSTRICT_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_BROW_DOWN_RIGHT,
+ { XRFaceTracker::FT_BROW_LOWERER_RIGHT, XRFaceTracker::FT_BROW_PINCH_RIGHT, -1, -1 } },
+ { XRFaceTracker::FT_BROW_DOWN_LEFT,
+ { XRFaceTracker::FT_BROW_LOWERER_LEFT, XRFaceTracker::FT_BROW_PINCH_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_BROW_DOWN,
+ { XRFaceTracker::FT_BROW_LOWERER_RIGHT, XRFaceTracker::FT_BROW_PINCH_RIGHT, XRFaceTracker::FT_BROW_LOWERER_LEFT, XRFaceTracker::FT_BROW_PINCH_LEFT } },
+ { XRFaceTracker::FT_BROW_UP_RIGHT,
+ { XRFaceTracker::FT_BROW_INNER_UP_RIGHT, XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, -1, -1 } },
+ { XRFaceTracker::FT_BROW_UP_LEFT,
+ { XRFaceTracker::FT_BROW_INNER_UP_LEFT, XRFaceTracker::FT_BROW_OUTER_UP_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_BROW_UP,
+ { XRFaceTracker::FT_BROW_INNER_UP_RIGHT, XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, XRFaceTracker::FT_BROW_INNER_UP_LEFT, XRFaceTracker::FT_BROW_OUTER_UP_LEFT } },
+ { XRFaceTracker::FT_NOSE_SNEER,
+ { XRFaceTracker::FT_NOSE_SNEER_RIGHT, XRFaceTracker::FT_NOSE_SNEER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_NASAL_DILATION,
+ { XRFaceTracker::FT_NASAL_DILATION_RIGHT, XRFaceTracker::FT_NASAL_DILATION_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_NASAL_CONSTRICT,
+ { XRFaceTracker::FT_NASAL_CONSTRICT_RIGHT, XRFaceTracker::FT_NASAL_CONSTRICT_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_CHEEK_PUFF,
+ { XRFaceTracker::FT_CHEEK_PUFF_RIGHT, XRFaceTracker::FT_CHEEK_PUFF_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_CHEEK_SUCK,
+ { XRFaceTracker::FT_CHEEK_SUCK_RIGHT, XRFaceTracker::FT_CHEEK_SUCK_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_CHEEK_SQUINT,
+ { XRFaceTracker::FT_CHEEK_SQUINT_RIGHT, XRFaceTracker::FT_CHEEK_SQUINT_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_SUCK_UPPER,
+ { XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_SUCK_LOWER,
+ { XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_SUCK,
+ { XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT } },
+ { XRFaceTracker::FT_LIP_FUNNEL_UPPER,
+ { XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_FUNNEL_LOWER,
+ { XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_FUNNEL,
+ { XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT } },
+ { XRFaceTracker::FT_LIP_PUCKER_UPPER,
+ { XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_PUCKER_LOWER,
+ { XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_LIP_PUCKER,
+ { XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT } },
+ { XRFaceTracker::FT_MOUTH_UPPER_UP,
+ { XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_LOWER_DOWN,
+ { XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_OPEN,
+ { XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT } },
+ { XRFaceTracker::FT_MOUTH_RIGHT,
+ { XRFaceTracker::FT_MOUTH_UPPER_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_RIGHT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_LEFT,
+ { XRFaceTracker::FT_MOUTH_UPPER_LEFT, XRFaceTracker::FT_MOUTH_LOWER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_SMILE_RIGHT,
+ { XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_SMILE_LEFT,
+ { XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_SMILE,
+ { XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT } },
+ { XRFaceTracker::FT_MOUTH_SAD_RIGHT,
+ { XRFaceTracker::FT_MOUTH_FROWN_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_SAD_LEFT,
+ { XRFaceTracker::FT_MOUTH_FROWN_LEFT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_SAD,
+ { XRFaceTracker::FT_MOUTH_FROWN_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, XRFaceTracker::FT_MOUTH_FROWN_LEFT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT } },
+ { XRFaceTracker::FT_MOUTH_STRETCH,
+ { XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_DIMPLE,
+ { XRFaceTracker::FT_MOUTH_DIMPLE_RIGHT, XRFaceTracker::FT_MOUTH_DIMPLE_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_TIGHTENER,
+ { XRFaceTracker::FT_MOUTH_TIGHTENER_RIGHT, XRFaceTracker::FT_MOUTH_TIGHTENER_LEFT, -1, -1 } },
+ { XRFaceTracker::FT_MOUTH_PRESS,
+ { XRFaceTracker::FT_MOUTH_PRESS_RIGHT, XRFaceTracker::FT_MOUTH_PRESS_LEFT, -1, -1 } }
+ };
+
+ // Remove unified blend shapes if individual blend shapes are found.
+ for (const unified_blend_entry &entry : unified_blends) {
+ // Check if all individual blend shapes are found.
+ bool found = true;
+ for (const int i : entry.individual) {
+ if (i >= 0 && !p_blend_mapping.find(i)) {
+ found = false;
+ break;
+ }
+ }
+
+ // If all individual blend shapes are found then remove the unified blend shape.
+ if (found) {
+ p_blend_mapping.erase(entry.unified);
+ }
+ }
+}
+
+void XRFaceModifier3D::_bind_methods() {
+ ClassDB::bind_method(D_METHOD("set_face_tracker", "tracker_name"), &XRFaceModifier3D::set_face_tracker);
+ ClassDB::bind_method(D_METHOD("get_face_tracker"), &XRFaceModifier3D::get_face_tracker);
+ ADD_PROPERTY(PropertyInfo(Variant::STRING, "face_tracker", PROPERTY_HINT_ENUM_SUGGESTION, "/user/head"), "set_face_tracker", "get_face_tracker");
+
+ ClassDB::bind_method(D_METHOD("set_target", "target"), &XRFaceModifier3D::set_target);
+ ClassDB::bind_method(D_METHOD("get_target"), &XRFaceModifier3D::get_target);
+ ADD_PROPERTY(PropertyInfo(Variant::NODE_PATH, "target", PROPERTY_HINT_NODE_PATH_VALID_TYPES, "MeshInstance3D"), "set_target", "get_target");
+}
+
+void XRFaceModifier3D::set_face_tracker(const StringName &p_tracker_name) {
+ tracker_name = p_tracker_name;
+}
+
+StringName XRFaceModifier3D::get_face_tracker() const {
+ return tracker_name;
+}
+
+void XRFaceModifier3D::set_target(const NodePath &p_target) {
+ target = p_target;
+
+ if (is_inside_tree()) {
+ _get_blend_data();
+ }
+}
+
+NodePath XRFaceModifier3D::get_target() const {
+ return target;
+}
+
+MeshInstance3D *XRFaceModifier3D::get_mesh_instance() const {
+ if (!has_node(target)) {
+ return nullptr;
+ }
+
+ Node *node = get_node(target);
+ if (!node) {
+ return nullptr;
+ }
+
+ return Object::cast_to(node);
+}
+
+void XRFaceModifier3D::_get_blend_data() {
+ // This method constructs the blend mapping from the XRFaceTracker
+ // blend shapes to the available blend shapes of the target mesh. It does this
+ // by:
+ //
+ // 1. Identifying the blend shapes of the target mesh and identifying what
+ // XRFaceTracker blend shape they correspond to. The results are
+ // placed in the blend_mapping map.
+ // 2. Prevent over-driving facial blend-shapes by removing any unified blend
+ // shapes from the map if all the individual blend shapes are already
+ // found and going to be driven.
+
+ blend_mapping.clear();
+
+ // Get the target MeshInstance3D.
+ const MeshInstance3D *mesh_instance = get_mesh_instance();
+ if (!mesh_instance) {
+ return;
+ }
+
+ // Get the mesh.
+ const Ref mesh = mesh_instance->get_mesh();
+ if (mesh.is_null()) {
+ return;
+ }
+
+ // Identify all face blend shapes and populate the map.
+ identify_face_blend_shapes(blend_mapping, mesh);
+
+ // Remove the unified blend shapes if all the individual blend shapes are found.
+ remove_driven_unified_blend_shapes(blend_mapping);
+}
+
+void XRFaceModifier3D::_update_face_blends() const {
+ // Get the XR Server.
+ const XRServer *xr_server = XRServer::get_singleton();
+ if (!xr_server) {
+ return;
+ }
+
+ // Get the face tracker.
+ const Ref p = xr_server->get_face_tracker(tracker_name);
+ if (!p.is_valid()) {
+ return;
+ }
+
+ // Get the face mesh.
+ MeshInstance3D *mesh_instance = get_mesh_instance();
+ if (!mesh_instance) {
+ return;
+ }
+
+ // Get the blend weights.
+ const PackedFloat32Array weights = p->get_blend_shapes();
+
+ // Apply all the face blend weights to the mesh.
+ for (const KeyValue &it : blend_mapping) {
+ mesh_instance->set_blend_shape_value(it.value, weights[it.key]);
+ }
+}
+
+void XRFaceModifier3D::_notification(int p_what) {
+ switch (p_what) {
+ case NOTIFICATION_ENTER_TREE: {
+ _get_blend_data();
+ set_process_internal(true);
+ } break;
+ case NOTIFICATION_EXIT_TREE: {
+ set_process_internal(false);
+ blend_mapping.clear();
+ } break;
+ case NOTIFICATION_INTERNAL_PROCESS: {
+ _update_face_blends();
+ } break;
+ default: {
+ } break;
+ }
+}
diff --git a/scene/3d/xr_face_modifier_3d.h b/scene/3d/xr_face_modifier_3d.h
new file mode 100644
index 00000000000..147c374e95e
--- /dev/null
+++ b/scene/3d/xr_face_modifier_3d.h
@@ -0,0 +1,73 @@
+/**************************************************************************/
+/* xr_face_modifier_3d.h */
+/**************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* https://godotengine.org */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/**************************************************************************/
+
+#ifndef XR_FACE_MODIFIER_3D_H
+#define XR_FACE_MODIFIER_3D_H
+
+#include "mesh_instance_3d.h"
+#include "scene/3d/node_3d.h"
+
+/**
+ The XRFaceModifier3D node drives the blend shapes of a MeshInstance3D
+ with facial expressions from an XRFaceTracking instance.
+
+ The blend shapes provided by the mesh are interrogated, and used to
+ deduce an optimal mapping from the Unified Expressions blend shapes
+ provided by the XRFaceTracking instance to drive the face.
+ */
+
+class XRFaceModifier3D : public Node3D {
+ GDCLASS(XRFaceModifier3D, Node3D);
+
+private:
+ StringName tracker_name = "/user/head";
+ NodePath target;
+
+ // Map from XRFaceTracker blend shape index to mesh blend shape index.
+ RBMap blend_mapping;
+
+ MeshInstance3D *get_mesh_instance() const;
+ void _get_blend_data();
+ void _update_face_blends() const;
+
+protected:
+ static void _bind_methods();
+
+public:
+ void set_face_tracker(const StringName &p_tracker_name);
+ StringName get_face_tracker() const;
+
+ void set_target(const NodePath &p_target);
+ NodePath get_target() const;
+
+ void _notification(int p_what);
+};
+
+#endif // XR_FACE_MODIFIER_3D_H
diff --git a/scene/register_scene_types.cpp b/scene/register_scene_types.cpp
index 64a1c72f9d9..2a5e32f0256 100644
--- a/scene/register_scene_types.cpp
+++ b/scene/register_scene_types.cpp
@@ -271,6 +271,7 @@
#include "scene/3d/visible_on_screen_notifier_3d.h"
#include "scene/3d/voxel_gi.h"
#include "scene/3d/world_environment.h"
+#include "scene/3d/xr_face_modifier_3d.h"
#include "scene/3d/xr_nodes.h"
#include "scene/animation/root_motion_view.h"
#include "scene/resources/environment.h"
@@ -516,6 +517,7 @@ void register_scene_types() {
GDREGISTER_CLASS(XRController3D);
GDREGISTER_CLASS(XRAnchor3D);
GDREGISTER_CLASS(XROrigin3D);
+ GDREGISTER_CLASS(XRFaceModifier3D);
GDREGISTER_CLASS(MeshInstance3D);
GDREGISTER_CLASS(OccluderInstance3D);
GDREGISTER_ABSTRACT_CLASS(Occluder3D);
diff --git a/servers/register_server_types.cpp b/servers/register_server_types.cpp
index 0ee2984a8ca..b42e7bf9bb6 100644
--- a/servers/register_server_types.cpp
+++ b/servers/register_server_types.cpp
@@ -80,6 +80,7 @@
#include "text/text_server_dummy.h"
#include "text/text_server_extension.h"
#include "text_server.h"
+#include "xr/xr_face_tracker.h"
#include "xr/xr_interface.h"
#include "xr/xr_interface_extension.h"
#include "xr/xr_positional_tracker.h"
@@ -189,6 +190,7 @@ void register_server_types() {
GDREGISTER_CLASS(XRInterfaceExtension); // can't register this as virtual because we need a creation function for our extensions.
GDREGISTER_CLASS(XRPose);
GDREGISTER_CLASS(XRPositionalTracker);
+ GDREGISTER_CLASS(XRFaceTracker);
GDREGISTER_CLASS(AudioStream);
GDREGISTER_CLASS(AudioStreamPlayback);
diff --git a/servers/xr/xr_face_tracker.cpp b/servers/xr/xr_face_tracker.cpp
new file mode 100644
index 00000000000..a38ccfd527a
--- /dev/null
+++ b/servers/xr/xr_face_tracker.cpp
@@ -0,0 +1,222 @@
+/**************************************************************************/
+/* xr_face_tracker.cpp */
+/**************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* https://godotengine.org */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/**************************************************************************/
+
+#include "xr_face_tracker.h"
+
+void XRFaceTracker::_bind_methods() {
+ // Base Shapes
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_OUT_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_IN_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_UP_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_DOWN_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_OUT_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_IN_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_UP_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_LOOK_DOWN_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_CLOSED_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_CLOSED_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_SQUINT_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_SQUINT_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_WIDE_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_WIDE_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_DILATION_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_DILATION_LEFT);
+ BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT_RIGHT);
+ BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT_LEFT);
+ BIND_ENUM_CONSTANT(FT_BROW_PINCH_RIGHT);
+ BIND_ENUM_CONSTANT(FT_BROW_PINCH_LEFT);
+ BIND_ENUM_CONSTANT(FT_BROW_LOWERER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_BROW_LOWERER_LEFT);
+ BIND_ENUM_CONSTANT(FT_BROW_INNER_UP_RIGHT);
+ BIND_ENUM_CONSTANT(FT_BROW_INNER_UP_LEFT);
+ BIND_ENUM_CONSTANT(FT_BROW_OUTER_UP_RIGHT);
+ BIND_ENUM_CONSTANT(FT_BROW_OUTER_UP_LEFT);
+ BIND_ENUM_CONSTANT(FT_NOSE_SNEER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_NOSE_SNEER_LEFT);
+ BIND_ENUM_CONSTANT(FT_NASAL_DILATION_RIGHT);
+ BIND_ENUM_CONSTANT(FT_NASAL_DILATION_LEFT);
+ BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT_RIGHT);
+ BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT_LEFT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT_RIGHT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT_LEFT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_PUFF_RIGHT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_PUFF_LEFT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_SUCK_RIGHT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_SUCK_LEFT);
+ BIND_ENUM_CONSTANT(FT_JAW_OPEN);
+ BIND_ENUM_CONSTANT(FT_MOUTH_CLOSED);
+ BIND_ENUM_CONSTANT(FT_JAW_RIGHT);
+ BIND_ENUM_CONSTANT(FT_JAW_LEFT);
+ BIND_ENUM_CONSTANT(FT_JAW_FORWARD);
+ BIND_ENUM_CONSTANT(FT_JAW_BACKWARD);
+ BIND_ENUM_CONSTANT(FT_JAW_CLENCH);
+ BIND_ENUM_CONSTANT(FT_JAW_MANDIBLE_RAISE);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER_LEFT);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER_LEFT);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_CORNER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_CORNER_LEFT);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER_LEFT);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER_LEFT);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER_LEFT);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_DEEPEN_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_DEEPEN_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_PULL_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_PULL_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_SLANT_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_SLANT_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_FROWN_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_FROWN_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_RAISER_UPPER);
+ BIND_ENUM_CONSTANT(FT_MOUTH_RAISER_LOWER);
+ BIND_ENUM_CONSTANT(FT_MOUTH_PRESS_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_PRESS_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER_LEFT);
+ BIND_ENUM_CONSTANT(FT_TONGUE_OUT);
+ BIND_ENUM_CONSTANT(FT_TONGUE_UP);
+ BIND_ENUM_CONSTANT(FT_TONGUE_DOWN);
+ BIND_ENUM_CONSTANT(FT_TONGUE_RIGHT);
+ BIND_ENUM_CONSTANT(FT_TONGUE_LEFT);
+ BIND_ENUM_CONSTANT(FT_TONGUE_ROLL);
+ BIND_ENUM_CONSTANT(FT_TONGUE_BLEND_DOWN);
+ BIND_ENUM_CONSTANT(FT_TONGUE_CURL_UP);
+ BIND_ENUM_CONSTANT(FT_TONGUE_SQUISH);
+ BIND_ENUM_CONSTANT(FT_TONGUE_FLAT);
+ BIND_ENUM_CONSTANT(FT_TONGUE_TWIST_RIGHT);
+ BIND_ENUM_CONSTANT(FT_TONGUE_TWIST_LEFT);
+ BIND_ENUM_CONSTANT(FT_SOFT_PALATE_CLOSE);
+ BIND_ENUM_CONSTANT(FT_THROAT_SWALLOW);
+ BIND_ENUM_CONSTANT(FT_NECK_FLEX_RIGHT);
+ BIND_ENUM_CONSTANT(FT_NECK_FLEX_LEFT);
+ // Blended Shapes
+ BIND_ENUM_CONSTANT(FT_EYE_CLOSED);
+ BIND_ENUM_CONSTANT(FT_EYE_WIDE);
+ BIND_ENUM_CONSTANT(FT_EYE_SQUINT);
+ BIND_ENUM_CONSTANT(FT_EYE_DILATION);
+ BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT);
+ BIND_ENUM_CONSTANT(FT_BROW_DOWN_RIGHT);
+ BIND_ENUM_CONSTANT(FT_BROW_DOWN_LEFT);
+ BIND_ENUM_CONSTANT(FT_BROW_DOWN);
+ BIND_ENUM_CONSTANT(FT_BROW_UP_RIGHT);
+ BIND_ENUM_CONSTANT(FT_BROW_UP_LEFT);
+ BIND_ENUM_CONSTANT(FT_BROW_UP);
+ BIND_ENUM_CONSTANT(FT_NOSE_SNEER);
+ BIND_ENUM_CONSTANT(FT_NASAL_DILATION);
+ BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT);
+ BIND_ENUM_CONSTANT(FT_CHEEK_PUFF);
+ BIND_ENUM_CONSTANT(FT_CHEEK_SUCK);
+ BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER);
+ BIND_ENUM_CONSTANT(FT_LIP_SUCK);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER);
+ BIND_ENUM_CONSTANT(FT_LIP_FUNNEL);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER);
+ BIND_ENUM_CONSTANT(FT_LIP_PUCKER);
+ BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP);
+ BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN);
+ BIND_ENUM_CONSTANT(FT_MOUTH_OPEN);
+ BIND_ENUM_CONSTANT(FT_MOUTH_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_SMILE_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_SMILE_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_SMILE);
+ BIND_ENUM_CONSTANT(FT_MOUTH_SAD_RIGHT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_SAD_LEFT);
+ BIND_ENUM_CONSTANT(FT_MOUTH_SAD);
+ BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH);
+ BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE);
+ BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER);
+ BIND_ENUM_CONSTANT(FT_MOUTH_PRESS);
+ BIND_ENUM_CONSTANT(FT_MAX);
+
+ ClassDB::bind_method(D_METHOD("get_blend_shape", "blend_shape"), &XRFaceTracker::get_blend_shape);
+ ClassDB::bind_method(D_METHOD("set_blend_shape", "blend_shape", "weight"), &XRFaceTracker::set_blend_shape);
+
+ ClassDB::bind_method(D_METHOD("get_blend_shapes"), &XRFaceTracker::get_blend_shapes);
+ ClassDB::bind_method(D_METHOD("set_blend_shapes", "weights"), &XRFaceTracker::set_blend_shapes);
+ ADD_PROPERTY(PropertyInfo(Variant::PACKED_FLOAT32_ARRAY, "blend_shapes"), "set_blend_shapes", "get_blend_shapes");
+ ADD_PROPERTY_DEFAULT("blend_shapes", PackedFloat32Array()); // To prevent ludicrously large default values.
+}
+
+float XRFaceTracker::get_blend_shape(BlendShapeEntry p_blend_shape) const {
+ // Fail if the blend shape index is out of range.
+ ERR_FAIL_INDEX_V(p_blend_shape, FT_MAX, 0.0f);
+
+ // Return the blend shape value.
+ return blend_shape_values[p_blend_shape];
+}
+
+void XRFaceTracker::set_blend_shape(BlendShapeEntry p_blend_shape, float p_value) {
+ // Fail if the blend shape index is out of range.
+ ERR_FAIL_INDEX(p_blend_shape, FT_MAX);
+
+ // Save the new blend shape value.
+ blend_shape_values[p_blend_shape] = p_value;
+}
+
+PackedFloat32Array XRFaceTracker::get_blend_shapes() const {
+ // Create a packed float32 array and copy the blend shape values into it.
+ PackedFloat32Array data;
+ data.resize(FT_MAX);
+ memcpy(data.ptrw(), blend_shape_values, sizeof(blend_shape_values));
+
+ // Return the blend shape array.
+ return data;
+}
+
+void XRFaceTracker::set_blend_shapes(const PackedFloat32Array &p_blend_shapes) {
+ // Fail if the blend shape array is not the correct size.
+ ERR_FAIL_COND(p_blend_shapes.size() != FT_MAX);
+
+ // Copy the blend shape values into the blend shape array.
+ memcpy(blend_shape_values, p_blend_shapes.ptr(), sizeof(blend_shape_values));
+}
diff --git a/servers/xr/xr_face_tracker.h b/servers/xr/xr_face_tracker.h
new file mode 100644
index 00000000000..b9f553cba69
--- /dev/null
+++ b/servers/xr/xr_face_tracker.h
@@ -0,0 +1,213 @@
+/**************************************************************************/
+/* xr_face_tracker.h */
+/**************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* https://godotengine.org */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/**************************************************************************/
+
+#ifndef XR_FACE_TRACKER_H
+#define XR_FACE_TRACKER_H
+
+#include "core/object/ref_counted.h"
+
+/**
+ The XRFaceTracker class provides face blend shape weights.
+
+ The supported blend shapes are based on the Unified Expressions
+ standard, and as such have a well defined mapping to ARKit, SRanipal,
+ and Meta Movement standards.
+ */
+
+class XRFaceTracker : public RefCounted {
+ GDCLASS(XRFaceTracker, RefCounted);
+ _THREAD_SAFE_CLASS_
+
+public:
+ enum BlendShapeEntry {
+ // Base Shapes
+ FT_EYE_LOOK_OUT_RIGHT, // Right eye looks outwards.
+ FT_EYE_LOOK_IN_RIGHT, // Right eye looks inwards.
+ FT_EYE_LOOK_UP_RIGHT, // Right eye looks upwards.
+ FT_EYE_LOOK_DOWN_RIGHT, // Right eye looks downwards.
+ FT_EYE_LOOK_OUT_LEFT, // Left eye looks outwards.
+ FT_EYE_LOOK_IN_LEFT, // Left eye looks inwards.
+ FT_EYE_LOOK_UP_LEFT, // Left eye looks upwards.
+ FT_EYE_LOOK_DOWN_LEFT, // Left eye looks downwards.
+ FT_EYE_CLOSED_RIGHT, // Closes the right eyelid.
+ FT_EYE_CLOSED_LEFT, // Closes the left eyelid.
+ FT_EYE_SQUINT_RIGHT, // Squeezes the right eye socket muscles.
+ FT_EYE_SQUINT_LEFT, // Squeezes the left eye socket muscles.
+ FT_EYE_WIDE_RIGHT, // Right eyelid widens beyond relaxed.
+ FT_EYE_WIDE_LEFT, // Left eyelid widens beyond relaxed.
+ FT_EYE_DILATION_RIGHT, // Dilates the right eye pupil.
+ FT_EYE_DILATION_LEFT, // Dilates the left eye pupil.
+ FT_EYE_CONSTRICT_RIGHT, // Constricts the right eye pupil.
+ FT_EYE_CONSTRICT_LEFT, // Constricts the left eye pupil.
+ FT_BROW_PINCH_RIGHT, // Right eyebrow pinches in.
+ FT_BROW_PINCH_LEFT, // Left eyebrow pinches in.
+ FT_BROW_LOWERER_RIGHT, // Outer right eyebrow pulls down.
+ FT_BROW_LOWERER_LEFT, // Outer left eyebrow pulls down.
+ FT_BROW_INNER_UP_RIGHT, // Inner right eyebrow pulls up.
+ FT_BROW_INNER_UP_LEFT, // Inner left eyebrow pulls up.
+ FT_BROW_OUTER_UP_RIGHT, // Outer right eyebrow pulls up.
+ FT_BROW_OUTER_UP_LEFT, // Outer left eyebrow pulls up.
+ FT_NOSE_SNEER_RIGHT, // Right side face sneers.
+ FT_NOSE_SNEER_LEFT, // Left side face sneers.
+ FT_NASAL_DILATION_RIGHT, // Right side nose canal dilates.
+ FT_NASAL_DILATION_LEFT, // Left side nose canal dilates.
+ FT_NASAL_CONSTRICT_RIGHT, // Right side nose canal constricts.
+ FT_NASAL_CONSTRICT_LEFT, // Left side nose canal constricts.
+ FT_CHEEK_SQUINT_RIGHT, // Raises the right side cheek.
+ FT_CHEEK_SQUINT_LEFT, // Raises the left side cheek.
+ FT_CHEEK_PUFF_RIGHT, // Puffs the right side cheek.
+ FT_CHEEK_PUFF_LEFT, // Puffs the left side cheek.
+ FT_CHEEK_SUCK_RIGHT, // Sucks in the right side cheek.
+ FT_CHEEK_SUCK_LEFT, // Sucks in the left side cheek.
+ FT_JAW_OPEN, // Opens jawbone.
+ FT_MOUTH_CLOSED, // Closes the mouth.
+ FT_JAW_RIGHT, // Pushes jawbone right.
+ FT_JAW_LEFT, // Pushes jawbone left.
+ FT_JAW_FORWARD, // Pushes jawbone forward.
+ FT_JAW_BACKWARD, // Pushes jawbone backward.
+ FT_JAW_CLENCH, // Flexes jaw muscles.
+ FT_JAW_MANDIBLE_RAISE, // Raises the jawbone.
+ FT_LIP_SUCK_UPPER_RIGHT, // Upper right lip part tucks in the mouth.
+ FT_LIP_SUCK_UPPER_LEFT, // Upper left lip part tucks in the mouth.
+ FT_LIP_SUCK_LOWER_RIGHT, // Lower right lip part tucks in the mouth.
+ FT_LIP_SUCK_LOWER_LEFT, // Lower left lip part tucks in the mouth.
+ FT_LIP_SUCK_CORNER_RIGHT, // Right lip corner folds into the mouth.
+ FT_LIP_SUCK_CORNER_LEFT, // Left lip corner folds into the mouth.
+ FT_LIP_FUNNEL_UPPER_RIGHT, // Upper right lip part pushes into a funnel.
+ FT_LIP_FUNNEL_UPPER_LEFT, // Upper left lip part pushes into a funnel.
+ FT_LIP_FUNNEL_LOWER_RIGHT, // Lower right lip part pushes into a funnel.
+ FT_LIP_FUNNEL_LOWER_LEFT, // Lower left lip part pushes into a funnel.
+ FT_LIP_PUCKER_UPPER_RIGHT, // Upper right lip part pushes outwards.
+ FT_LIP_PUCKER_UPPER_LEFT, // Upper left lip part pushes outwards.
+ FT_LIP_PUCKER_LOWER_RIGHT, // Lower right lip part pushes outwards.
+ FT_LIP_PUCKER_LOWER_LEFT, // Lower left lip part pushes outwards.
+ FT_MOUTH_UPPER_UP_RIGHT, // Upper right part of the lip pulls up.
+ FT_MOUTH_UPPER_UP_LEFT, // Upper left part of the lip pulls up.
+ FT_MOUTH_LOWER_DOWN_RIGHT, // Lower right part of the lip pulls up.
+ FT_MOUTH_LOWER_DOWN_LEFT, // Lower left part of the lip pulls up.
+ FT_MOUTH_UPPER_DEEPEN_RIGHT, // Upper right lip part pushes in the cheek.
+ FT_MOUTH_UPPER_DEEPEN_LEFT, // Upper left lip part pushes in the cheek.
+ FT_MOUTH_UPPER_RIGHT, // Moves upper lip right.
+ FT_MOUTH_UPPER_LEFT, // Moves upper lip left.
+ FT_MOUTH_LOWER_RIGHT, // Moves lower lip right.
+ FT_MOUTH_LOWER_LEFT, // Moves lower lip left.
+ FT_MOUTH_CORNER_PULL_RIGHT, // Right lip corner pulls diagonally up and out.
+ FT_MOUTH_CORNER_PULL_LEFT, // Left lip corner pulls diagonally up and out.
+ FT_MOUTH_CORNER_SLANT_RIGHT, // Right corner lip slants up.
+ FT_MOUTH_CORNER_SLANT_LEFT, // Left corner lip slants up.
+ FT_MOUTH_FROWN_RIGHT, // Right corner lip pulls down.
+ FT_MOUTH_FROWN_LEFT, // Left corner lip pulls down.
+ FT_MOUTH_STRETCH_RIGHT, // Mouth corner lip pulls out and down.
+ FT_MOUTH_STRETCH_LEFT, // Mouth corner lip pulls out and down.
+ FT_MOUTH_DIMPLE_RIGHT, // Right lip corner is pushed backwards.
+ FT_MOUTH_DIMPLE_LEFT, // Left lip corner is pushed backwards.
+ FT_MOUTH_RAISER_UPPER, // Raises and slightly pushes out the upper mouth.
+ FT_MOUTH_RAISER_LOWER, // Raises and slightly pushes out the lower mouth.
+ FT_MOUTH_PRESS_RIGHT, // Right side lips press and flatten together vertically.
+ FT_MOUTH_PRESS_LEFT, // Left side lips press and flatten together vertically.
+ FT_MOUTH_TIGHTENER_RIGHT, // Right side lips squeeze together horizontally.
+ FT_MOUTH_TIGHTENER_LEFT, // Left side lips squeeze together horizontally.
+ FT_TONGUE_OUT, // Tongue visibly sticks out of the mouth.
+ FT_TONGUE_UP, // Tongue points upwards.
+ FT_TONGUE_DOWN, // Tongue points downwards.
+ FT_TONGUE_RIGHT, // Tongue points right.
+ FT_TONGUE_LEFT, // Tongue points left.
+ FT_TONGUE_ROLL, // Sides of the tongue funnel, creating a roll.
+ FT_TONGUE_BLEND_DOWN, // Tongue arches up then down inside the mouth.
+ FT_TONGUE_CURL_UP, // Tongue arches down then up inside the mouth.
+ FT_TONGUE_SQUISH, // Tongue squishes together and thickens.
+ FT_TONGUE_FLAT, // Tongue flattens and thins out.
+ FT_TONGUE_TWIST_RIGHT, // Tongue tip rotates clockwise, with the rest following gradually.
+ FT_TONGUE_TWIST_LEFT, // Tongue tip rotates counter-clockwise, with the rest following gradually.
+ FT_SOFT_PALATE_CLOSE, // Inner mouth throat closes.
+ FT_THROAT_SWALLOW, // The Adam's apple visibly swallows.
+ FT_NECK_FLEX_RIGHT, // Right side neck visibly flexes.
+ FT_NECK_FLEX_LEFT, // Left side neck visibly flexes.
+ // Blended Shapes
+ FT_EYE_CLOSED, // Closes both eye lids.
+ FT_EYE_WIDE, // Widens both eye lids.
+ FT_EYE_SQUINT, // Squints both eye lids.
+ FT_EYE_DILATION, // Dilates both pupils.
+ FT_EYE_CONSTRICT, // Constricts both pupils.
+ FT_BROW_DOWN_RIGHT, // Pulls the right eyebrow down and in.
+ FT_BROW_DOWN_LEFT, // Pulls the left eyebrow down and in.
+ FT_BROW_DOWN, // Pulls both eyebrows down and in.
+ FT_BROW_UP_RIGHT, // Right brow appears worried.
+ FT_BROW_UP_LEFT, // Left brow appears worried.
+ FT_BROW_UP, // Both brows appear worried.
+ FT_NOSE_SNEER, // Entire face sneers.
+ FT_NASAL_DILATION, // Both nose canals dilate.
+ FT_NASAL_CONSTRICT, // Both nose canals constrict.
+ FT_CHEEK_PUFF, // Puffs both cheeks.
+ FT_CHEEK_SUCK, // Sucks in both cheeks.
+ FT_CHEEK_SQUINT, // Raises both cheeks.
+ FT_LIP_SUCK_UPPER, // Tucks in the upper lips.
+ FT_LIP_SUCK_LOWER, // Tucks in the lower lips.
+ FT_LIP_SUCK, // Tucks in both lips.
+ FT_LIP_FUNNEL_UPPER, // Funnels in the upper lips.
+ FT_LIP_FUNNEL_LOWER, // Funnels in the lower lips.
+ FT_LIP_FUNNEL, // Funnels in both lips.
+ FT_LIP_PUCKER_UPPER, // Upper lip part pushes outwards.
+ FT_LIP_PUCKER_LOWER, // Lower lip part pushes outwards.
+ FT_LIP_PUCKER, // Lips push outwards.
+ FT_MOUTH_UPPER_UP, // Raises the upper lips.
+ FT_MOUTH_LOWER_DOWN, // Lowers the lower lips.
+ FT_MOUTH_OPEN, // Mouth opens, revealing teeth.
+ FT_MOUTH_RIGHT, // Moves mouth right.
+ FT_MOUTH_LEFT, // Moves mouth left.
+ FT_MOUTH_SMILE_RIGHT, // Right side of the mouth smiles.
+ FT_MOUTH_SMILE_LEFT, // Left side of the mouth smiles.
+ FT_MOUTH_SMILE, // Mouth expresses a smile.
+ FT_MOUTH_SAD_RIGHT, // Right side of the mouth expresses sadness.
+ FT_MOUTH_SAD_LEFT, // Left side of the mouth expresses sadness.
+ FT_MOUTH_SAD, // Mouth expresses sadness.
+ FT_MOUTH_STRETCH, // Mouth stretches.
+ FT_MOUTH_DIMPLE, // Lip corners dimple.
+ FT_MOUTH_TIGHTENER, // Mouth tightens.
+ FT_MOUTH_PRESS, // Mouth presses together.
+ FT_MAX // Maximum blend shape.
+ };
+
+ float get_blend_shape(BlendShapeEntry p_blend_shape) const;
+ void set_blend_shape(BlendShapeEntry p_blend_shape, float p_value);
+
+ PackedFloat32Array get_blend_shapes() const;
+ void set_blend_shapes(const PackedFloat32Array &p_blend_shapes);
+
+protected:
+ static void _bind_methods();
+
+private:
+ float blend_shape_values[FT_MAX] = {};
+};
+
+VARIANT_ENUM_CAST(XRFaceTracker::BlendShapeEntry);
+
+#endif // XR_FACE_TRACKER_H
diff --git a/servers/xr_server.cpp b/servers/xr_server.cpp
index e7f644d53fb..b3bb0a3702a 100644
--- a/servers/xr_server.cpp
+++ b/servers/xr_server.cpp
@@ -30,6 +30,7 @@
#include "xr_server.h"
#include "core/config/project_settings.h"
+#include "xr/xr_face_tracker.h"
#include "xr/xr_interface.h"
#include "xr/xr_positional_tracker.h"
@@ -74,6 +75,11 @@ void XRServer::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_trackers", "tracker_types"), &XRServer::get_trackers);
ClassDB::bind_method(D_METHOD("get_tracker", "tracker_name"), &XRServer::get_tracker);
+ ClassDB::bind_method(D_METHOD("add_face_tracker", "tracker_name", "face_tracker"), &XRServer::add_face_tracker);
+ ClassDB::bind_method(D_METHOD("remove_face_tracker", "tracker_name"), &XRServer::remove_face_tracker);
+ ClassDB::bind_method(D_METHOD("get_face_trackers"), &XRServer::get_face_trackers);
+ ClassDB::bind_method(D_METHOD("get_face_tracker", "tracker_name"), &XRServer::get_face_tracker);
+
ClassDB::bind_method(D_METHOD("get_primary_interface"), &XRServer::get_primary_interface);
ClassDB::bind_method(D_METHOD("set_primary_interface", "interface"), &XRServer::set_primary_interface);
@@ -97,6 +103,10 @@ void XRServer::_bind_methods() {
ADD_SIGNAL(MethodInfo("tracker_added", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type")));
ADD_SIGNAL(MethodInfo("tracker_updated", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type")));
ADD_SIGNAL(MethodInfo("tracker_removed", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type")));
+
+ ADD_SIGNAL(MethodInfo("face_tracker_added", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::OBJECT, "face_tracker", PROPERTY_HINT_RESOURCE_TYPE, "XRFaceTracker")));
+ ADD_SIGNAL(MethodInfo("face_tracker_updated", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::OBJECT, "face_tracker", PROPERTY_HINT_RESOURCE_TYPE, "XRFaceTracker")));
+ ADD_SIGNAL(MethodInfo("face_tracker_removed", PropertyInfo(Variant::STRING_NAME, "tracker_name")));
};
double XRServer::get_world_scale() const {
@@ -352,6 +362,44 @@ PackedStringArray XRServer::get_suggested_pose_names(const StringName &p_tracker
return arr;
}
+void XRServer::add_face_tracker(const StringName &p_tracker_name, Ref p_face_tracker) {
+ ERR_FAIL_COND(p_face_tracker.is_null());
+
+ if (!face_trackers.has(p_tracker_name)) {
+ // We don't have a tracker with this name, we're going to add it.
+ face_trackers[p_tracker_name] = p_face_tracker;
+ emit_signal(SNAME("face_tracker_added"), p_tracker_name, p_face_tracker);
+ } else if (face_trackers[p_tracker_name] != p_face_tracker) {
+ // We already have a tracker with this name, we're going to replace it.
+ face_trackers[p_tracker_name] = p_face_tracker;
+ emit_signal(SNAME("face_tracker_updated"), p_tracker_name, p_face_tracker);
+ }
+}
+
+void XRServer::remove_face_tracker(const StringName &p_tracker_name) {
+ // Skip if no face tracker is found.
+ if (!face_trackers.has(p_tracker_name)) {
+ return;
+ }
+
+ // Send the removed signal, then remove the face tracker.
+ emit_signal(SNAME("face_tracker_removed"), p_tracker_name);
+ face_trackers.erase(p_tracker_name);
+}
+
+Dictionary XRServer::get_face_trackers() const {
+ return face_trackers;
+}
+
+Ref XRServer::get_face_tracker(const StringName &p_tracker_name) const {
+ // Skip if no tracker is found.
+ if (!face_trackers.has(p_tracker_name)) {
+ return Ref();
+ }
+
+ return face_trackers[p_tracker_name];
+}
+
void XRServer::_process() {
// called from our main game loop before we handle physics and game logic
// note that we can have multiple interfaces active if we have interfaces that purely handle tracking
diff --git a/servers/xr_server.h b/servers/xr_server.h
index fe59fc22cb7..0a4e020a1f6 100644
--- a/servers/xr_server.h
+++ b/servers/xr_server.h
@@ -39,6 +39,7 @@
class XRInterface;
class XRPositionalTracker;
+class XRFaceTracker;
/**
The XR server is a singleton object that gives access to the various
@@ -86,6 +87,8 @@ private:
Vector> interfaces;
Dictionary trackers;
+ Dictionary face_trackers;
+
Ref primary_interface; /* we'll identify one interface as primary, this will be used by our viewports */
double world_scale; /* scale by which we multiply our tracker positions */
@@ -183,6 +186,14 @@ public:
PackedStringArray get_suggested_pose_names(const StringName &p_tracker_name) const;
// Q: Should we add get_suggested_input_names and get_suggested_haptic_names even though we don't use them for the IDE?
+ /*
+ Face trackers are objects that expose the tracked blend shapes of a face.
+ */
+ void add_face_tracker(const StringName &p_tracker_name, Ref p_face_tracker);
+ void remove_face_tracker(const StringName &p_tracker_name);
+ Dictionary get_face_trackers() const;
+ Ref get_face_tracker(const StringName &p_tracker_name) const;
+
// Process is called before we handle our physics process and game process. This is where our interfaces will update controller data and such.
void _process();