Implement mesh skinning

Author: Looooong

Cargo.toml

@@ -21,6 +21,7 @@ members = ["crates/*", "examples/ios", "tools/ci"]
 
 [features]
 default = [
+  "bevy_animation_rig",
   "bevy_audio",
   "bevy_dynamic_plugin",
   "bevy_gilrs",
@@ -41,6 +42,7 @@ dynamic = ["bevy_dylib"]
 render = ["bevy_internal/bevy_pbr", "bevy_internal/bevy_render", "bevy_internal/bevy_sprite", "bevy_internal/bevy_text", "bevy_internal/bevy_ui"]
 
 # Optional bevy crates
+bevy_animation_rig = ["bevy_internal/bevy_animation_rig"]
 bevy_audio = ["bevy_internal/bevy_audio"]
 bevy_dynamic_plugin = ["bevy_internal/bevy_dynamic_plugin"]
 bevy_gilrs = ["bevy_internal/bevy_gilrs"]
@@ -176,6 +178,15 @@ path = "examples/3d/wireframe.rs"
 name = "z_sort_debug"
 path = "examples/3d/z_sort_debug.rs"
 
+# Animation
+[[example]]
+name = "custom_skinned_mesh"
+path = "examples/animation/custom_skinned_mesh.rs"
+
+[[example]]
+name = "gltf_skinned_mesh"
+path = "examples/animation/gltf_skinned_mesh.rs"
+
 # Application
 [[example]]
 name = "custom_loop"

assets/models/SimpleSkin/SimpleSkin.gltf

@@ -0,0 +1 @@
+{"scenes":[{"nodes":[0]}],"nodes":[{"skin":0,"mesh":0,"children":[1]},{"children":[2],"translation":[0,1,0]},{"rotation":[0,0,0,1]}],"meshes":[{"primitives":[{"attributes":{"POSITION":1,"JOINTS_0":2,"WEIGHTS_0":3},"indices":0}]}],"skins":[{"inverseBindMatrices":4,"joints":[1,2]}],"animations":[{"channels":[{"sampler":0,"target":{"node":2,"path":"rotation"}}],"samplers":[{"input":5,"interpolation":"LINEAR","output":6}]}],"buffers":[{"uri":"data:application/gltf-buffer;base64,AAABAAMAAAADAAIAAgADAAUAAgAFAAQABAAFAAcABAAHAAYABgAHAAkABgAJAAgAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAAD8AAAAAAACAPwAAAD8AAAAAAAAAAAAAgD8AAAAAAACAPwAAgD8AAAAAAAAAAAAAwD8AAAAAAACAPwAAwD8AAAAAAAAAAAAAAEAAAAAAAACAPwAAAEAAAAAA","byteLength":168},{"uri":"data:application/gltf-buffer;base64,AAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAABAPwAAgD4AAAAAAAAAAAAAQD8AAIA+AAAAAAAAAAAAAAA/AAAAPwAAAAAAAAAAAAAAPwAAAD8AAAAAAAAAAAAAgD4AAEA/AAAAAAAAAAAAAIA+AABAPwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAA=","byteLength":320},{"uri":"data:application/gltf-buffer;base64,AACAPwAAAAAAAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAvwAAgL8AAAAAAACAPwAAgD8AAAAAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAAAAAAIA/AAAAAAAAAL8AAIC/AAAAAAAAgD8=","byteLength":128},{"uri":"data:application/gltf-buffer;base64,AAAAAAAAAD8AAIA/AADAPwAAAEAAACBAAABAQAAAYEAAAIBAAACQQAAAoEAAALBAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAkxjEPkSLbD8AAAAAAAAAAPT9ND/0/TQ/AAAAAAAAAAD0/TQ/9P00PwAAAAAAAAAAkxjEPkSLbD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAkxjEvkSLbD8AAAAAAAAAAPT9NL/0/TQ/AAAAAAAAAAD0/TS/9P00PwAAAAAAAAAAkxjEvkSLbD8AAAAAAAAAAAAAAAAAAIA/","byteLength":240}],"bufferViews":[{"buffer":0,"byteOffset":0,"byteLength":48,"target":34963},{"buffer":0,"byteOffset":48,"byteLength":120,"target":34962},{"buffer":1,"byteOffset":0,"byteLength":320,"byteStride":16},{"buffer":2,"byteOffset":0,"byteLength":128},{"buffer":3,"byteOffset":0,"byteLength":240}],"accessors":[{"bufferView":0,"byteOffset":0,"componentType":5123,"count":24,"type":"SCALAR","max":[9],"min":[0]},{"bufferView":1,"byteOffset":0,"componentType":5126,"count":10,"type":"VEC3","max":[1,2,0],"min":[0,0,0]},{"bufferView":2,"byteOffset":0,"componentType":5123,"count":10,"type":"VEC4","max":[0,1,0,0],"min":[0,1,0,0]},{"bufferView":2,"byteOffset":160,"componentType":5126,"count":10,"type":"VEC4","max":[1,1,0,0],"min":[0,0,0,0]},{"bufferView":3,"byteOffset":0,"componentType":5126,"count":2,"type":"MAT4","max":[1,0,0,0,0,1,0,0,0,0,1,0,-0.5,-1,0,1],"min":[1,0,0,0,0,1,0,0,0,0,1,0,-0.5,-1,0,1]},{"bufferView":4,"byteOffset":0,"componentType":5126,"count":12,"type":"SCALAR","max":[5.5],"min":[0]},{"bufferView":4,"byteOffset":48,"componentType":5126,"count":12,"type":"VEC4","max":[0,0,0.707,1],"min":[0,0,-0.707,0.707]}],"asset":{"version":"2.0"}}

crates/bevy_animation_rig/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "bevy_animation_rig"
+version = "0.5.0"
+edition = "2018"
+authors = [
+    "Bevy Contributors <[email protected]>",
+    "Carter Anderson <[email protected]>",
+]
+description = "Bevy Engine Animation Rigging System"
+homepage = "https://bevyengine.org"
+repository = "https://github.com/bevyengine/bevy"
+license = "MIT"
+keywords = ["bevy", "animation", "rig", "skeleton"]
+
+[dependencies]
+# bevy
+bevy_app = { path = "../bevy_app", version = "0.5.0" }
+bevy_asset = { path = "../bevy_asset", version = "0.5.0" }
+bevy_ecs = { path = "../bevy_ecs", version = "0.5.0" }
+bevy_math = { path = "../bevy_math", version = "0.5.0" }
+bevy_pbr = { path = "../bevy_pbr", version = "0.5.0" }
+bevy_reflect = { path = "../bevy_reflect", version = "0.5.0", features = ["bevy"] }
+bevy_render = { path = "../bevy_render", version = "0.5.0" }
+bevy_transform = { path = "../bevy_transform", version = "0.5.0" }

crates/bevy_animation_rig/src/lib.rs

@@ -0,0 +1,39 @@
+use bevy_app::{AppBuilder, CoreStage, Plugin, StartupStage};
+use bevy_asset::AddAsset;
+use bevy_ecs::{
+    schedule::{ParallelSystemDescriptorCoercion, SystemLabel},
+    system::IntoSystem,
+};
+use bevy_transform::TransformSystem;
+
+mod skinned_mesh;
+pub use skinned_mesh::*;
+
+#[derive(Debug, Hash, PartialEq, Eq, Clone, SystemLabel)]
+pub enum AnimationRigSystem {
+    SkinnedMeshSetup,
+    SkinnedMeshUpdate,
+}
+
+#[derive(Default)]
+pub struct AnimationRigPlugin;
+
+impl Plugin for AnimationRigPlugin {
+    fn build(&self, app: &mut AppBuilder) {
+        app.register_type::<SkinnedMesh>()
+            .add_asset::<SkinnedMeshInverseBindposes>()
+            .add_startup_system_to_stage(
+                StartupStage::PreStartup,
+                skinned_mesh_setup
+                    .system()
+                    .label(AnimationRigSystem::SkinnedMeshSetup),
+            )
+            .add_system_to_stage(
+                CoreStage::PostUpdate,
+                skinned_mesh_update
+                    .system()
+                    .label(AnimationRigSystem::SkinnedMeshUpdate)
+                    .after(TransformSystem::TransformPropagate),
+            );
+    }
+}

crates/bevy_animation_rig/src/skinned_mesh.rs

@@ -0,0 +1,227 @@
+use bevy_asset::{Assets, Handle, HandleUntyped};
+use bevy_ecs::{
+    entity::{Entity, EntityMap, MapEntities, MapEntitiesError},
+    reflect::{ReflectComponent, ReflectMapEntities},
+    system::{Query, Res, ResMut},
+};
+use bevy_math::Mat4;
+use bevy_pbr::render_graph;
+use bevy_reflect::{Reflect, TypeUuid};
+use bevy_render::{
+    pipeline::PipelineDescriptor,
+    render_graph::{base::node, RenderGraph, RenderResourcesNode},
+    renderer::{
+        RenderResource, RenderResourceHints, RenderResourceIterator, RenderResourceType,
+        RenderResources,
+    },
+    shader::{Shader, ShaderStage},
+    texture::Texture,
+};
+use bevy_transform::components::GlobalTransform;
+
+/// Specify RenderPipelines with this handle to render the skinned mesh.
+pub const SKINNED_MESH_PIPELINE_HANDLE: HandleUntyped =
+    HandleUntyped::weak_from_u64(PipelineDescriptor::TYPE_UUID, 0x14db1922328e7fcc);
+
+/// Used to update and bind joint transforms to the skinned mesh render pipeline specified with [`SKINNED_MESH_PIPELINE_HANDLE`].
+///
+/// The length of `joint_entities` and `joint_transforms` should equal to the number of matrices inside [`SkinnedMeshInverseBindposes`].
+///
+/// The content of `joint_transforms` can be modified manually if [`skinned_mesh_update`] system is disabled.
+///
+/// # Example
+/// ```
+/// use bevy_animation_rig::{SkinnedMesh, SKINNED_MESH_PIPELINE_HANDLE};
+/// use bevy_ecs::{entity::Entity, system::Commands};
+/// use bevy_pbr::prelude::PbrBundle;
+/// use bevy_render::pipeline::{RenderPipeline, RenderPipelines};
+///
+/// fn example_system(mut commands: Commands) {
+///     commands.spawn_bundle(PbrBundle {
+///         render_pipelines: RenderPipelines::from_pipelines(
+///             vec![RenderPipeline::new(SKINNED_MESH_PIPELINE_HANDLE.typed())]
+///         ),
+///         ..Default::default()
+///     }).insert(SkinnedMesh::new(
+///         // Refer to [`SkinnedMeshInverseBindposes`] example on how to create inverse bindposes data.
+///         Default::default(),
+///         // Specify joint entities here.
+///         vec![Entity::new(0)]
+///     ));
+/// }
+/// ```
+#[derive(Debug, Default, Clone, Reflect)]
+#[reflect(Component, MapEntities)]
+pub struct SkinnedMesh {
+    pub inverse_bindposes: Handle<SkinnedMeshInverseBindposes>,
+    pub joint_entities: Vec<Entity>,
+    pub joint_transforms: Vec<Mat4>,
+}
+
+impl SkinnedMesh {
+    pub fn new(
+        inverse_bindposes: Handle<SkinnedMeshInverseBindposes>,
+        joint_entities: Vec<Entity>,
+    ) -> Self {
+        let entities_count = joint_entities.len();
+
+        Self {
+            inverse_bindposes,
+            joint_entities,
+            joint_transforms: vec![Mat4::IDENTITY; entities_count],
+        }
+    }
+
+    pub fn update_joint_transforms(
+        &mut self,
+        inverse_bindposes_assets: &Res<Assets<SkinnedMeshInverseBindposes>>,
+        global_transform_query: &Query<&GlobalTransform>,
+    ) {
+        let inverse_bindposes = inverse_bindposes_assets
+            .get(self.inverse_bindposes.clone())
+            .unwrap();
+
+        for (joint_transform, (&joint_entity, &inverse_bindpose)) in self
+            .joint_transforms
+            .iter_mut()
+            .zip(self.joint_entities.iter().zip(inverse_bindposes.0.iter()))
+        {
+            let global_transform = global_transform_query.get(joint_entity).unwrap();
+            *joint_transform = global_transform.compute_matrix() * inverse_bindpose;
+        }
+    }
+}
+
+impl MapEntities for SkinnedMesh {
+    fn map_entities(&mut self, entity_map: &EntityMap) -> Result<(), MapEntitiesError> {
+        for entity in &mut self.joint_entities {
+            *entity = entity_map.get(*entity)?;
+        }
+
+        Ok(())
+    }
+}
+
+impl RenderResource for SkinnedMesh {
+    fn resource_type(&self) -> Option<RenderResourceType> {
+        Some(RenderResourceType::Buffer)
+    }
+
+    fn write_buffer_bytes(&self, buffer: &mut [u8]) {
+        let transform_size = std::mem::size_of::<[f32; 16]>();
+
+        for (index, transform) in self.joint_transforms.iter().enumerate() {
+            transform.write_buffer_bytes(
+                &mut buffer[index * transform_size..(index + 1) * transform_size],
+            );
+        }
+    }
+
+    fn buffer_byte_len(&self) -> Option<usize> {
+        Some(self.joint_transforms.len() * std::mem::size_of::<[f32; 16]>())
+    }
+
+    fn texture(&self) -> Option<&Handle<Texture>> {
+        None
+    }
+}
+
+impl RenderResources for SkinnedMesh {
+    fn render_resources_len(&self) -> usize {
+        1
+    }
+
+    fn get_render_resource(&self, index: usize) -> Option<&dyn RenderResource> {
+        (index == 0).then(|| self as &dyn RenderResource)
+    }
+
+    fn get_render_resource_name(&self, index: usize) -> Option<&str> {
+        (index == 0).then(|| "JointTransforms")
+    }
+
+    // Used to tell GLSL to use storage buffer instead of uniform buffer
+    fn get_render_resource_hints(&self, _index: usize) -> Option<RenderResourceHints> {
+        Some(RenderResourceHints::BUFFER)
+    }
+
+    fn iter(&self) -> RenderResourceIterator {
+        RenderResourceIterator::new(self)
+    }
+}
+
+/// Store joint inverse bindpose matrices. It can be shared between SkinnedMesh instances using assets.
+///
+/// The matrices can be loaded automatically from glTF or can be defined manually.
+///
+/// # Example
+/// ```
+/// use bevy_asset::Assets;
+/// use bevy_animation_rig::{SkinnedMesh, SkinnedMeshInverseBindposes, SKINNED_MESH_PIPELINE_HANDLE};
+/// use bevy_ecs::{entity::Entity, system::{Commands, ResMut}};
+/// use bevy_math::Mat4;
+/// use bevy_pbr::prelude::PbrBundle;
+/// use bevy_render::pipeline::{RenderPipeline, RenderPipelines};
+///
+/// fn example_system(mut commands: Commands, mut skinned_mesh_inverse_bindposes_assets: ResMut<Assets<SkinnedMeshInverseBindposes>>) {
+///     // A skeleton with only 2 joints
+///     let skinned_mesh_inverse_bindposes = skinned_mesh_inverse_bindposes_assets.add(SkinnedMeshInverseBindposes(vec![
+///         Mat4::IDENTITY,
+///         Mat4::IDENTITY,
+///     ]));
+///
+///     // The inverse bindposes then can be shared between multiple skinned mesh instances
+///     for _ in 0..3 {
+///         commands.spawn_bundle(PbrBundle {
+///             render_pipelines: RenderPipelines::from_pipelines(
+///                 vec![RenderPipeline::new(SKINNED_MESH_PIPELINE_HANDLE.typed())]
+///             ),
+///             ..Default::default()
+///         }).insert(SkinnedMesh::new(
+///             skinned_mesh_inverse_bindposes.clone(),
+///             // Remember to assign joint entity here!
+///             vec![Entity::new(0); 2],
+///         ));
+///     }
+/// }
+/// ```
+#[derive(Debug, TypeUuid)]
+#[uuid = "b9f155a9-54ec-4026-988f-e0a03e99a76f"]
+pub struct SkinnedMeshInverseBindposes(pub Vec<Mat4>);
+
+pub fn skinned_mesh_setup(
+    mut pipelines: ResMut<Assets<PipelineDescriptor>>,
+    mut shaders: ResMut<Assets<Shader>>,
+    mut render_graph: ResMut<RenderGraph>,
+) {
+    let mut skinned_mesh_pipeline = pipelines
+        .get(render_graph::PBR_PIPELINE_HANDLE)
+        .unwrap()
+        .clone();
+    skinned_mesh_pipeline.name = Some("Skinned Mesh Pipeline".into());
+    skinned_mesh_pipeline.shader_stages.vertex = shaders.add(Shader::from_glsl(
+        ShaderStage::Vertex,
+        include_str!("skinned_mesh.vert"),
+    ));
+    pipelines.set_untracked(SKINNED_MESH_PIPELINE_HANDLE, skinned_mesh_pipeline);
+
+    render_graph.add_system_node(
+        "JointTransforms",
+        RenderResourcesNode::<SkinnedMesh>::new(false),
+    );
+    render_graph
+        .add_node_edge("JointTransforms", node::MAIN_PASS)
+        .unwrap();
+}
+
+pub fn skinned_mesh_update(
+    skinned_mesh_inverse_bindposes_assets: Res<Assets<SkinnedMeshInverseBindposes>>,
+    global_transform_query: Query<&GlobalTransform>,
+    mut skinned_mesh_query: Query<&mut SkinnedMesh>,
+) {
+    skinned_mesh_query.for_each_mut(|mut skinned_mesh| {
+        skinned_mesh.update_joint_transforms(
+            &skinned_mesh_inverse_bindposes_assets,
+            &global_transform_query,
+        );
+    });
+}

crates/bevy_animation_rig/src/skinned_mesh.vert

@@ -0,0 +1,44 @@
+#version 450
+
+layout(location = 0) in vec3 Vertex_Position;
+layout(location = 1) in vec3 Vertex_Normal;
+layout(location = 2) in vec2 Vertex_Uv;
+layout(location = 3) in vec4 Vertex_JointWeight;
+layout(location = 4) in uvec4 Vertex_JointIndex;
+
+#ifdef STANDARDMATERIAL_NORMAL_MAP
+layout(location = 5) in vec4 Vertex_Tangent;
+#endif
+
+layout(location = 0) out vec3 v_WorldPosition;
+layout(location = 1) out vec3 v_WorldNormal;
+layout(location = 2) out vec2 v_Uv;
+
+layout(set = 0, binding = 0) uniform CameraViewProj {
+    mat4 ViewProj;
+};
+
+#ifdef STANDARDMATERIAL_NORMAL_MAP
+layout(location = 3) out vec4 v_WorldTangent;
+#endif
+
+layout(set = 2, binding = 1) buffer JointTransforms {
+    mat4[] Joints;
+};
+
+void main() {
+    mat4 Model =
+        Vertex_JointWeight.x * Joints[Vertex_JointIndex.x] +
+        Vertex_JointWeight.y * Joints[Vertex_JointIndex.y] +
+        Vertex_JointWeight.z * Joints[Vertex_JointIndex.z] +
+        Vertex_JointWeight.w * Joints[Vertex_JointIndex.w];
+
+    vec4 world_position = Model * vec4(Vertex_Position, 1.0);
+    v_WorldPosition = world_position.xyz;
+    v_WorldNormal = mat3(Model) * Vertex_Normal;
+    v_Uv = Vertex_Uv;
+#ifdef STANDARDMATERIAL_NORMAL_MAP
+    v_WorldTangent = vec4(mat3(Model) * Vertex_Tangent.xyz, Vertex_Tangent.w);
+#endif
+    gl_Position = ViewProj * world_position;
+}

crates/bevy_gltf/Cargo.toml

@@ -14,6 +14,7 @@ keywords = ["bevy"]
 
 [dependencies]
 # bevy
+bevy_animation_rig = { path = "../bevy_animation_rig", version = "0.5.0" }
 bevy_app = { path = "../bevy_app", version = "0.5.0" }
 bevy_asset = { path = "../bevy_asset", version = "0.5.0" }
 bevy_core = { path = "../bevy_core", version = "0.5.0" }