Add custom cursors (#14284)

# Objective

- Add custom images as cursors
- Fixes #9557 

## Solution

- Change cursor type to accommodate both native and image cursors
- I don't really like this solution because I couldn't use
`Handle<Image>` directly. I would need to import `bevy_assets` and that
causes a circular dependency. Alternatively we could use winit's
`CustomCursor` smart pointers, but that seems hard because the event
loop is needed to create those and is not easily accessable for users.
So now I need to copy around rgba buffers which is sad.
- I use a cache because especially on the web creating cursor images is
really slow
- Sorry to #14196 for yoinking, I just wanted to make a quick solution
for myself and thought that I should probably share it too.

Update:
- Now uses `Handle<Image>`, reads rgba data in `bevy_render` and uses
resources to send the data to `bevy_winit`, where the final cursors are
created.

## Testing

- Added example which works fine at least on Linux Wayland (winit side
has been tested with all platforms).
- I haven't tested if the url cursor works.

## Migration Guide

- `CursorIcon` is no longer a field in `Window`, but a separate
component can be inserted to a window entity. It has been changed to an
enum that can hold custom images in addition to system icons.
- `Cursor` is renamed to `CursorOptions` and `cursor` field of `Window`
is renamed to `cursor_options`
- `CursorIcon` is renamed to `SystemCursorIcon`

---------

Co-authored-by: Alice Cecile <[email protected]>
Co-authored-by: Jan Hohenheim <[email protected]>

Author: 3 people

crates/bevy_render/Cargo.toml

@@ -58,6 +58,7 @@ bevy_render_macros = { path = "macros", version = "0.15.0-dev" }
 bevy_time = { path = "../bevy_time", version = "0.15.0-dev" }
 bevy_transform = { path = "../bevy_transform", version = "0.15.0-dev" }
 bevy_window = { path = "../bevy_window", version = "0.15.0-dev" }
+bevy_winit = { path = "../bevy_winit", version = "0.15.0-dev" }
 bevy_utils = { path = "../bevy_utils", version = "0.15.0-dev" }
 bevy_tasks = { path = "../bevy_tasks", version = "0.15.0-dev" }
 

crates/bevy_render/src/view/window/cursor.rs

@@ -0,0 +1,175 @@
+use bevy_asset::{AssetId, Assets, Handle};
+use bevy_ecs::{
+    change_detection::DetectChanges,
+    component::Component,
+    entity::Entity,
+    query::With,
+    reflect::ReflectComponent,
+    system::{Commands, Local, Query, Res},
+    world::Ref,
+};
+use bevy_reflect::{std_traits::ReflectDefault, Reflect};
+use bevy_utils::{tracing::warn, HashSet};
+use bevy_window::{SystemCursorIcon, Window};
+use bevy_winit::{
+    convert_system_cursor_icon, CursorSource, CustomCursorCache, CustomCursorCacheKey,
+    PendingCursor,
+};
+use wgpu::TextureFormat;
+
+use crate::prelude::Image;
+
+/// Insert into a window entity to set the cursor for that window.
+#[derive(Component, Debug, Clone, Reflect, PartialEq, Eq)]
+#[reflect(Component, Debug, Default)]
+pub enum CursorIcon {
+    /// Custom cursor image.
+    Custom(CustomCursor),
+    /// System provided cursor icon.
+    System(SystemCursorIcon),
+}
+
+impl Default for CursorIcon {
+    fn default() -> Self {
+        CursorIcon::System(Default::default())
+    }
+}
+
+impl From<SystemCursorIcon> for CursorIcon {
+    fn from(icon: SystemCursorIcon) -> Self {
+        CursorIcon::System(icon)
+    }
+}
+
+impl From<CustomCursor> for CursorIcon {
+    fn from(cursor: CustomCursor) -> Self {
+        CursorIcon::Custom(cursor)
+    }
+}
+
+/// Custom cursor image data.
+#[derive(Debug, Clone, Reflect, PartialEq, Eq, Hash)]
+pub enum CustomCursor {
+    /// Image to use as a cursor.
+    Image {
+        /// The image must be in 8 bit int or 32 bit float rgba. PNG images
+        /// work well for this.
+        handle: Handle<Image>,
+        /// X and Y coordinates of the hotspot in pixels. The hotspot must be
+        /// within the image bounds.
+        hotspot: (u16, u16),
+    },
+    #[cfg(all(target_family = "wasm", target_os = "unknown"))]
+    /// A URL to an image to use as the cursor.
+    Url {
+        /// Web URL to an image to use as the cursor. PNGs preferred. Cursor
+        /// creation can fail if the image is invalid or not reachable.
+        url: String,
+        /// X and Y coordinates of the hotspot in pixels. The hotspot must be
+        /// within the image bounds.
+        hotspot: (u16, u16),
+    },
+}
+
+pub fn update_cursors(
+    mut commands: Commands,
+    mut windows: Query<(Entity, Ref<CursorIcon>), With<Window>>,
+    cursor_cache: Res<CustomCursorCache>,
+    images: Res<Assets<Image>>,
+    mut queue: Local<HashSet<Entity>>,
+) {
+    for (entity, cursor) in windows.iter_mut() {
+        if !(queue.remove(&entity) || cursor.is_changed()) {
+            continue;
+        }
+
+        let cursor_source = match cursor.as_ref() {
+            CursorIcon::Custom(CustomCursor::Image { handle, hotspot }) => {
+                let cache_key = match handle.id() {
+                    AssetId::Index { index, .. } => {
+                        CustomCursorCacheKey::AssetIndex(index.to_bits())
+                    }
+                    AssetId::Uuid { uuid } => CustomCursorCacheKey::AssetUuid(uuid.as_u128()),
+                };
+
+                if cursor_cache.0.contains_key(&cache_key) {
+                    CursorSource::CustomCached(cache_key)
+                } else {
+                    let Some(image) = images.get(handle) else {
+                        warn!(
+                            "Cursor image {handle:?} is not loaded yet and couldn't be used. Trying again next frame."
+                        );
+                        queue.insert(entity);
+                        continue;
+                    };
+                    let Some(rgba) = image_to_rgba_pixels(image) else {
+                        warn!("Cursor image {handle:?} not accepted because it's not rgba8 or rgba32float format");
+                        continue;
+                    };
+
+                    let width = image.texture_descriptor.size.width;
+                    let height = image.texture_descriptor.size.height;
+                    let source = match bevy_winit::WinitCustomCursor::from_rgba(
+                        rgba,
+                        width as u16,
+                        height as u16,
+                        hotspot.0,
+                        hotspot.1,
+                    ) {
+                        Ok(source) => source,
+                        Err(err) => {
+                            warn!("Cursor image {handle:?} is invalid: {err}");
+                            continue;
+                        }
+                    };
+
+                    CursorSource::Custom((cache_key, source))
+                }
+            }
+            #[cfg(all(target_family = "wasm", target_os = "unknown"))]
+            CursorIcon::Custom(CustomCursor::Url { url, hotspot }) => {
+                let cache_key = CustomCursorCacheKey::Url(url.clone());
+
+                if cursor_cache.0.contains_key(&cache_key) {
+                    CursorSource::CustomCached(cache_key)
+                } else {
+                    use bevy_winit::CustomCursorExtWebSys;
+                    let source =
+                        bevy_winit::WinitCustomCursor::from_url(url.clone(), hotspot.0, hotspot.1);
+                    CursorSource::Custom((cache_key, source))
+                }
+            }
+            CursorIcon::System(system_cursor_icon) => {
+                CursorSource::System(convert_system_cursor_icon(*system_cursor_icon))
+            }
+        };
+
+        commands
+            .entity(entity)
+            .insert(PendingCursor(Some(cursor_source)));
+    }
+}
+
+/// Returns the image data as a `Vec<u8>`.
+/// Only supports rgba8 and rgba32float formats.
+fn image_to_rgba_pixels(image: &Image) -> Option<Vec<u8>> {
+    match image.texture_descriptor.format {
+        TextureFormat::Rgba8Unorm
+        | TextureFormat::Rgba8UnormSrgb
+        | TextureFormat::Rgba8Snorm
+        | TextureFormat::Rgba8Uint
+        | TextureFormat::Rgba8Sint => Some(image.data.clone()),
+        TextureFormat::Rgba32Float => Some(
+            image
+                .data
+                .chunks(4)
+                .map(|chunk| {
+                    let chunk = chunk.try_into().unwrap();
+                    let num = bytemuck::cast_ref::<[u8; 4], f32>(chunk);
+                    (num * 255.0) as u8
+                })
+                .collect(),
+        ),
+        _ => None,
+    }
+}

crates/bevy_render/src/view/window/mod.rs

@@ -6,14 +6,15 @@ use crate::{
     texture::TextureFormatPixelInfo,
     Extract, ExtractSchedule, Render, RenderApp, RenderSet, WgpuWrapper,
 };
-use bevy_app::{App, Plugin};
+use bevy_app::{App, Last, Plugin};
 use bevy_ecs::{entity::EntityHashMap, prelude::*};
 #[cfg(target_os = "linux")]
 use bevy_utils::warn_once;
 use bevy_utils::{default, tracing::debug, HashSet};
 use bevy_window::{
     CompositeAlphaMode, PresentMode, PrimaryWindow, RawHandleWrapper, Window, WindowClosing,
 };
+use bevy_winit::CustomCursorCache;
 use std::{
     num::NonZeroU32,
     ops::{Deref, DerefMut},
@@ -24,17 +25,22 @@ use wgpu::{
     TextureViewDescriptor,
 };
 
+pub mod cursor;
 pub mod screenshot;
 
 use screenshot::{
     ScreenshotManager, ScreenshotPlugin, ScreenshotPreparedState, ScreenshotToScreenPipeline,
 };
 
+use self::cursor::update_cursors;
+
 pub struct WindowRenderPlugin;
 
 impl Plugin for WindowRenderPlugin {
     fn build(&self, app: &mut App) {
-        app.add_plugins(ScreenshotPlugin);
+        app.add_plugins(ScreenshotPlugin)
+            .init_resource::<CustomCursorCache>()
+            .add_systems(Last, update_cursors);
 
         if let Some(render_app) = app.get_sub_app_mut(RenderApp) {
             render_app

crates/bevy_window/src/lib.rs

@@ -15,27 +15,27 @@ use std::sync::{Arc, Mutex};
 
 use bevy_a11y::Focus;
 
-mod cursor;
 mod event;
 mod monitor;
 mod raw_handle;
 mod system;
+mod system_cursor;
 mod window;
 
 pub use crate::raw_handle::*;
 
-pub use cursor::*;
 pub use event::*;
 pub use monitor::*;
 pub use system::*;
+pub use system_cursor::*;
 pub use window::*;
 
 #[allow(missing_docs)]
 pub mod prelude {
     #[allow(deprecated)]
     #[doc(hidden)]
     pub use crate::{
-        CursorEntered, CursorIcon, CursorLeft, CursorMoved, FileDragAndDrop, Ime, MonitorSelection,
+        CursorEntered, CursorLeft, CursorMoved, FileDragAndDrop, Ime, MonitorSelection,
         ReceivedCharacter, Window, WindowMoved, WindowPlugin, WindowPosition,
         WindowResizeConstraints,
     };

crates/bevy_window/src/cursor.rs renamed to crates/bevy_window/src/system_cursor.rs

@@ -73,7 +73,7 @@ use bevy_reflect::{prelude::ReflectDefault, Reflect};
 #[cfg(feature = "serialize")]
 use bevy_reflect::{ReflectDeserialize, ReflectSerialize};
 
-/// The icon to display for a [`Window`](crate::window::Window)'s [`Cursor`](crate::window::Cursor).
+/// The icon to display for a window.
 ///
 /// Examples of all of these cursors can be found [here](https://www.w3schools.com/cssref/playit.php?filename=playcss_cursor&preval=crosshair).
 /// This `enum` is simply a copy of a similar `enum` found in [`winit`](https://docs.rs/winit/latest/winit/window/enum.CursorIcon.html).
@@ -89,7 +89,7 @@ use bevy_reflect::{ReflectDeserialize, ReflectSerialize};
     reflect(Serialize, Deserialize)
 )]
 #[reflect(Debug, PartialEq, Default)]
-pub enum CursorIcon {
+pub enum SystemCursorIcon {
     /// The platform-dependent default cursor. Often rendered as arrow.
     #[default]
     Default,
@@ -107,7 +107,7 @@ pub enum CursorIcon {
     Pointer,
 
     /// A progress indicator. The program is performing some processing, but is
-    /// different from [`CursorIcon::Wait`] in that the user may still interact
+    /// different from [`SystemCursorIcon::Wait`] in that the user may still interact
     /// with the program.
     Progress,
 

crates/bevy_window/src/window.rs

@@ -12,8 +12,6 @@ use bevy_reflect::{ReflectDeserialize, ReflectSerialize};
 
 use bevy_utils::tracing::warn;
 
-use crate::CursorIcon;
-
 /// Marker [`Component`] for the window considered the primary window.
 ///
 /// Currently this is assumed to only exist on 1 entity at a time.
@@ -107,16 +105,16 @@ impl NormalizedWindowRef {
 ///
 /// Because this component is synchronized with `winit`, it can be used to perform
 /// OS-integrated windowing operations. For example, here's a simple system
-/// to change the cursor type:
+/// to change the window mode:
 ///
 /// ```
 /// # use bevy_ecs::query::With;
 /// # use bevy_ecs::system::Query;
-/// # use bevy_window::{CursorIcon, PrimaryWindow, Window};
-/// fn change_cursor(mut windows: Query<&mut Window, With<PrimaryWindow>>) {
+/// # use bevy_window::{WindowMode, PrimaryWindow, Window, MonitorSelection};
+/// fn change_window_mode(mut windows: Query<&mut Window, With<PrimaryWindow>>) {
 ///     // Query returns one window typically.
 ///     for mut window in windows.iter_mut() {
-///         window.cursor.icon = CursorIcon::Wait;
+///         window.mode = WindowMode::Fullscreen(MonitorSelection::Current);
 ///     }
 /// }
 /// ```
@@ -128,8 +126,9 @@ impl NormalizedWindowRef {
 )]
 #[reflect(Component, Default)]
 pub struct Window {
-    /// The cursor of this window.
-    pub cursor: Cursor,
+    /// The cursor options of this window. Cursor icons are set with the `Cursor` component on the
+    /// window entity.
+    pub cursor_options: CursorOptions,
     /// What presentation mode to give the window.
     pub present_mode: PresentMode,
     /// Which fullscreen or windowing mode should be used.
@@ -316,7 +315,7 @@ impl Default for Window {
         Self {
             title: "App".to_owned(),
             name: None,
-            cursor: Default::default(),
+            cursor_options: Default::default(),
             present_mode: Default::default(),
             mode: Default::default(),
             position: Default::default(),
@@ -543,23 +542,20 @@ impl WindowResizeConstraints {
 }
 
 /// Cursor data for a [`Window`].
-#[derive(Debug, Copy, Clone, Reflect)]
+#[derive(Debug, Clone, Reflect)]
 #[cfg_attr(
     feature = "serialize",
     derive(serde::Serialize, serde::Deserialize),
     reflect(Serialize, Deserialize)
 )]
 #[reflect(Debug, Default)]
-pub struct Cursor {
-    /// What the cursor should look like while inside the window.
-    pub icon: CursorIcon,
-
+pub struct CursorOptions {
     /// Whether the cursor is visible or not.
     ///
     /// ## Platform-specific
     ///
     /// - **`Windows`**, **`X11`**, and **`Wayland`**: The cursor is hidden only when inside the window.
-    ///     To stop the cursor from leaving the window, change [`Cursor::grab_mode`] to [`CursorGrabMode::Locked`] or [`CursorGrabMode::Confined`]
+    ///     To stop the cursor from leaving the window, change [`CursorOptions::grab_mode`] to [`CursorGrabMode::Locked`] or [`CursorGrabMode::Confined`]
     /// - **`macOS`**: The cursor is hidden only when the window is focused.
     /// - **`iOS`** and **`Android`** do not have cursors
     pub visible: bool,
@@ -583,10 +579,9 @@ pub struct Cursor {
     pub hit_test: bool,
 }
 
-impl Default for Cursor {
+impl Default for CursorOptions {
     fn default() -> Self {
-        Cursor {
-            icon: CursorIcon::Default,
+        CursorOptions {
             visible: true,
             grab_mode: CursorGrabMode::None,
             hit_test: true,
@@ -870,7 +865,7 @@ impl From<DVec2> for WindowResolution {
     }
 }
 
-/// Defines if and how the [`Cursor`] is grabbed by a [`Window`].
+/// Defines if and how the cursor is grabbed by a [`Window`].
 ///
 /// ## Platform-specific
 ///