feat: support stylus pressure in macos

Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 description = "Use your iPad or Android tablet as graphic tablet."
 
 [dependencies]
-autopilot = { git = "https://github.com/H-M-H/autopilot-rs.git", rev = "63eed09c715bfb665bb23172a3930a528e11691c" }
+autopilot = { git = "https://github.com/lyonbot/autopilot-rs.git", rev = "a92b41edb4818d0d4b4289d49e65420dbe9ab081", version = "0.4.0" }
 bitflags = { version = "^2.6", features = ["serde"] }
 bytes = "1.7.1"
 clap = { version = "4.5.18", features = ["derive"] }

src/input/autopilot_device.rs

@@ -3,20 +3,30 @@ use autopilot::mouse;
 use autopilot::mouse::ScrollDirection;
 use autopilot::screen::size as screen_size;
 
-use tracing::warn;
+use tracing::{debug, warn};
 
 use crate::input::device::{InputDevice, InputDeviceType};
-use crate::protocol::{Button, KeyboardEvent, KeyboardEventType, PointerEvent, WheelEvent};
+use crate::protocol::{
+    Button, KeyboardEvent, KeyboardEventType, PointerEvent, PointerEventType, PointerType,
+    WheelEvent,
+};
 
 use crate::capturable::{Capturable, Geometry};
 
+#[cfg(target_os = "macos")]
+use super::macos_tablet::{MacosPenEventType, macos_send_tablet_event};
+
 pub struct AutoPilotDevice {
+    tablet_down: bool,
     capturable: Box<dyn Capturable>,
 }
 
 impl AutoPilotDevice {
     pub fn new(capturable: Box<dyn Capturable>) -> Self {
-        Self { capturable }
+        Self {
+            tablet_down: false,
+            capturable,
+        }
     }
 }
 
@@ -30,7 +40,7 @@ impl InputDevice for AutoPilotDevice {
     }
 
     fn send_pointer_event(&mut self, event: &PointerEvent) {
-        if !event.is_primary {
+        if !event.is_primary && event.pointer_type != PointerType::Pen {
             return;
         }
         if let Err(err) = self.capturable.before_input() {
@@ -55,12 +65,62 @@ impl InputDevice for AutoPilotDevice {
                 return;
             }
         };
-        if let Err(err) = mouse::move_to(autopilot::geometry::Point::new(
+
+        let point = autopilot::geometry::Point::new(
             (event.x * width_rel + x_rel) * width,
             (event.y * height_rel + y_rel) * height,
-        )) {
+        );
+
+        // MacOS only: send tablet (stylus) events
+        #[cfg(target_os = "macos")]
+        if event.pointer_type == PointerType::Pen {
+            let pe_type = match event.event_type {
+                PointerEventType::DOWN => MacosPenEventType::Down,
+                PointerEventType::UP => MacosPenEventType::Up,
+                PointerEventType::CANCEL => MacosPenEventType::Up,
+                PointerEventType::ENTER => MacosPenEventType::Enter,
+                PointerEventType::LEAVE => MacosPenEventType::Leave,
+                _ => MacosPenEventType::Move,
+            };
+
+            match event.event_type {
+                PointerEventType::DOWN => {
+                    self.tablet_down = true;
+                }
+                PointerEventType::CANCEL | PointerEventType::UP | PointerEventType::LEAVE => {
+                    self.tablet_down = false;
+                }
+                _ => (),
+            }
+
+            match event.event_type {
+                PointerEventType::ENTER => {
+                    debug!("Entering tablet");
+                }
+                PointerEventType::LEAVE => {
+                    debug!("Leaving tablet");
+                }
+                _ => (),
+            }
+
+            let buttons = if self.tablet_down { 1 } else { 0 };
+            if let Err(err) = macos_send_tablet_event(
+                point,
+                pe_type,
+                event.button.bits().into(),
+                buttons,
+                event.pressure,
+            ) {
+                warn!("Could not send pressure: {}", err);
+            }
+
+            return;
+        }
+
+        if let Err(err) = mouse::move_to(point) {
             warn!("Could not move mouse: {}", err);
         }
+
         match event.button {
             Button::PRIMARY => {
                 mouse::toggle(mouse::Button::Left, event.buttons.contains(event.button))

src/input/macos_tablet.rs

@@ -0,0 +1,224 @@
+// References:
+//
+// 1. MacOS Declarations, see /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/IOKit.framework/Headers
+//   IOKit/hidsystem/IOLLEvent.h
+//
+// 2. GitHub project "TabletMagic"
+//    https://github.com/thinkyhead/TabletMagic/blob/master/daemon/SerialDaemon.cpp
+
+use core_graphics::base::CGFloat;
+use core_graphics::geometry::CGPoint;
+use core_graphics::event::*;
+use core_graphics::event_source::CGEventSourceStateID::HIDSystemState;
+use core_graphics::event_source::CGEventSource;
+
+use autopilot::geometry::Point;
+use autopilot::mouse::MouseError;
+
+pub const NX_SUBTYPE_DEFAULT: i64 = 0;
+pub const NX_SUBTYPE_TABLET_POINT: i64 = 1;
+pub const NX_SUBTYPE_TABLET_PROXIMITY: i64 = 2;
+pub const NX_SUBTYPE_MOUSE_TOUCH: i64 = 3;
+
+const NX_TABLET_POINTER_UNKNOWN: i64 = 0;
+const NX_TABLET_POINTER_PEN: i64 = 1;
+const NX_TABLET_POINTER_CURSOR: i64 = 2;
+const NX_TABLET_POINTER_ERASER: i64 = 3;
+
+const NX_TABLET_CAPABILITY_DEVICEIDMASK: i64 = 0x0001;
+const NX_TABLET_CAPABILITY_ABSXMASK: i64 = 0x0002;
+const NX_TABLET_CAPABILITY_ABSYMASK: i64 = 0x0004;
+const NX_TABLET_CAPABILITY_VENDOR1MASK: i64 = 0x0008;
+const NX_TABLET_CAPABILITY_VENDOR2MASK: i64 = 0x0010;
+const NX_TABLET_CAPABILITY_VENDOR3MASK: i64 = 0x0020;
+const NX_TABLET_CAPABILITY_BUTTONSMASK: i64 = 0x0040;
+const NX_TABLET_CAPABILITY_TILTXMASK: i64 = 0x0080;
+const NX_TABLET_CAPABILITY_TILTYMASK: i64 = 0x0100;
+const NX_TABLET_CAPABILITY_ABSZMASK: i64 = 0x0200;
+const NX_TABLET_CAPABILITY_PRESSUREMASK: i64 = 0x0400;
+const NX_TABLET_CAPABILITY_TANGENTIALPRESSUREMASK: i64 = 0x0800;
+const NX_TABLET_CAPABILITY_ORIENTINFOMASK: i64 = 0x1000;
+const NX_TABLET_CAPABILITY_ROTATIONMASK: i64 = 0x2000;
+
+// mock
+const MOCK_VENDOR_UNIQUE_ID: i64 = 0xdeadbeef;
+const MOCK_DEVICE_ID: i64 = 0x81; // just a single device for now
+fn populate_tablet_point_event(event: &CGEvent, buttons: i64, point: Point, pressure: f64) {
+    // note:
+    // 1. assume subtype already set.
+
+    // CGEventSetIntegerValueField(move1, kCGTabletEventPointX, stylus.point.x);
+    // CGEventSetIntegerValueField(move1, kCGTabletEventPointY, stylus.point.y);
+    // CGEventSetIntegerValueField(move1, kCGTabletEventPointButtons, 0x0000);
+    // CGEventSetDoubleValueField(move1, kCGTabletEventPointPressure, stylus.pressure / PRESSURE_SCALE);
+    // CGEventSetDoubleValueField(move1, kCGTabletEventTiltX, stylus.tilt.x);
+    // CGEventSetDoubleValueField(move1, kCGTabletEventTiltY, stylus.tilt.y);
+    // CGEventSetIntegerValueField(move1, kCGTabletEventDeviceID, stylus.proximity.deviceID);
+    // CGEventSetIntegerValueField(move1, kCGTabletEventPointZ, 0);
+    // CGEventSetDoubleValueField(move1, kCGTabletEventRotation, 0);
+    // CGEventSetDoubleValueField(move1, kCGTabletEventTangentialPressure, 0);
+
+    event.set_integer_value_field(EventField::TABLET_EVENT_POINT_X, point.x as i64);
+    event.set_integer_value_field(EventField::TABLET_EVENT_POINT_Y, point.y as i64);
+    event.set_integer_value_field(EventField::TABLET_EVENT_POINT_BUTTONS, buttons);
+    event.set_double_value_field(EventField::TABLET_EVENT_POINT_PRESSURE, pressure);
+    event.set_double_value_field(EventField::TABLET_EVENT_TILT_X, 0.0);     // tilt is yet zero
+    event.set_double_value_field(EventField::TABLET_EVENT_TILT_Y, 0.0);
+    event.set_integer_value_field(EventField::TABLET_EVENT_DEVICE_ID, MOCK_DEVICE_ID);
+    event.set_integer_value_field(EventField::TABLET_EVENT_POINT_Z, 0);
+    event.set_double_value_field(EventField::TABLET_EVENT_ROTATION, 0.0);  // yet not rotated
+    event.set_double_value_field(EventField::TABLET_EVENT_TANGENTIAL_PRESSURE, 0.0);
+
+    event.set_double_value_field(EventField::MOUSE_EVENT_PRESSURE, pressure);
+}
+
+fn populate_tablet_proximity_event(event: &CGEvent, enter_tablet: bool, is_enter_eraser: bool) {
+    // note:
+    // 1. assume subtype already set.
+
+    event.set_integer_value_field(
+        EventField::TABLET_PROXIMITY_EVENT_ENTER_PROXIMITY,
+        enter_tablet as i64,
+    );
+    event.set_integer_value_field(
+        EventField::TABLET_PROXIMITY_EVENT_POINTER_TYPE,
+        match is_enter_eraser {
+            true => NX_TABLET_POINTER_ERASER,
+            false => NX_TABLET_POINTER_PEN,
+        },
+    );
+
+    event.set_integer_value_field(EventField::TABLET_PROXIMITY_EVENT_VENDOR_ID, 0xbeef); // A made-up Vendor ID (Wacom's is 0x056A)
+    event.set_integer_value_field(EventField::TABLET_PROXIMITY_EVENT_TABLET_ID, 1);
+    event.set_integer_value_field(EventField::TABLET_PROXIMITY_EVENT_DEVICE_ID, MOCK_DEVICE_ID);
+    event.set_integer_value_field(EventField::TABLET_PROXIMITY_EVENT_POINTER_ID, 0);
+    event.set_integer_value_field(EventField::TABLET_PROXIMITY_EVENT_SYSTEM_TABLET_ID, 0);
+    event.set_integer_value_field(
+        EventField::TABLET_PROXIMITY_EVENT_VENDOR_POINTER_TYPE,
+        0x0802,
+    ); // basic stylus
+    event.set_integer_value_field(
+        EventField::TABLET_PROXIMITY_EVENT_VENDOR_POINTER_SERIAL_NUMBER,
+        1,
+    );
+    event.set_integer_value_field(
+        EventField::TABLET_PROXIMITY_EVENT_VENDOR_UNIQUE_ID,
+        MOCK_VENDOR_UNIQUE_ID,
+    );
+
+    // Indicate which fields in the point event contain valid data. This allows
+    // applications to handle devices with varying capabilities.
+    let capability_mask = NX_TABLET_CAPABILITY_DEVICEIDMASK
+        | NX_TABLET_CAPABILITY_ABSXMASK
+        | NX_TABLET_CAPABILITY_ABSYMASK
+        | NX_TABLET_CAPABILITY_BUTTONSMASK
+        | NX_TABLET_CAPABILITY_TILTXMASK
+        | NX_TABLET_CAPABILITY_TILTYMASK
+        | NX_TABLET_CAPABILITY_PRESSUREMASK;
+    //      |   NX_TABLET_CAPABILITY_TANGENTIALPRESSUREMASK
+    //      |   NX_TABLET_CAPABILITY_ORIENTINFOMASK
+    //      |   NX_TABLET_CAPABILITY_ROTATIONMASK
+
+    event.set_integer_value_field(
+        EventField::TABLET_PROXIMITY_EVENT_CAPABILITY_MASK,
+        capability_mask,
+    );
+}
+
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum MacosPenEventType {
+    Move,
+    Down,
+    Up,
+    Enter, // proximity enter
+    Leave, // proximity leave
+}
+
+const ERASER_BUTTON: i64 = 32;
+
+#[cfg(target_os = "macos")]
+pub fn macos_send_tablet_event(
+    point: Point,
+    pe_type: MacosPenEventType,
+    button: i64,
+    buttons: i64,
+    pressure: f64,
+) -> Result<(), MouseError> {
+
+
+    let make_event = |event_type: CGEventType| {
+        let source = CGEventSource::new(HIDSystemState).unwrap();
+        let event = CGEvent::new_mouse_event(
+            source,
+            event_type,
+            CGPoint::new(point.x as CGFloat, point.y as CGFloat),
+            CGMouseButton::Left,
+        );
+
+        return event;
+    };
+
+    // let is_eraser = button == ERASER_BUTTON || (buttons & ERASER_BUTTON) != 0;
+
+    match pe_type {
+        MacosPenEventType::Enter | MacosPenEventType::Leave => {
+            // send proximity event
+            let event = make_event(CGEventType::MouseMoved).unwrap();
+            event.set_type(CGEventType::TabletProximity);
+            event.set_integer_value_field(
+                EventField::MOUSE_EVENT_SUB_TYPE,
+                NX_SUBTYPE_TABLET_PROXIMITY,
+            );
+            populate_tablet_proximity_event(&event, pe_type == MacosPenEventType::Enter, false);
+
+            event.post(CGEventTapLocation::HID);
+        },
+        _ => {
+            // then send a MouseMoved event
+            let event_type: CGEventType = match pe_type {
+                MacosPenEventType::Down => match button {
+                    1 => CGEventType::LeftMouseDown,
+                    2 => CGEventType::RightMouseDown,
+                    _ => CGEventType::OtherMouseDown, // eg: 32 for eraser button
+                },
+                MacosPenEventType::Up => match button {
+                    1 => CGEventType::LeftMouseUp,
+                    2 => CGEventType::RightMouseUp,
+                    _ => CGEventType::OtherMouseUp,
+                },
+                _ => match buttons {
+                    0 => CGEventType::MouseMoved,
+                    1 => CGEventType::LeftMouseDragged,
+                    2 => CGEventType::RightMouseDragged,
+                    _ => CGEventType::OtherMouseDragged,
+                },
+            };
+
+            let event = make_event(event_type).unwrap();
+            event.set_double_value_field(EventField::MOUSE_EVENT_PRESSURE, pressure);
+            event.set_integer_value_field(
+                EventField::MOUSE_EVENT_SUB_TYPE,
+                NX_SUBTYPE_TABLET_POINT,
+            );
+            populate_tablet_proximity_event(&event, true, false);
+            populate_tablet_point_event(&event, buttons, point, pressure);
+            event.post(CGEventTapLocation::HID);
+        },
+    }
+
+    // if pe_type == PressureEventType::Down || pe_type == PressureEventType::Up {
+    //     // when pointer down or up, use TabletProximity event to notify the OS
+    // } else {
+    //     // regular daily notify
+
+    //     let event = make_event(CGEventType::MouseMoved).unwrap();
+    //     event.set_type(CGEventType::TabletPointer);
+    //     event.set_integer_value_field(EventField::MOUSE_EVENT_SUB_TYPE, 1); // 1 for https://developer.apple.com/documentation/coregraphics/cgeventmousesubtype/tabletpoint
+    //     populate_tablet_point_event(&event, point, pressure);
+
+    //     event.post(CGEventTapLocation::HID);
+    // }
+
+    Ok(())
+}

src/input/mod.rs

@@ -8,3 +8,6 @@ pub mod uinput_device;
 #[cfg(target_os = "linux")]
 #[allow(dead_code)]
 pub mod uinput_keys;
+
+#[cfg(target_os = "macos")]
+mod macos_tablet;

src/protocol.rs

@@ -60,7 +60,7 @@ pub struct CustomInputAreas {
     pub pen: Option<Rect>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
 pub enum PointerType {
     #[serde(rename = "")]
     Unknown,
@@ -72,7 +72,7 @@ pub enum PointerType {
     Touch,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)]
 pub enum PointerEventType {
     #[serde(rename = "pointerdown")]
     DOWN,

ts/lib.ts

@@ -728,6 +728,8 @@ class PointerHandler {
             canvas.onpointerenter = (e) => this.onEvent(e, "pointerenter");
             canvas.onpointerover = (e) => this.onEvent(e, "pointerover");
         }
+        canvas.onpointerenter = (e) => this.onEvent(e, "pointerenter");
+        canvas.onpointerleave = (e) => this.onEvent(e, "pointerleave");
 
         // This is a workaround for the following Safari/WebKit bug:
         // https://bugs.webkit.org/show_bug.cgi?id=217430