refactored client, with mouse_win/mac/nix

Author: philipbrocoum-ww

.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+.vscode

client_win-mac-nix/main.py

@@ -1,75 +1,35 @@
 import argparse
 import logging
 from time import time, ctime
-from dataclasses import dataclass
 from collections import deque  # for storing x, y time series
-import numpy as np  # for smoothing moving average
-import ctypes  # for windows mouse POINT struct
 import socket  # udp networking
 import struct  # binary unpacking
-# import keyboard # keyboard shortcuts
+from pynput import keyboard # for hotkeys
 
-from pynput import keyboard
-
-
-
-
-# Collect events until released
-# with keyboard.Listener(
-#         on_press=on_press,
-#         on_release=on_release) as listener:
-#     listener.join()
-
-# ...or, in a non-blocking fashion:
-
-
-enabled = True
-
-def toggle():
-    global enabled
-    logging.info("\nToggled PhilNav on/off\n")
-    enabled = not enabled
-
-# https://pynput.readthedocs.io/en/latest/keyboard.html#global-hotkeys
-
-def for_canonical(f):
-    return lambda k: f(listener.canonical(k))
-
-hotkey = keyboard.HotKey(
-    [keyboard.Key.shift, keyboard.Key.f7.value],
-    toggle)
-
-listener = keyboard.Listener(
-        on_press=for_canonical(hotkey.press),
-        on_release=for_canonical(hotkey.release))
-
-listener.start()
-
-# keyboard.add_hotkey('space', toggle)
-# keyboard.add_hotkey('space', toggle)
-
-# keyboard.write('The quick brown fox jumps over the lazy dog.')
-# keyboard.add_hotkey('space', lambda: print('space was pressed!'))
-
-# returned from ctypes.windll.user32.GetCursorPos
-# simple point.x, point.y
-class POINT(ctypes.Structure):
-    _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)]
+print("\n\nCLIENT: Starting PhilNav\n")
 
 import platform
-# Done: Windows
-# TODO: Mac, Linux
 match platform.system():
     case "Darwin": # macOS
         from mouse_mac import getCursorPos, setCursorPos
+    case "Windows":
+        from mouse_win import getCursorPos, setCursorPos
+    case "Linux":
+        from mouse_nix import getCursorPos, setCursorPos
+    case _:
+        raise RuntimeError(f"Platform {platform.system()} not supported (not Win, Mac, or Nix)")
 
 
-print("\n\nCLIENT: Starting PhilNav\n")
-
 # parse command line arguments
 parser = argparse.ArgumentParser()
 parser.add_argument(
-    "-v", "--verbose", action="store_true", help="provide verbose logging"
+    "-v", "--verbose", action="store_true", help="enable verbose logging"
+)
+parser.add_argument(
+    "-s", "--speed", type=int, default=25, help="mouse speed, default 25"
+)
+parser.add_argument(
+    "-S", "--smooth", type=int, default=3, help="averages mouse movements to smooth out jittering, default 3"
 )
 parser.add_argument(
     "-H",
@@ -81,22 +41,40 @@ class POINT(ctypes.Structure):
 parser.add_argument(
     "-p", "--port", type=int, default=4245, help="bind to port, default 4245"
 )
-parser.add_argument(
-    "-s", "--speed", type=int, default=25, help="mouse speed, default 25"
-)
-parser.add_argument(
-    "-S", "--smooth", type=int, default=3, help="averages mouse movements to smooth out jittering, default 3"
-)
+
 args = parser.parse_args()
+
 if args.smooth < 1:
     args.smooth = 1
 
-
 if args.verbose:
     logging.getLogger().setLevel(logging.DEBUG)
-    logging.info(" Logging verbosely\n")
+    logging.info("\nLogging verbosely\n")
+
+
+# Hotkey to pause/resume moving the mouse
+enabled = True
 
+def toggle():
+    global enabled
+    enabled = not enabled
+    logging.info("\nToggled PhilNav on/off\n")
+
+# Shift-F7 hard-coded for now
+hotkey_toggle = keyboard.HotKey(
+    [keyboard.Key.shift, keyboard.Key.f7.value],
+    toggle)
+
+# https://pynput.readthedocs.io/en/latest/keyboard.html#global-hotkeys
+def for_canonical(f):
+    global listener
+    return lambda k: f(listener.canonical(k))
+
+listener = keyboard.Listener(
+        on_press=for_canonical(hotkey_toggle.press),
+        on_release=for_canonical(hotkey_toggle.release))
 
+listener.start()
 
 
 # initialize networking
@@ -106,41 +84,37 @@ class POINT(ctypes.Structure):
 sock.settimeout(1)
 sock.bind((args.host, args.port))  # Register our socket
 
-# How to get local IP address?
-# Doesn't work for me: socket.gethostbyname(socket.gethostname())
-# For now, just manually go to settings -> Wi-fi and look up your *local* IP
-# address. It will be something like 192.x.x.x or 10.x.x.x This is not your
-# public Internet address. This is your local area network address. On Windows,
-# make sure it's set to a private network to allow discovery. You'd think public
-# would allow discovery, but when you are *in public* - like at a coffee shop -
-# you don't want strangers to access your PC.
+# How to get local IP address in python?
 text_listening = (
     f"Listening on {sock.getsockname()} for mouse data from Raspberry Pi server..."
 )
 print(ctime() + " - " + text_listening)
-print("\nPress Ctrl-C to exit\n")
+print("\nPress Ctrl-C to exit, press Shift-F7 to pause/resume\n")
 
 
 # Stats for debugging & performance. The goal is 60 frames per second, or
 # 16.67ms per frame. That leads to a very smooth mouse cursor. (SmartNav was 100
 # fps) A standard non-gaming monitor is also 60Hz. (TV is 30 fps)
-@dataclass
-class PhilNav:
-    time_start = time()
-    time_debug = time()
-    debug_num = 0
-    msg_time_start = time()
-    msg_time_total = 0
-    msg_num = 0
-    x_q = deque([], args.smooth)
-    y_q = deque([], args.smooth)
-    x_q_long = deque([], args.smooth*3)
-    y_q_long = deque([], args.smooth*3)
-
-
+phil = {}
+phil.time_start = time()
+phil.time_debug = time()
+phil.debug_num = 0
+
+# mouse (x_diff, y_diff) smoothing running averages
+smooth_long = args.smooth*3+1
+phil.x_q = deque([], args.smooth)
+phil.x_q_smooth = 0
+phil.x_q_long = deque([], smooth_long)
+phil.x_q_long_smooth = 0
+phil.y_q = deque([], args.smooth)
+phil.y_q_smooth = 0
+phil.y_q_long = deque([], smooth_long)
+phil.y_q_long_smooth = 0
+
+
+# simple moving average to reduce mouse jitter
 def smooth(q):
-    sum = np.sum(q)
-    avg = sum / len(q)
+    avg = sum(q) / len(q)
     return avg
 
 
@@ -158,59 +132,62 @@ def smooth(q):
         # https://github.com/opentrack/opentrack/issues/747
         data, addr = sock.recvfrom(48)
     except TimeoutError:
-        if int(time() - PhilNav.time_start) % 5 == 0:
-            logging.info(f" {ctime()} - {text_listening}")
+        if int(time() - phil.time_start) % 5 == 0:
+            logging.info(f"{ctime()} - {text_listening}")
         continue
     else:
-        # measure time
-        PhilNav.msg_time_start = time()
-        PhilNav.msg_num += 1
+        if not enabled:
+            continue
 
         # Using OpenTrack protocol, but PhilNav uses:
-        #  x_diff, y_diff, n/a, n/a, n/a, camera capture time
-        x, y, z, pitch, yaw, roll = struct.unpack("dddddd", data)
-
-        # Simple moving average to smooth out jitters
-        PhilNav.x_q.append(x)
-        PhilNav.y_q.append(y)
-        PhilNav.x_q_long.append(x)
-        PhilNav.y_q_long.append(y)
-        if x**2 + y**2 < 0.2:
-            x_smooth = smooth(PhilNav.x_q_long)
-            y_smooth = smooth(PhilNav.y_q_long)
-        elif x**2 + y**2 < 0.5:
-            x_smooth = smooth(PhilNav.x_q)
-            y_smooth = smooth(PhilNav.y_q)
-        else:
-            x_smooth = x
-            y_smooth = y
-
-        # The Magic Happens Now! eg. move mouse cursor =P
+        #  x_diff, y_diff, n/a, n/a, camera capture time, OpenCV processing time
+        x_diff, y_diff, a, b, time_cam, ms_opencv = struct.unpack("dddddd", data)
+
+        # store recent mouse movements
+        phil.x_q.append(x_diff)
+        phil.y_q.append(y_diff)
+        phil.x_q_long.append(x_diff)
+        phil.y_q_long.append(y_diff)
+
+        # Perform more smoothing the *slower* the mouse is moving.
+        # A slow-moving cursor means the user is trying to precisely
+        # point at something.
+        if x_diff**2 + y_diff**2 < 0.2: # more smoothing
+            x_smooth = smooth(phil.x_q_long)
+            y_smooth = smooth(phil.y_q_long)
+        elif x_diff**2 + y_diff**2 < 0.5: # less smoothing
+            x_smooth = smooth(phil.x_q)
+            y_smooth = smooth(phil.y_q)
+        else: # moving fast, no smoothing
+            x_smooth = x_diff
+            y_smooth = y_diff
+
+        # The Magic Happens Now!
         x_cur, y_cur = getCursorPos()
-        # get current mouse position by reference (C++ thing)
         # I'm moving the Y axis slightly faster because looking left and right
         # is easier than nodding up and down. Also, monitors are wider than they
         # are tall.
         x_new = round(x_cur + x_smooth * args.speed)
         y_new = round(y_cur + y_smooth * args.speed * 1.25)
-        if enabled:
-            setCursorPos(x_new, y_new)  # move mouse cursor
+        setCursorPos(x_new, y_new)  # move mouse cursor
 
         # I'm trying to measure the total time from capturing the frame on the
         # camera to moving the mouse cursor on my PC. This isn't super accurate.
         # It's sometimes negative (TIME TRAVEL!!!). The clock difference between
         # the Raspberry Pi and my PC seems to be around 10-20ms?
-        time_diff_ms = int((time() - roll) * 1000)
+        now = time()
+        now_str = ctime()
+        ms_time_diff = int((now - time_cam) * 1000)
 
         # it's 60 FPS, so only debug once per second
-        if time() - PhilNav.time_debug > 1:
-            PhilNav.time_debug = time()
-            PhilNav.debug_num += 1
+        if now - phil.time_debug > 1:
+            phil.time_debug = now
+            phil.debug_num += 1
             # display legend every 5 seconds
-            if PhilNav.debug_num % 5 == 1:
+            if phil.debug_num % 5 == 1:
                 logging.info(
-                    f" {ctime()} - Received: ({'x_diff':>8},{'y_diff':>8},{'n/a':>8},{'n/a':>8},{'loc ns':>8},{'net ms':>8}  )"
+                    f"{now_str} - Received: ({'x_diff':>8},{'y_diff':>8}),{'n/a':>8},{'n/a':>8},{'time ms':>8},{'time cv':>8}"
                 )
             logging.info(
-                f" {ctime()} - Received: ({x:> 8.2f},{y:> 8.2f},{z:> 8.2f},{pitch:> 8.2f},{(time() - PhilNav.msg_time_start)*1000:> 8.2f},{time_diff_ms:> 8}  )"
+                f"{now_str} - Received: ({x_diff:> 8.2f},{y_diff:> 8.2f}),{a:> 8.2f},{b:> 8.2f},{ms_time_diff:> 8},{ms_opencv:>8}"
             )

client_win-mac-nix/mouse_mac.py

@@ -1,36 +1,36 @@
 import Quartz
 
+
 # PhilNav will only move the mouse on the main display. This might not be what you want, but I like it.
 Qcgd_origin, Qcgd_size = Quartz.CGDisplayBounds(0)
 
+
 def getCursorPos():
     blankEvent = Quartz.CGEventCreate(None)
     x, y = Quartz.CGEventGetLocation(blankEvent)
     return x, y
 
 def setCursorPos(x, y):
-    global Qcgd_origin, Qcgd_size
-    # https://developer.apple.com/documentation/coregraphics/1454356-cgeventcreatemouseevent
-    # CGEventType: case mouseMoved = 5
-    # Ignored. CGMouseButton: case left = 0
-
     # On a Mac, the mouse is allowed to be "out of bounds". I'm keeping it in-bounds.
     # https://developer.apple.com/documentation/coregraphics/1456395-cgdisplaybounds
     if x < Qcgd_origin.x:
         x = Qcgd_origin.x
     if y < Qcgd_origin.y:
         y = Qcgd_origin.y
-    if x > Qcgd_size.width:
-        x = Qcgd_size.width
-    if y > Qcgd_size.height:
-        y = Qcgd_size.height
+    right = Qcgd_origin.x + Qcgd_size.width
+    bottom = Qcgd_origin.y + Qcgd_size.height
+    if x > right:
+        x = right
+    if y > bottom:
+        y = bottom
 
+    # https://developer.apple.com/documentation/coregraphics/1454356-cgeventcreatemouseevent
+    # CGEventType: case mouseMoved = 5
+    # Ignored. CGMouseButton: case left = 0
     mouseEvent = Quartz.CGEventCreateMouseEvent(None, 5, (x, y), 0)
     # CGEventTapLocation: case cghidEventTap = 0
     Quartz.CGEventPost(0, mouseEvent)
 
-    Quartz.CGAssociateMouseAndMouseCursorPosition(True)
-
     # Two other ways of doing the same thing.
     # But these "lock" the physical mouse and don't send events.
     # Quartz.CGDisplayMoveCursorToPoint(0, (x, y))

client_win-mac-nix/mouse_nix.py

@@ -0,0 +1 @@
+# TODO

client_win-mac-nix/mouse_win.py

@@ -0,0 +1,14 @@
+import ctypes
+
+
+# simple point.x, point.y
+class POINT(ctypes.Structure):
+    _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)]
+
+def getCursorPos():
+    pt = POINT()
+    ctypes.windll.user32.GetCursorPos(ctypes.byref(pt))
+    return pt.x, pt.y
+
+def setCursorPos(x, y):
+    ctypes.windll.user32.SetCursorPos(x, y)