wasapideviceprovider: Add support for dynamic device add/remove

Adding IMMDeviceEnumerator::RegisterEndpointNotificationCallback
in order to support device monitoring.
On OnDeviceAdded(), OnDeviceRemoved(), and OnDefaultDeviceChanged()
callback, wasapi device provider implementation will enumerate
devices again and will notify newly added and removed device
via GstDeviceProvider API.

As a bonus point, this IMMDeviceEnumerator abstraction object
will spawn a dedicated internal COM thread, so various COM thread
related issues of WASAPI plugin can be resolved by this commit.

Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1649
Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1110
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2484>

Author: seungha-yang

sys/wasapi/gstmmdeviceenumerator.cpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2021 Seungha Yang <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_MM_DEVICE_ENUMERATOR_H__
+#define __GST_MM_DEVICE_ENUMERATOR_H__
+
+#include <gst/gst.h>
+#include <mmdeviceapi.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_MM_DEVICE_ENUMERATOR (gst_mm_device_enumerator_get_type ())
+G_DECLARE_FINAL_TYPE (GstMMDeviceEnumerator, gst_mm_device_enumerator,
+    GST, MM_DEVICE_ENUMERATOR, GstObject);
+
+typedef struct
+{
+  HRESULT (*device_state_changed)    (GstMMDeviceEnumerator * enumerator,
+                                      LPCWSTR device_id,
+                                      DWORD new_state,
+                                      gpointer user_data);
+
+  HRESULT (*device_added)            (GstMMDeviceEnumerator * enumerator,
+                                      LPCWSTR device_id,
+                                      gpointer user_data);
+
+  HRESULT (*device_removed)          (GstMMDeviceEnumerator * provider,
+                                      LPCWSTR device_id,
+                                      gpointer user_data);
+
+  HRESULT (*default_device_changed)  (GstMMDeviceEnumerator * provider,
+                                      EDataFlow flow,
+                                      ERole role,
+                                      LPCWSTR default_device_id,
+                                      gpointer user_data);
+
+  HRESULT (*property_value_changed)  (GstMMDeviceEnumerator * provider,
+                                      LPCWSTR device_id,
+                                      const PROPERTYKEY key,
+                                      gpointer user_data);
+} GstMMNotificationClientCallbacks;
+
+GstMMDeviceEnumerator * gst_mm_device_enumerator_new (void);
+
+IMMDeviceEnumerator * gst_mm_device_enumerator_get_handle (GstMMDeviceEnumerator * enumerator);
+
+gboolean gst_mm_device_enumerator_set_notification_callback (GstMMDeviceEnumerator * enumerator,
+                                                             GstMMNotificationClientCallbacks * callbacks,
+                                                             gpointer user_data);
+
+G_END_DECLS
+
+#endif /* __GST_MM_DEVICE_ENUMERATOR_H__ */

sys/wasapi/gstmmdeviceenumerator.h

@@ -1,5 +1,6 @@
 /* GStreamer
  * Copyright (C) 2018 Nirbheek Chauhan <[email protected]>
+ * Copyright (C) 2021 Seungha Yang <[email protected]>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Library General Public
@@ -23,11 +24,27 @@
 
 #include "gstwasapidevice.h"
 
+GST_DEBUG_CATEGORY_EXTERN (gst_wasapi_debug);
+#define GST_CAT_DEFAULT gst_wasapi_debug
+
 G_DEFINE_TYPE (GstWasapiDeviceProvider, gst_wasapi_device_provider,
     GST_TYPE_DEVICE_PROVIDER);
 
 static void gst_wasapi_device_provider_finalize (GObject * object);
 static GList *gst_wasapi_device_provider_probe (GstDeviceProvider * provider);
+static gboolean gst_wasapi_device_provider_start (GstDeviceProvider * provider);
+static void gst_wasapi_device_provider_stop (GstDeviceProvider * provider);
+
+static HRESULT
+gst_wasapi_device_provider_device_added (GstMMDeviceEnumerator * enumerator,
+    LPCWSTR device_id, gpointer user_data);
+static HRESULT
+gst_wasapi_device_provider_device_removed (GstMMDeviceEnumerator * enumerator,
+    LPCWSTR device_id, gpointer user_data);
+static HRESULT
+gst_wasapi_device_provider_default_device_changed (GstMMDeviceEnumerator *
+    enumerator, EDataFlow flow, ERole role, LPCWSTR device_id,
+    gpointer user_data);
 
 static void
 gst_wasapi_device_provider_class_init (GstWasapiDeviceProviderClass * klass)
@@ -38,6 +55,8 @@ gst_wasapi_device_provider_class_init (GstWasapiDeviceProviderClass * klass)
   gobject_class->finalize = gst_wasapi_device_provider_finalize;
 
   dm_class->probe = gst_wasapi_device_provider_probe;
+  dm_class->start = gst_wasapi_device_provider_start;
+  dm_class->stop = gst_wasapi_device_provider_stop;
 
   gst_device_provider_class_set_static_metadata (dm_class,
       "WASAPI (Windows Audio Session API) Device Provider",
@@ -46,15 +65,68 @@ gst_wasapi_device_provider_class_init (GstWasapiDeviceProviderClass * klass)
 }
 
 static void
-gst_wasapi_device_provider_init (GstWasapiDeviceProvider * provider)
+gst_wasapi_device_provider_init (GstWasapiDeviceProvider * self)
 {
-  CoInitializeEx (NULL, COINIT_MULTITHREADED);
+  self->enumerator = gst_mm_device_enumerator_new ();
+}
+
+static gboolean
+gst_wasapi_device_provider_start (GstDeviceProvider * provider)
+{
+  GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (provider);
+  GstMMNotificationClientCallbacks callbacks = { NULL, };
+  GList *devices = NULL;
+  GList *iter;
+
+  if (!self->enumerator) {
+    GST_WARNING_OBJECT (self, "Enumerator wasn't configured");
+    return FALSE;
+  }
+
+  callbacks.device_added = gst_wasapi_device_provider_device_added;
+  callbacks.device_removed = gst_wasapi_device_provider_device_removed;
+  callbacks.default_device_changed =
+      gst_wasapi_device_provider_default_device_changed;
+
+  if (!gst_mm_device_enumerator_set_notification_callback (self->enumerator,
+          &callbacks, self)) {
+    GST_WARNING_OBJECT (self, "Failed to set callbacks");
+    return FALSE;
+  }
+
+  /* baseclass will not call probe() once it's started, but we can get
+   * notification only add/remove or change case. To this manually */
+  devices = gst_wasapi_device_provider_probe (provider);
+  if (devices) {
+    for (iter = devices; iter; iter = g_list_next (iter)) {
+      gst_device_provider_device_add (provider, GST_DEVICE (iter->data));
+    }
+
+    g_list_free (devices);
+  }
+
+  return TRUE;
+}
+
+static void
+gst_wasapi_device_provider_stop (GstDeviceProvider * provider)
+{
+  GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (provider);
+
+  if (self->enumerator) {
+    gst_mm_device_enumerator_set_notification_callback (self->enumerator,
+        NULL, NULL);
+  }
 }
 
 static void
 gst_wasapi_device_provider_finalize (GObject * object)
 {
-  CoUninitialize ();
+  GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (object);
+
+  gst_clear_object (&self->enumerator);
+
+  G_OBJECT_CLASS (gst_wasapi_device_provider_parent_class)->finalize (object);
 }
 
 static GList *
@@ -63,12 +135,137 @@ gst_wasapi_device_provider_probe (GstDeviceProvider * provider)
   GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (provider);
   GList *devices = NULL;
 
-  if (!gst_wasapi_util_get_devices (GST_OBJECT (self), TRUE, &devices))
+  if (!gst_wasapi_util_get_devices (self->enumerator, TRUE, &devices))
     GST_ERROR_OBJECT (self, "Failed to enumerate devices");
 
   return devices;
 }
 
+static gboolean
+gst_wasapi_device_is_in_list (GList * list, GstDevice * device)
+{
+  GList *iter;
+  GstStructure *s;
+  const gchar *device_id;
+  gboolean found = FALSE;
+
+  s = gst_device_get_properties (device);
+  g_assert (s);
+
+  device_id = gst_structure_get_string (s, "device.strid");
+  g_assert (device_id);
+
+  for (iter = list; iter; iter = g_list_next (iter)) {
+    GstStructure *other_s;
+    const gchar *other_id;
+
+    other_s = gst_device_get_properties (GST_DEVICE (iter->data));
+    g_assert (other_s);
+
+    other_id = gst_structure_get_string (other_s, "device.strid");
+    g_assert (other_id);
+
+    if (g_ascii_strcasecmp (device_id, other_id) == 0) {
+      found = TRUE;
+    }
+
+    gst_structure_free (other_s);
+    if (found)
+      break;
+  }
+
+  gst_structure_free (s);
+
+  return found;
+}
+
+static void
+gst_wasapi_device_provider_update_devices (GstWasapiDeviceProvider * self)
+{
+  GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (self);
+  GList *prev_devices = NULL;
+  GList *new_devices = NULL;
+  GList *to_add = NULL;
+  GList *to_remove = NULL;
+  GList *iter;
+
+  GST_OBJECT_LOCK (self);
+  prev_devices = g_list_copy_deep (provider->devices,
+      (GCopyFunc) gst_object_ref, NULL);
+  GST_OBJECT_UNLOCK (self);
+
+  new_devices = gst_wasapi_device_provider_probe (provider);
+
+  /* Ownership of GstDevice for gst_device_provider_device_add()
+   * and gst_device_provider_device_remove() is a bit complicated.
+   * Remove floating reference here for things to be clear */
+  for (iter = new_devices; iter; iter = g_list_next (iter))
+    gst_object_ref_sink (iter->data);
+
+  /* Check newly added devices */
+  for (iter = new_devices; iter; iter = g_list_next (iter)) {
+    if (!gst_wasapi_device_is_in_list (prev_devices, GST_DEVICE (iter->data))) {
+      to_add = g_list_prepend (to_add, gst_object_ref (iter->data));
+    }
+  }
+
+  /* Check removed device */
+  for (iter = prev_devices; iter; iter = g_list_next (iter)) {
+    if (!gst_wasapi_device_is_in_list (new_devices, GST_DEVICE (iter->data))) {
+      to_remove = g_list_prepend (to_remove, gst_object_ref (iter->data));
+    }
+  }
+
+  for (iter = to_remove; iter; iter = g_list_next (iter))
+    gst_device_provider_device_remove (provider, GST_DEVICE (iter->data));
+
+  for (iter = to_add; iter; iter = g_list_next (iter))
+    gst_device_provider_device_add (provider, GST_DEVICE (iter->data));
+
+  if (prev_devices)
+    g_list_free_full (prev_devices, (GDestroyNotify) gst_object_unref);
+
+  if (to_add)
+    g_list_free_full (to_add, (GDestroyNotify) gst_object_unref);
+
+  if (to_remove)
+    g_list_free_full (to_remove, (GDestroyNotify) gst_object_unref);
+}
+
+static HRESULT
+gst_wasapi_device_provider_device_added (GstMMDeviceEnumerator * enumerator,
+    LPCWSTR device_id, gpointer user_data)
+{
+  GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (user_data);
+
+  gst_wasapi_device_provider_update_devices (self);
+
+  return S_OK;
+}
+
+static HRESULT
+gst_wasapi_device_provider_device_removed (GstMMDeviceEnumerator * enumerator,
+    LPCWSTR device_id, gpointer user_data)
+{
+  GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (user_data);
+
+  gst_wasapi_device_provider_update_devices (self);
+
+  return S_OK;
+}
+
+static HRESULT
+gst_wasapi_device_provider_default_device_changed (GstMMDeviceEnumerator *
+    enumerator, EDataFlow flow, ERole role, LPCWSTR device_id,
+    gpointer user_data)
+{
+  GstWasapiDeviceProvider *self = GST_WASAPI_DEVICE_PROVIDER (user_data);
+
+  gst_wasapi_device_provider_update_devices (self);
+
+  return S_OK;
+}
+
 /* GstWasapiDevice begins */
 
 enum

sys/wasapi/gstwasapidevice.c

@@ -38,6 +38,8 @@ typedef struct _GstWasapiDeviceProviderClass GstWasapiDeviceProviderClass;
 struct _GstWasapiDeviceProvider
 {
   GstDeviceProvider parent;
+
+  GstMMDeviceEnumerator *enumerator;
 };
 
 struct _GstWasapiDeviceProviderClass

sys/wasapi/gstwasapidevice.h

@@ -180,7 +180,7 @@ gst_wasapi_sink_init (GstWasapiSink * self)
   self->cancellable = CreateEvent (NULL, TRUE, FALSE, NULL);
   self->client_needs_restart = FALSE;
 
-  CoInitializeEx (NULL, COINIT_MULTITHREADED);
+  self->enumerator = gst_mm_device_enumerator_new ();
 }
 
 static void
@@ -208,6 +208,8 @@ gst_wasapi_sink_dispose (GObject * object)
     self->render_client = NULL;
   }
 
+  gst_clear_object (&self->enumerator);
+
   G_OBJECT_CLASS (gst_wasapi_sink_parent_class)->dispose (object);
 }
 
@@ -219,8 +221,6 @@ gst_wasapi_sink_finalize (GObject * object)
   CoTaskMemFree (self->mix_format);
   self->mix_format = NULL;
 
-  CoUninitialize ();
-
   if (self->cached_caps != NULL) {
     gst_caps_unref (self->cached_caps);
     self->cached_caps = NULL;
@@ -412,7 +412,7 @@ gst_wasapi_sink_open (GstAudioSink * asink)
    * even if the old device was unplugged. We need to handle this somehow.
    * For example, perhaps we should automatically switch to the new device if
    * the default device is changed and a device isn't explicitly selected. */
-  if (!gst_wasapi_util_get_device (GST_ELEMENT (self), eRender,
+  if (!gst_wasapi_util_get_device (self->enumerator, eRender,
           self->role, self->device_strid, &device)
       || !gst_wasapi_util_get_audio_client (GST_ELEMENT (self),
           device, &client)) {
@@ -485,8 +485,6 @@ gst_wasapi_sink_prepare (GstAudioSink * asink, GstAudioRingBufferSpec * spec)
   guint bpf, rate, devicep_frames;
   HRESULT hr;
 
-  CoInitializeEx (NULL, COINIT_MULTITHREADED);
-
   if (!self->client) {
     GST_DEBUG_OBJECT (self, "no IAudioClient, creating a new one");
     if (!gst_wasapi_util_get_audio_client (GST_ELEMENT (self),
@@ -608,8 +606,6 @@ gst_wasapi_sink_unprepare (GstAudioSink * asink)
     self->render_client = NULL;
   }
 
-  CoUninitialize ();
-
   return TRUE;
 }
 

sys/wasapi/gstwasapisink.c

@@ -40,6 +40,8 @@ struct _GstWasapiSink
 {
   GstAudioSink parent;
 
+  GstMMDeviceEnumerator *enumerator;
+
   IMMDevice *device;
   IAudioClient *client;
   IAudioRenderClient *render_client;