Music Assistant coverart usermod

platformio.ini

@@ -39,7 +39,7 @@ default_envs = nodemcuv2
                esp32s3dev_8MB_qspi ;; TODO: disabled NeoEsp32RmtMethodIsr
                esp32s3dev_8MB_none ;; TODO: disabled NeoEsp32RmtMethodIsr
                esp32s3_4M_qspi ;; TODO: disabled NeoEsp32RmtMethodIsr
-               usermods ;; TODO: make usermods compile in V5
+               usermods
 
 src_dir  = ./wled00
 data_dir = ./wled00/data
@@ -216,7 +216,7 @@ build_flags =
 lib_deps =
   #https://github.com/lorol/LITTLEFS.git
   ESPAsyncTCP @ 1.2.2
-  ESPAsyncUDP
+  https://github.com/me-no-dev/ESPAsyncUDP.git#697c75a0255e5dccf91b4ef8b5cc28c569b3bac9
   ESP8266PWM
   https://github.com/tignioj/ArduinoUZlib.git#20aff95cd80c141f80bdbf66895409a0046d2c2f
   ${env.lib_deps}
@@ -247,23 +247,25 @@ platform_packages_compat =
 ;; experimental - for using older NeoPixelBus 2.7.9
 lib_deps_compat =
   ESPAsyncTCP @ 1.2.2
-  ESPAsyncUDP
+  https://github.com/me-no-dev/ESPAsyncUDP.git#697c75a0255e5dccf91b4ef8b5cc28c569b3bac9
   ESP8266PWM
   IRremoteESP8266 @ 2.8.2
   makuna/NeoPixelBus @ 2.7.9
   https://github.com/tignioj/ArduinoUZlib.git#20aff95cd80c141f80bdbf66895409a0046d2c2f
-  https://github.com/Aircoookie/ESPAsyncWebServer.git#v2.4.0
+  https://github.com/Aircoookie/ESPAsyncWebServer.git#2d3af2c32b1dd1a8fd9fb01d7b51fa3653221a89 ;; v2.4.0 (pinned by commit to avoid PIO annotated-tag checkout warning)
 
 [esp32_all_variants]
 lib_deps =
   esp32async/AsyncTCP @ 3.4.10
   bitbank2/AnimatedGIF@^1.4.7
   https://github.com/Aircoookie/GifDecoder.git#bc3af189b6b1e06946569f6b4287f0b79a860f8e
+  bitbank2/JPEGDEC@^1.8.4
   ;; neopixelbus dependancy temporarily moved into [esp32_idf_V4] and [esp32_idf_V5] sections
 build_flags =
   -D CONFIG_ASYNC_TCP_USE_WDT=0
   -D CONFIG_ASYNC_TCP_STACK_SIZE=8192
   -D WLED_ENABLE_GIF
+  -D WLED_ENABLE_JPEG
 
 [esp32]
 platform = ${esp32_idf_V5.platform}
@@ -312,7 +314,7 @@ lib_deps =
   ${esp32_all_variants.lib_deps}
   https://github.com/someweisguy/esp_dmx.git#47db25d8c515e76fabcf5fc5ab0b786f98eeade0
   ${env.lib_deps}
-  NeoPixelBus = git+https://github.com/Makuna/NeoPixelBus#1d7ff38f14d04a976f9c6e365c83a232bcba04f ;; latest release + properly inialized buffers
+  NeoPixelBus = git+https://github.com/Makuna/NeoPixelBus#1d7ff38f14d04a976f9c6e365c83a232bcba04fc ;; latest release + properly inialized buffers
 lib_ignore = 
 
 [esp32_idf_V5]
@@ -334,7 +336,7 @@ build_flags = -g
   -D ESP32_ARDUINO_NO_RGB_BUILTIN ;; avoids RMT driver abort on startup "E (98) rmt(legacy): CONFLICT! driver_ng is not allowed to be used with the legacy driver"
 lib_deps =
   ${esp32_all_variants.lib_deps}
-  https://github.com/netmindz/esp_dmx/#esp-idf-v5-fixes  ;; TODO: need an upstream update for V5
+  https://github.com/netmindz/esp_dmx/#ed12a2909960c92d7f144d368564e2473bd38c63  ;; esp-idf-v5-fixes branch, pinned; TODO: need an upstream update for V5
   ${env.lib_deps}
   NeoPixelBus = git+https://github.com/Makuna/NeoPixelBus#76afe832f74b0738a3fa1bba0caf389ade9e7693 ;; CORE3 branch
   ;; AsyncMqttClient = git+https://github.com/marvinroger/async-mqtt-client#3d93fc7f662e65366f8e2b0d88b108f874f035b9 ;; override to use latest commmit from Sep 25, 2024

usermods/usermod_v2_musicassistant/library.json

@@ -0,0 +1,4 @@
+{
+  "name": "usermod_v2_musicassistant",
+  "build": { "libArchive": false }
+}

usermods/usermod_v2_musicassistant/readme.md

@@ -0,0 +1,66 @@
+# Music Assistant usermod
+
+Ties WLED's built-in "Image" effect to the cover art of whatever is currently
+playing on a [Music Assistant](https://music-assistant.io) server. Poll a
+configured player queue on a timer; when the track's cover art changes, fetch
+it from Music Assistant's `/imageproxy` endpoint and feed it to the core
+"Image" effect's live-render path.
+
+## Requirements
+
+- ESP32 (JPEG decoding is not available on ESP8266).
+- Core must be built with both `-D WLED_ENABLE_GIF` and `-D WLED_ENABLE_JPEG`
+  (the `esp32_all_variants` PlatformIO environment already sets both).
+- A **2D** segment running the **Image** effect with its **Live** checkbox
+  ticked (the live-render path requires a matrix; unlike the .gif path it
+  does not support 1D strips). This usermod never changes effects or
+  segments itself unless "Auto switch effect" is enabled - it only keeps the
+  live cover-art buffer updated.
+
+## Usage
+
+```ini
+[env:esp32_musicassistant]
+extends = env:esp32dev
+custom_usermods = ${env:esp32dev.custom_usermods} musicassistant
+```
+
+## Settings
+
+### Enabled
+Turns polling on/off.
+
+### Host / Port
+Address of your Music Assistant server, e.g. `mass` / `8095`.
+
+### Token
+A Bearer token for the Music Assistant HTTP API. Music Assistant requires
+authentication for all API calls since API schema v28. Create a long-lived
+token with the `auth/token/create` command (WebSocket or `POST /api`), or log
+in via `POST /auth/login` and use the returned short-lived token. This token
+is stored in `cfg.json`, same as any other usermod credential.
+
+### Queue ID
+The `queue_id` of the player/zone to track (a server can have many). Find it
+via the `player_queues/all` command - it's the currently-active queue you
+want reflected on your LEDs.
+
+### Poll interval (ms)
+How often to check for a track change. The queue-state poll is small and
+cheap; the (larger) cover art JPEG is only fetched when the track actually
+changes.
+
+### Image size
+Must be one of Music Assistant's supported `/imageproxy` sizes: `80`, `160`,
+`256`, `512`, `1024` (or `0` for original size - not recommended, can be
+large). `80` is plenty for most LED matrices.
+
+### Auto switch effect
+When enabled, the usermod forces the configured segment into the "Image"
+effect with "Live" ticked whenever playback starts and a cover art image is
+available. It only acts on that start-of-playback transition - it won't
+fight you if you manually switch away to another effect while still playing.
+Off by default: the usermod otherwise never touches segment/effect selection.
+
+### Segment ID
+Which segment "Auto switch effect" controls. Ignored if that option is off.

usermods/usermod_v2_musicassistant/usermod_v2_musicassistant.cpp

@@ -0,0 +1,211 @@
+#include "wled.h"
+
+/*
+ * Polls a Music Assistant server for the currently-playing track on a configured
+ * player queue. When the track's cover art changes, fetches the JPEG from MA's
+ * /imageproxy endpoint (unauthenticated) and hands it to the core "Image" effect's
+ * live-render path (setLiveCoverArtFromStream(), wled00/image_loader.cpp), which is
+ * only compiled in when WLED_ENABLE_JPEG is defined (see platformio.ini).
+ *
+ * By default this usermod only keeps that buffer fed - the user selects the "Image"
+ * effect and ticks its "Live" checkbox on whichever segment/matrix should show cover
+ * art. If "Auto switch effect" is enabled, it will also force the configured segment
+ * into Image+Live whenever playback starts and a cover art image is available.
+ */
+class MusicAssistantUsermod : public Usermod {
+  private:
+    bool     enabled        = false;
+    String   host           = "mass";
+    uint16_t port           = 8095;
+    String   token          = "";
+    String   queueId        = "";
+    uint32_t pollIntervalMs = 4000;
+    uint16_t imageSize      = 80; // must be one of MA's supported sizes: 0,80,160,256,512,1024
+    bool     autoSwitchEffect = false;
+    uint8_t  segmentId        = 0; // which segment to control when autoSwitchEffect is on
+
+    WiFiClient client;
+    unsigned long lastPoll = 0;
+    String lastProxyId = "";
+    bool wasPlaying = false;
+    char errorMessage[96] = "";
+
+    static const char _name[];
+
+    bool beginRequest(const __FlashStringHelper *method, const String &path, bool withAuth) {
+      if (!client.connect(host.c_str(), port)) {
+        strlcpy(errorMessage, "connection failed", sizeof(errorMessage));
+        return false;
+      }
+      client.print(method); client.print(' '); client.print(path); client.println(F(" HTTP/1.1"));
+      client.print(F("Host: ")); client.println(host);
+      if (withAuth && token.length()) {
+        client.print(F("Authorization: Bearer ")); client.println(token);
+      }
+      client.println(F("Connection: close"));
+      return true;
+    }
+
+    // reads the status line + headers of a response already sent by the caller; leaves 'client' positioned at the body
+    bool readStatusAndHeaders(int &statusCode, long &contentLength) {
+      statusCode = 0;
+      contentLength = -1;
+      client.setTimeout(2000); // this blocks loop()/rendering while waiting - keep it short
+      String statusLine = client.readStringUntil('\n');
+      statusLine.trim();
+      if (statusLine.length() == 0) {
+        strlcpy(errorMessage, "no response", sizeof(errorMessage));
+        return false;
+      }
+      int sp = statusLine.indexOf(' ');
+      if (sp > 0) statusCode = statusLine.substring(sp + 1).toInt();
+      while (client.connected() || client.available()) {
+        String line = client.readStringUntil('\n');
+        line.trim();
+        if (line.length() == 0) break; // blank line: end of headers
+        if (line.startsWith(F("Content-Length:"))) contentLength = line.substring(line.indexOf(':') + 1).toInt();
+      }
+      return true;
+    }
+
+    void pollQueue() {
+      if (queueId.isEmpty()) return;
+      if (!beginRequest(F("POST"), F("/api"), true)) return;
+
+      StaticJsonDocument<160> body;
+      body["command"] = "player_queues/get";
+      body["args"]["queue_id"] = queueId;
+      String bodyStr;
+      serializeJson(body, bodyStr);
+
+      client.println(F("Content-Type: application/json"));
+      client.print(F("Content-Length: ")); client.println(bodyStr.length());
+      client.println();
+      client.print(bodyStr);
+
+      int statusCode; long contentLength;
+      if (!readStatusAndHeaders(statusCode, contentLength) || statusCode != 200) {
+        if (statusCode) { snprintf(errorMessage, sizeof(errorMessage), "HTTP %d from /api", statusCode); }
+        client.stop();
+        return;
+      }
+
+      // Filtered parse: PlayerQueue responses also include the full media_item/streamdetails
+      // tree (many KB) which we don't need and can't afford to buffer on an ESP32.
+      StaticJsonDocument<192> filter;
+      filter["state"] = true;
+      filter["current_item"]["image"]["proxy_id"] = true;
+      DynamicJsonDocument doc(768);
+      DeserializationError err = deserializeJson(doc, client, DeserializationOption::Filter(filter));
+      client.stop();
+      if (err) {
+        strlcpy(errorMessage, err.c_str(), sizeof(errorMessage));
+        return;
+      }
+
+      const char* state   = doc["state"] | "";
+      const char* proxyId = doc["current_item"]["image"]["proxy_id"] | "";
+      bool nowPlaying = strcmp(state, "playing") == 0 && proxyId[0];
+
+      if (autoSwitchEffect && nowPlaying && !wasPlaying) switchToImageEffect();
+      wasPlaying = nowPlaying;
+
+      if (!nowPlaying) return; // nothing playing: leave last art showing
+
+      if (lastProxyId != proxyId) {
+        lastProxyId = proxyId;
+        fetchCoverArt(proxyId);
+      } else {
+        errorMessage[0] = '\0';
+      }
+    }
+
+    void switchToImageEffect() {
+      // getSegment() silently falls back to the main segment for an out-of-range id,
+      // rather than crashing - so a stale/typo'd segmentId degrades gracefully.
+      Segment &seg = strip.getSegment(segmentId);
+      if (!seg.isActive()) return;
+      if (seg.mode != FX_MODE_IMAGE) seg.setMode(FX_MODE_IMAGE);
+      seg.check1 = true;
+    }
+
+    void fetchCoverArt(const String &proxyId) {
+      #if defined(WLED_ENABLE_GIF) && defined(WLED_ENABLE_JPEG)
+      String path = F("/imageproxy/");
+      path += proxyId;
+      path += F("?size=");
+      path += imageSize;
+      if (!beginRequest(F("GET"), path, false)) return;
+      client.println();
+
+      int statusCode; long contentLength;
+      if (!readStatusAndHeaders(statusCode, contentLength)
+          || statusCode != 200 || contentLength <= 0 || contentLength > LIVE_COVERART_MAX_BYTES) {
+        strlcpy(errorMessage, "imageproxy fetch failed", sizeof(errorMessage));
+        client.stop();
+        return;
+      }
+
+      bool ok = setLiveCoverArtFromStream(client, (size_t)contentLength);
+      client.stop();
+      strlcpy(errorMessage, ok ? "" : "short read from imageproxy", sizeof(errorMessage));
+      #else
+      strlcpy(errorMessage, "core built without WLED_ENABLE_JPEG", sizeof(errorMessage));
+      #endif
+    }
+
+  public:
+    void setup() {}
+    void connected() {}
+
+    void loop() {
+      if (!enabled || !WLED_CONNECTED) return;
+      if (millis() - lastPoll < pollIntervalMs) return;
+      lastPoll = millis();
+      pollQueue();
+    }
+
+    void addToConfig(JsonObject &root) {
+      JsonObject top = root.createNestedObject(FPSTR(_name));
+      top[F("enabled")]        = enabled;
+      top[F("host")]           = host;
+      top[F("port")]           = port;
+      top[F("token")]          = token;
+      top[F("queueId")]        = queueId;
+      top[F("pollIntervalMs")]   = pollIntervalMs;
+      top[F("imageSize")]        = imageSize;
+      top[F("autoSwitchEffect")] = autoSwitchEffect;
+      top[F("segmentId")]        = segmentId;
+    }
+
+    bool readFromConfig(JsonObject &root) {
+      JsonObject top = root[FPSTR(_name)];
+      bool configComplete = !top.isNull();
+      configComplete &= getJsonValue(top[F("enabled")],        enabled);
+      configComplete &= getJsonValue(top[F("host")],           host);
+      configComplete &= getJsonValue(top[F("port")],           port);
+      configComplete &= getJsonValue(top[F("token")],          token);
+      configComplete &= getJsonValue(top[F("queueId")],        queueId);
+      configComplete &= getJsonValue(top[F("pollIntervalMs")],   pollIntervalMs);
+      configComplete &= getJsonValue(top[F("imageSize")],        imageSize);
+      configComplete &= getJsonValue(top[F("autoSwitchEffect")], autoSwitchEffect);
+      configComplete &= getJsonValue(top[F("segmentId")],        segmentId);
+      return configComplete;
+    }
+
+    void addToJsonInfo(JsonObject &root) {
+      JsonObject user = root["u"];
+      if (user.isNull()) user = root.createNestedObject("u");
+      JsonArray infoArr = user.createNestedArray(FPSTR(_name));
+      if (!enabled) { infoArr.add(F("disabled")); return; }
+      if (errorMessage[0]) infoArr.add(errorMessage);
+      else infoArr.add(lastProxyId.length() ? F("ok") : F("waiting for playback"));
+    }
+
+    uint16_t getId() { return USERMOD_ID_MUSICASSISTANT; }
+};
+
+const char MusicAssistantUsermod::_name[] PROGMEM = "MusicAssistant";
+
+static MusicAssistantUsermod musicassistant_usermod;
+REGISTER_USERMOD(musicassistant_usermod);

wled00/FX.cpp

@@ -4643,14 +4643,17 @@ void mode_image(void) {
   #ifndef WLED_ENABLE_GIF
   FX_FALLBACK_STATIC;
   #else
+  #ifdef WLED_ENABLE_JPEG
+  if (SEGMENT.check1) { renderLiveCoverArtToSegment(SEGMENT); return; }
+  #endif
   renderImageToSegment(SEGMENT);
   #endif
   // if (status != 0 && status != 254 && status != 255) {
   //   Serial.print("GIF renderer return: ");
   //   Serial.println(status);
   // }
 }
-static const char _data_FX_MODE_IMAGE[] PROGMEM = "Image@!,Blur,;;;12;sx=128,ix=0";
+static const char _data_FX_MODE_IMAGE[] PROGMEM = "Image@!,Blur,,,,Live;;;12;sx=128,ix=0,o1=0";
 
 /*
   Blends random colors across palette

wled00/const.h

@@ -252,6 +252,7 @@ static_assert(WLED_MAX_BUSSES <= 32, "WLED_MAX_BUSSES exceeds hard limit");
 #define USERMOD_ID_RF433                 56     //Usermod "usermod_v2_RF433.h"
 #define USERMOD_ID_BRIGHTNESS_FOLLOW_SUN 57     //Usermod "usermod_v2_brightness_follow_sun.h"
 #define USERMOD_ID_USER_FX               58     //Usermod "user_fx"
+#define USERMOD_ID_MUSICASSISTANT        59     //Usermod "usermod_musicassistant.h"
 
 //Wifi encryption type
 #ifdef WLED_ENABLE_WPA_ENTERPRISE

wled00/fcn_declare.h

@@ -152,6 +152,12 @@ int fileReadBlockCallback(void * buffer, int numberOfBytes);
 int fileSizeCallback(void);
 byte renderImageToSegment(Segment &seg);
 void endImagePlayback(Segment* seg);
+#ifdef WLED_ENABLE_JPEG
+#define LIVE_COVERART_MAX_BYTES 8192 // cap on a single decoded cover-art JPEG
+class Stream;
+bool setLiveCoverArtFromStream(Stream &stream, size_t len);
+byte renderLiveCoverArtToSegment(Segment &seg);
+#endif
 #endif
 
 //improv.cpp