custom_usermod improvements

pio-scripts/dynarray.py

@@ -0,0 +1,19 @@
+# Add a section to the linker script to store our dynamic arrays
+# This is implemented as a pio post-script to ensure that we can
+# place our linker script at the correct point in the command arguments.
+Import("env")
+from pathlib import Path
+
+platform = env.get("PIOPLATFORM")
+script_file = Path(f"tools/dynarray_{platform}.ld")
+if script_file.is_file():
+    linker_script = f"-T{script_file}"
+    if platform == "espressif32":
+        # For ESP32, the script must be added at the right point in the list
+        linkflags = env.get("LINKFLAGS", [])
+        idx = linkflags.index("memory.ld")    
+        linkflags.insert(idx+1, linker_script)    
+        env.Replace(LINKFLAGS=linkflags)
+    else:
+        # For other platforms, put it in last
+        env.Append(LINKFLAGS=[linker_script])

pio-scripts/load_usermods.py

@@ -1,6 +1,8 @@
 Import('env')
 from collections import deque
 from pathlib import Path   # For OS-agnostic path manipulation
+import re
+from urllib.parse import urlparse
 from click import secho
 from SCons.Script import Exit
 from platformio.builder.tools.piolib import LibBuilderBase
@@ -25,25 +27,115 @@ def find_usermod(mod: str) -> Path:
     return mp
   raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!")
 
-def is_wled_module(dep: LibBuilderBase) -> bool:
-  """Returns true if the specified library is a wled module
+# Names of external/registry deps listed in custom_usermods.
+# Populated during parsing below; read by is_wled_module() at configure time.
+_custom_usermod_names: set[str] = set()
+
+# Matches any RFC-valid URL scheme (http, https, git, git+https, symlink, file, hg+ssh, etc.)
+_URL_SCHEME_RE = re.compile(r'^[a-zA-Z][a-zA-Z0-9+.-]*://')
+# SSH git URL: user@host:path  (e.g. git@github.com:user/repo.git#tag)
+_SSH_URL_RE = re.compile(r'^[^@\s]+@[^@:\s]+:[^:\s]')
+# Explicit custom name: "LibName = <spec>"  (PlatformIO [<name>=]<spec> form)
+_NAME_EQ_RE = re.compile(r'^([A-Za-z0-9_.-]+)\s*=\s*(\S.*)')
+
+
+def _is_external_entry(line: str) -> bool:
+  """Return True if line is a lib_deps-style external/registry entry."""
+  if _NAME_EQ_RE.match(line):              # "LibName = <spec>"
+    return True
+  if _URL_SCHEME_RE.match(line):           # https://, git://, symlink://, etc.
+    return True
+  if _SSH_URL_RE.match(line):              # git@github.com:user/repo.git
+    return True
+  if '@' in line:                          # "owner/Name @ ^1.0.0"
+    return True
+  if re.match(r'^[^/\s]+/[^/\s]+$', line):  # "owner/Name"
+    return True
+  return False
+
+
+def _predict_dep_name(entry: str) -> str | None:
+  """Predict the library name PlatformIO will assign to this dep (best-effort).
+
+  Accuracy relies on the library's manifest "name" matching the repo/package
+  name in the spec. This holds for well-authored libraries; the libArchive
+  check (which requires library.json) provides an early-failure safety net.
   """
-  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
-
-## Script starts here
-# Process usermod option
-usermods = env.GetProjectOption("custom_usermods","")
+  entry = entry.strip()
+  # "LibName = <spec>" — name is given explicitly; always use it
+  m = _NAME_EQ_RE.match(entry)
+  if m:
+    return m.group(1).strip()
+  # URL scheme: extract name from path
+  if _URL_SCHEME_RE.match(entry):
+    parsed = urlparse(entry)
+    if parsed.netloc in ('github.com', 'gitlab.com', 'bitbucket.com'):
+      parts = [p for p in parsed.path.split('/') if p]
+      if len(parts) >= 2:
+        name = parts[1]
+        return name[:-4] if name.endswith('.git') else name
+    name = Path(parsed.path.rstrip('/')).name
+    return name.split('.')[0].strip() or None
+  # SSH git URL: git@github.com:user/repo.git#tag → repo
+  if _SSH_URL_RE.match(entry):
+    path_part = entry.split(':', 1)[1].split('#')[0].rstrip('/')
+    name = Path(path_part).name
+    return (name[:-4] if name.endswith('.git') else name) or None
+  # Versioned registry: "owner/Name @ version" → Name
+  if '@' in entry:
+    name_part = entry.split('@')[0].strip()
+    return name_part.split('/')[-1].strip() if '/' in name_part else name_part
+  # Plain registry: "owner/Name" → Name
+  if re.match(r'^[^/\s]+/[^/\s]+$', entry):
+    return entry.split('/')[-1].strip()
+  return None
 
-# Handle "all usermods" case
-if usermods == '*':
-  usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
-else:
-  usermods = usermods.split()
 
-if usermods:
-  # Inject usermods in to project lib_deps
-  symlinks = [f"symlink://{find_usermod(mod).resolve()}" for mod in usermods]
-  env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + symlinks)
+def is_wled_module(dep: LibBuilderBase) -> bool:
+  """Returns true if the specified library is a wled module."""
+  return (
+    usermod_dir in Path(dep.src_dir).parents
+    or str(dep.name).startswith("wled-")
+    or dep.name in _custom_usermod_names
+  )
+
+
+## Script starts here — parse custom_usermods
+raw_usermods = env.GetProjectOption("custom_usermods", "")
+usermods_libdeps: list[str] = []
+
+for line in raw_usermods.splitlines():
+  line = line.strip()
+  if not line or line.startswith('#') or line.startswith(';'):
+    continue
+
+  if _is_external_entry(line):
+    # External URL or registry entry: pass through to lib_deps unchanged.
+    predicted = _predict_dep_name(line)
+    if predicted:
+      _custom_usermod_names.add(predicted)
+    else:
+      secho(
+        f"WARNING: Cannot determine library name for custom_usermods entry "
+        f"{line!r}. If it is not recognised as a WLED module at build time, "
+        f"ensure its library.json 'name' matches the repo name.",
+        fg="yellow", err=True)
+    usermods_libdeps.append(line)
+  else:
+    # Bare name(s): split on whitespace for backwards compatibility.
+    for token in line.split():
+      if token == '*':
+        for mod_path in sorted(usermod_dir.iterdir()):
+          if mod_path.is_dir() and (mod_path / 'library.json').exists():
+            _custom_usermod_names.add(mod_path.name)
+            usermods_libdeps.append(f"symlink://{mod_path.resolve()}")
+      else:
+        resolved = find_usermod(token)
+        _custom_usermod_names.add(resolved.name)
+        usermods_libdeps.append(f"symlink://{resolved.resolve()}")
+
+if usermods_libdeps:
+  env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + usermods_libdeps)
 
 # Utility function for assembling usermod include paths
 def cached_add_includes(dep, dep_cache: set, includes: deque):
@@ -93,9 +185,10 @@ def wrapped_ConfigureProjectLibBuilder(xenv):
   if broken_usermods:
     broken_usermods = [usermod.name for usermod in broken_usermods]
     secho(
-      f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly",
-      fg="red",
-      err=True)
+      f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- "
+      f"modules will not compile in correctly. Add '\"build\": {{\"libArchive\": false}}' "
+      f"to their library.json.",
+      fg="red", err=True)
     Exit(1)
 
   # Save the depbuilders list for later validation

pio-scripts/validate_modules.py

@@ -3,14 +3,7 @@
 from typing import Iterable
 from click import secho
 from SCons.Script import Action, Exit
-from platformio.builder.tools.piolib import LibBuilderBase
-
-
-def is_wled_module(env, dep: LibBuilderBase) -> bool:
-  """Returns true if the specified library is a wled module
-  """
-  usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
-  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
+Import("env")
 
 
 def read_lines(p: Path):
@@ -37,11 +30,13 @@ def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]
             found.add(m)
     return found
 
+DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray"
+USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1"
 
 def count_usermod_objects(map_file: list[str]) -> int:
     """ Returns the number of usermod objects in the usermod list """
     # Count the number of entries in the usermods table section
-    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
+    return len([x for x in map_file if USERMODS_SECTION in x])
 
 
 def validate_map_file(source, target, env):
@@ -75,6 +70,5 @@ def validate_map_file(source, target, env):
         Exit(1)
     return None
 
-Import("env")
 env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
 env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

platformio.ini

@@ -135,6 +135,7 @@ extra_scripts =
   pre:pio-scripts/set_metadata.py
   post:pio-scripts/output_bins.py
   post:pio-scripts/strip-floats.py
+  post:pio-scripts/dynarray.py
   pre:pio-scripts/user_config_copy.py
   pre:pio-scripts/load_usermods.py
   pre:pio-scripts/build_ui.py

platformio_override.sample.ini

@@ -538,9 +538,21 @@ monitor_filters = esp32_exception_decoder
 # 433MHz RF remote example for esp32dev
 [env:esp32dev_usermod_RF433]
 extends = env:esp32dev
-build_flags = ${env:esp32dev.build_flags} -D USERMOD_RF433
-lib_deps = ${env:esp32dev.lib_deps}
-  sui77/rc-switch @ 2.6.4
+custom_usermods =
+  ${env:esp32dev.custom_usermods}
+  RF433
+
+# External usermod from a git repository.
+# The library's `library.json` must include `"build": {"libArchive": false}`.
+# The name PlatformIO assigns is taken from the library's `library.json` "name" field.
+# If that name doesn't match the repo name in the URL, use the "LibName = URL" form
+# shown in the commented-out line below to supply the name explicitly.
+[env:esp32dev_external_usermod]
+extends = env:esp32dev
+custom_usermods =
+  ${env:esp32dev.custom_usermods}
+  https://github.com/wled/wled-usermod-example.git#main
+
 
 # ------------------------------------------------------------------------------
 # Hub75 examples

tools/dynarray_espressif32.ld

@@ -0,0 +1,10 @@
+/* ESP32 linker script fragment to add dynamic array section to binary */
+SECTIONS
+{
+  .dynarray :
+  {
+    . = ALIGN(0x10);
+    KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))
+  } > default_rodata_seg
+}
+INSERT AFTER .flash.rodata;

wled00/dynarray.h

@@ -0,0 +1,34 @@
+/* dynarray.h
+
+Macros for generating a "dynamic array", a static array of objects declared in different translation units
+
+*/
+
+#pragma once
+
+// Declare the beginning and ending elements of a dynamic array of 'type'.
+// This must be used in only one translation unit in your program for any given array.
+#define DECLARE_DYNARRAY(type, array_name) \
+  static type const DYNARRAY_BEGIN(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".0"), unused)) = {}; \
+  static type const DYNARRAY_END(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".99999"), unused)) = {};
+
+// Declare an object that is a member of a dynamic array.  "member name" must be unique; "array_section" is an integer for ordering items.
+// It is legal to define multiple items with the same section name; the order of those items will be up to the linker.
+#define DYNARRAY_MEMBER(type, array_name, member_name, array_section) type const member_name __attribute__((__section__(DYNARRAY_SECTION "." #array_name "." #array_section), used))
+
+#define DYNARRAY_BEGIN(array_name) array_name##_begin
+#define DYNARRAY_END(array_name) array_name##_end
+#define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0])
+
+#ifdef ESP8266
+// ESP8266 linker script cannot be extended with a unique section for dynamic arrays.
+// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash
+// (but will never be used in the embedded system)
+#define DYNARRAY_SECTION ".dtors"
+
+#else /* ESP8266 */
+
+// Use a unique named section; the linker script must be extended to ensure it's correctly placed.
+#define DYNARRAY_SECTION ".dynarray"
+
+#endif

wled00/fcn_declare.h

@@ -1,6 +1,7 @@
 #pragma once
 #ifndef WLED_FCN_DECLARE_H
 #define WLED_FCN_DECLARE_H
+#include <dynarray.h>
 
 /*
  * All globally accessible functions are declared here
@@ -381,7 +382,7 @@ namespace UsermodManager {
 };
 
 // Register usermods by building a static list via a linker section
-#define REGISTER_USERMOD(x) Usermod* const um_##x __attribute__((__section__(".dtors.tbl.usermods.1"), used)) = &x
+#define REGISTER_USERMOD(x) DYNARRAY_MEMBER(Usermod*, usermods, um_##x, 1) = &x
 
 //usermod.cpp
 void userSetup();