lua - wrap lpeg parsing around unicode protection calls

Author: cscheid

src/resources/filters/customnodes/shortcodes.lua

@@ -225,8 +225,6 @@ function shortcodes_filter()
       end
       local result = callShortcodeHandler(handler, shortcode_struct)
       return pandoc.utils.stringify(result) 
-    
-      -- return "<<<" .. table.concat(lst, " ") .. ">>>"
     end, 
   })
   local filter
@@ -265,7 +263,7 @@ function shortcodes_filter()
       return
     end
 
-    el.text = code_shortcode:match(el.text)
+    el.text = shortcode_lpeg.wrap_lpeg_match(code_shortcode, el.text)
     return el
   end
 
@@ -284,11 +282,11 @@ function shortcodes_filter()
         Shortcode = inline_handler,
         RawInline = code_handler,
         Image = function(el)
-          el.src = code_shortcode:match(el.src)
+          el.src = shortcode_lpeg.wrap_lpeg_match(code_shortcode, el.src)
           return el
         end,
         Link = function(el)
-          el.target = code_shortcode:match(el.target)
+          el.target = shortcode_lpeg.wrap_lpeg_match(code_shortcode, el.target)
           return el
         end,
         Span = function(el)

src/resources/pandoc/datadir/lpegshortcode.lua

@@ -280,8 +280,57 @@ if os.getenv("LUA_TESTING") ~= nil then
   print("Tests passed")
 end
 
+-- replace multi-character code points with an escaped version
+-- that contains an UUID that we can use to restore the original
+-- without worrying about collisions from user code that uses
+-- the same escape syntax
+local function escape_unicode(txt)
+  local result = {}
+  for _, c in utf8.codes(txt) do
+    if c > 127 then
+      table.insert(result, string.format("cf5733e5-0370-4aae-8689-61bad1dd9ec0&#x%x;", c))
+    else
+      table.insert(result, utf8.char(c))
+    end
+  end
+  return table.concat(result, "")
+end
+
+-- replace escaped code points with their unescaped version
+local function unescape_unicode(txt)
+  return txt:gsub("cf5733e5%-0370%-4aae%-8689%-61bad1dd9ec0&#x([0-9a-fA-F]+);", function (c)
+    return utf8.char(tonumber(c, 16))
+  end)
+end
+
+local function wrap_lpeg_match(pattern, txt)
+  txt = escape_unicode(txt)
+  txt = pattern:match(txt)
+  if txt == nil then
+    return nil
+  end
+  txt = unescape_unicode(txt)
+  return txt
+end
+
 return {
-  md_shortcode = md_shortcode,
+  lpegs = {
+    md_shortcode = md_shortcode,
+    unshortcode = unshortcode -- for undoing shortcodes in non-markdown contexts
+  },
+
+  parse_md_shortcode = function(txt)
+    return wrap_lpeg_match(md_shortcode, txt)
+  end,
+
+  -- use this to undo shortcode parsing in non-markdown contexts
+  unparse_md_shortcode = function(txt)
+    return wrap_lpeg_match(unshortcode, txt)
+  end,
+
   make_shortcode_parser = make_shortcode_parser,
-  unshortcode = unshortcode -- for undoing shortcodes in non-markdown contexts
+
+  -- use this to safely call an lpeg pattern with a string
+  -- that contains multi-byte code points
+  wrap_lpeg_match = wrap_lpeg_match
 }

src/resources/pandoc/datadir/readqmd.lua

@@ -126,8 +126,7 @@ end
 
 local function readqmd(txt, opts)
   txt, tags = escape_invalid_tags(txt)
-  txt = md_shortcode.md_shortcode:match(txt)
-
+  txt = md_shortcode.parse_md_shortcode(txt)
   local flavor = {
     format = "markdown",
     extensions = {},
@@ -150,7 +149,7 @@ local function readqmd(txt, opts)
 
   local unshortcode_text = function (c)
     if c.text:match("data%-is%-shortcode%=%\"1%\"") then
-      c.text = md_shortcode.unshortcode:match(c.text)
+      c.text = md_shortcode.unparse_md_shortcode(c.text)
     end
     return c
   end
@@ -159,7 +158,7 @@ local function readqmd(txt, opts)
     CodeBlock = function (cb)
       cb.classes = cb.classes:map(restore_invalid_tags)
       if cb.text:match("data%-is%-shortcode%=%\"1%\"") then
-        cb.text = md_shortcode.unshortcode:match(cb.text)
+        cb.text = md_shortcode.unparse_md_shortcode(cb.text)
       end
       cb.text = unescape_invalid_tags(cb.text, tags)
       return cb
@@ -169,13 +168,13 @@ local function readqmd(txt, opts)
     RawBlock = unshortcode_text,
     Link = function (l)
       if l.target:match("data%-is%-shortcode%=%%221%%22") then
-        l.target = md_shortcode.unshortcode:match(urldecode(l.target))
+        l.target = md_shortcode.unparse_md_shortcode(urldecode(l.target))
         return l
       end
     end,
     Image = function (i)
       if i.src:match("data%-is%-shortcode%=%%221%%22") then
-        i.src = md_shortcode.unshortcode:match(urldecode(i.src))
+        i.src = md_shortcode.unparse_md_shortcode(urldecode(i.src))
         return i
       end
     end,

tests/docs/smoke-all/2024/01/29/8485.qmd

@@ -0,0 +1,9 @@
+---
+title: issue-8485
+format: html
+classe_rugosità: value
+---
+
+this shortcode will raise a encoding error on 1.4.549
+
+{{< meta classe_rugosità >}}