Preserve source text and annotation metadata

.changeset/valid-decoration-offsets.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/highlight': patch
+---
+
+Ignore non-integer character-range offsets (NaN, infinities, and fractions), plus empty or reversed ranges, so rendering preserves source text and valid annotations. Preserve arbitrary line-decoration data keys.

docs/guides/annotations.md

@@ -44,7 +44,7 @@ const result = highlighter.highlight(code, {
 })
 ```
 
-The renderer splits token boundaries as needed and wraps the selected text with `th-decoration` plus your classes. Overlapping decorations remain valid nested HTML.
+The renderer splits token boundaries as needed and wraps the selected text with `th-decoration` plus your classes. Overlapping decorations remain valid nested HTML. Offsets must be integers with `start < end`; invalid ranges are ignored, and ranges extending outside the source are clipped.
 
 ## Line numbers
 

docs/reference/core.md

@@ -159,7 +159,7 @@ type HighlightRangeDecoration = {
 }
 ```
 
-The range uses zero-based, end-exclusive UTF-16 offsets.
+The range uses zero-based, end-exclusive UTF-16 offsets. Both offsets must be integers with `start < end`; otherwise the decoration is ignored. Ranges extending outside the source are clipped to its bounds.
 
 ### `HighlightLineDecoration`
 

src/core.ts

@@ -246,26 +246,26 @@ export function renderTokens(
   const rangeDecorations: Array<HighlightRangeDecoration> = []
   const lineDecorations: Array<HighlightLineDecoration> = []
   for (const decoration of options.decorations || []) {
-    if (decoration.range) rangeDecorations.push(decoration)
-    else lineDecorations.push(decoration)
+    if (decoration.range) {
+      const [start, end] = decoration.range
+      if (hasIntegerOffsets(start, end) && start < end) {
+        rangeDecorations.push(decoration)
+      }
+    } else lineDecorations.push(decoration)
   }
-  const wrapLines = Boolean(options.lineNumbers || lineDecorations.length)
   const cursor = { index: 0, offset: 0 }
 
-  if (!wrapLines) {
+  if (!options.lineNumbers && !lineDecorations.length) {
     return renderTokenSlice(tokens, 0, Infinity, rangeDecorations, cursor)
   }
 
   const code = tokens.map((token) => token.value).join('')
   const nodes: Array<HighlightRenderNode> = []
-  const lineStarts = getLineStarts(code)
-
-  for (let index = 0; index < lineStarts.length; index++) {
-    const line = index + 1
-    const start = lineStarts[index]
-    const nextStart = lineStarts[index + 1] ?? code.length
-    const hasNewline = nextStart > start && code[nextStart - 1] === '\n'
-    const end = hasNewline ? nextStart - 1 : nextStart
+  let start = 0
+
+  for (let line = 1; ; line++) {
+    const newline = code.indexOf('\n', start)
+    const end = newline < 0 ? code.length : newline
     const active = lineDecorations.filter((decoration) =>
       includesLine(decoration.lines, line),
     )
@@ -283,7 +283,10 @@ export function renderTokens(
       children: renderTokenSlice(tokens, start, end, rangeDecorations, cursor),
     })
 
-    if (hasNewline) nodes.push({ type: 'text', value: '\n' })
+    if (newline < 0) break
+    nodes.push({ type: 'text', value: '\n' })
+    start = end + 1
+    if (start === code.length) break
   }
 
   return nodes
@@ -327,6 +330,10 @@ function normalizeName(value: string) {
   return value.trim().toLowerCase()
 }
 
+function hasIntegerOffsets(start: number, end: number) {
+  return Number.isInteger(start) && Number.isInteger(end)
+}
+
 function normalizeTokenRanges(
   codeLength: number,
   input: ReadonlyArray<TokenRange>,
@@ -335,7 +342,7 @@ function normalizeTokenRanges(
   let sorted = true
   let previousStart = 0
   for (const candidate of input) {
-    if (!Number.isInteger(candidate.start) || !Number.isInteger(candidate.end)) continue
+    if (!hasIntegerOffsets(candidate.start, candidate.end)) continue
     const start = Math.max(0, Math.min(codeLength, candidate.start))
     const end = Math.max(0, Math.min(codeLength, candidate.end))
     if (start >= end) continue
@@ -449,14 +456,6 @@ function renderTokenSlice(
   return nodes
 }
 
-function getLineStarts(code: string) {
-  const starts = [0]
-  for (let index = 0; index < code.length; index++) {
-    if (code[index] === '\n' && index + 1 < code.length) starts.push(index + 1)
-  }
-  return starts
-}
-
 function includesLine(
   lines: number | readonly [start: number, end: number],
   line: number,
@@ -467,9 +466,14 @@ function includesLine(
 }
 
 function mergeData(decorations: ReadonlyArray<HighlightLineDecoration>) {
-  const data: Record<string, string> = {}
+  let data: Record<string, string> | undefined
   for (const decoration of decorations) {
-    Object.assign(data, stringifyData(decoration.data))
+    if (decoration.data) {
+      data = Object.assign(
+        data || Object.create(null),
+        stringifyData(decoration.data),
+      )
+    }
   }
   return data
 }

test/core.test.ts

@@ -5,6 +5,7 @@ import {
   renderNodesToHtml,
   renderTokens,
   type HighlightRenderNode,
+  type HighlightRangeDecoration,
   type HighlightToken,
   type TokenRange,
 } from '../src/core'
@@ -68,6 +69,65 @@ describe('rendering', () => {
     expect(reads).toBeLessThan(code.length * 8)
   })
 
+  it.each([false, true])(
+    'ignores invalid decoration offsets without changing source or valid ranges (lineNumbers: %s)',
+    (lineNumbers) => {
+      const code = 'a😀\r\nbc\ndef'
+      const tokens: Array<HighlightToken> = [
+        { value: 'a😀\r\nb', className: 'string' },
+        { value: 'c\ndef', className: 'keyword' },
+      ]
+      const valid: ReadonlyArray<HighlightRangeDecoration> = [
+        { range: [-5, 1], className: 'first' },
+        { range: [3, code.length + 5], className: 'rest' },
+      ]
+      const invalid: ReadonlyArray<HighlightRangeDecoration> = [
+        { range: [NaN, 1] },
+        { range: [2, NaN] },
+        { range: [-Infinity, 3] },
+        { range: [1, Infinity] },
+        { range: [1.5, 4] },
+        { range: [1, 4.5] },
+        { range: [5, 2] },
+        { range: [2, 2] },
+      ]
+
+      const expected = renderTokens(tokens, { lineNumbers, decorations: valid })
+      if (!lineNumbers) {
+        expect(renderNodesToHtml(expected)).toBe(
+          '<span class="th-decoration first"><span class="th-token th-string">a</span></span>' +
+          '<span class="th-token th-string">😀</span>' +
+          '<span class="th-decoration rest"><span class="th-token th-string">\r\nb</span></span>' +
+          '<span class="th-decoration rest"><span class="th-token th-keyword">c\ndef</span></span>',
+        )
+      }
+      for (const decoration of invalid) {
+        const actual = renderTokens(tokens, {
+          lineNumbers,
+          decorations: [decoration, ...valid],
+        })
+        expect(textContent(actual), String(decoration.range)).toBe(code)
+        expect(actual, String(decoration.range)).toEqual(expected)
+      }
+    },
+  )
+
+  it('preserves an own __proto__ line metadata key and later values', () => {
+    const data = Object.fromEntries([
+      ['__proto__', '<&"'],
+      ['constructor', 'before'],
+    ])
+    const nodes = renderTokens([{ value: 'x' }], {
+      decorations: [
+        { lines: 1, data },
+        { lines: 1, data: { constructor: 'after' } },
+      ],
+    })
+    expect(renderNodesToHtml(nodes)).toBe(
+      '<span class="th-line" data-__proto__="&lt;&amp;&quot;" data-constructor="after" data-line="1">x</span>',
+    )
+  })
+
   it('escapes source text and public render node attributes', () => {
     expect(escapeHtml(`&<>"' &amp; 😀`)).toBe('&amp;&lt;&gt;&quot;&#39; &amp;amp; 😀')
     expect(renderNodesToHtml([{