feat(mdxish): add new MDXish engine (#1243)

[![PR App][icn]][demo] | Fix RM-XYZ
:-------------------:|:----------:

> [!WARNING]
> As of `5 Dec 2025` (keeping this to track timeline) SSR for `mdxish`
has been disabled. All contents that are rendered by `mdxish` will be
done client-side. For more context, refer to this
[thread](#1243 (comment))

## 🧰 Changes

#### Context
This PR exports 2 new libraries which provides a new way to render mixed
Markdown + MDX content in our application.
This allows customers to flexibly embed MDX inside Markdown without
relying on the strict MDX renderer or needing to migrate everything to
MDX (which currently causes many errors and requires hours of cleanup)

> [!IMPORTANT]
> With the addition of the new libraries, we unfortunately have exceeded
the maximum bundle size allowed. Specifically the current bundle size is
`762KB` and the limit was `750KB`, this has been increased to `775KB`

#### Changes
1. `mdxish.ts`

- Engine to convert a Markdown + MDX string into an HTML AST.
- Based on Greg’s prototype and uses Unified plugins.
- Handles MDX by preprocessing its syntax.
- Additional logic:
  - Reuses existing transformers (e.g., callouts).
  - Adjusts MDX nodes when spacing breaks the AST.
- Custom component handling:
  - Recursively parses inner content.
  - Renames nodes to PascalCase.
- Includes heuristics to determine whether a tag is a real component vs.
an HTML tag.

2. `renderMdxish.tsx`

- Converts the HTML AST into React JSX components
- Mimics the existing `run.tsx` behaviour used in production, returns an
RMDXModule which contains the content react component, and the table of
contents

> _We also expose another library called `mix` but this isnt actually
used for rendering. This is simply a wrapper around `mdxish` that
returns stringified HTML instead of HAST. This can be useful for
testing/development or when we need a stringified version of the HAST._

## 🧬 QA & Testing

#### How to Test Locally
To test this new rendering engine directly in the ReadMe app:
- Open two terminals:
  - ReadMe (branch: mdxish-demo)
  - Markdown (this branch)
- Link Markdown to ReadMe:
  - In markdown: `npm ci` && `npm run build`
  - In readme: `make link-markdown`
- Run the ReadMe repo and open any project — it should not crash
- Create docs using mixed Markdown/MDX (via Raw mode) and verify they
render correctly
- You can use the files in `tests/lib/mdxish/demo-docs` as examples in
your editor

#### Testing In The PR App
We have prepared a PR app that has the new mdxish engine enabled by
default for all projects. See it
[here](https://readme-pr-16565.readme.ninja/)

#### Things to Test in Docs

- Built-in ReadMe components
- User-defined components
- Table of contents
- Unclosed tags (e.g., `<br>`)
- Links, headings, formatting, etc.

## 📸 Some Screenshots
These screenshots are sample MD/MDX pages that is rendered using the new
libraries. All screenshots here and all demo does not have correct
validation _yet_. We purposefuly disabled validation to demo this new
engine/library.

|  |  |  |
| --- | --- | --- |
| <img width="1920" height="1113" alt="Screenshot 2025-11-27 at 01 05
44"
src="https://github.com/user-attachments/assets/1b8037c2-1d80-4980-8417-5f61411c5df8"
/> | <img width="1920" height="1113" alt="Screenshot 2025-11-27 at 01 06
10"
src="https://github.com/user-attachments/assets/37e597a3-6ef2-4844-b5fd-86604936b0b3"
/> | <img width="1920" height="1113" alt="Screenshot 2025-11-27 at 01 06
27"
src="https://github.com/user-attachments/assets/012b92f6-56f9-41b4-b000-6db4a1d346f1"
/> |


[demo]: https://markdown-pr-PR_NUMBER.herokuapp.com
[prod]: https://SUBDOMAIN.readme.io
[icn]:
https://user-images.githubusercontent.com/886627/160426047-1bee9488-305a-4145-bb2b-09d8b757d38a.svg

---------

Co-authored-by: Dimas Putra Anugerah <[email protected]>
Co-authored-by: eagletrhost <[email protected]>
Co-authored-by: Rafe Goldberg <[email protected]>

Author: 3 people

__tests__/benchmarks/engine.bench.ts

@@ -1,4 +1,6 @@
-import { mdast, mdx } from '../index';
+import type { Element } from 'hast';
+
+import { mdast, mdx, mdxish } from '../index';
 
 describe('ReadMe Flavored Blocks', () => {
   it('Embed', () => {
@@ -15,3 +17,28 @@ describe('ReadMe Flavored Blocks', () => {
     `);
   });
 });
+
+describe('mdxish ReadMe Flavored Blocks', () => {
+  it('Embed', () => {
+    const txt = '[Embedded meta links.](https://nyti.me/s/gzoa2xb2v3 "@embed")';
+    const hast = mdxish(txt);
+    const embed = hast.children[0] as Element;
+
+    expect(embed.type).toBe('element');
+    expect(embed.tagName).toBe('embed');
+    expect(embed.properties.url).toBe('https://nyti.me/s/gzoa2xb2v3');
+    expect(embed.properties.title).toBe('Embedded meta links.');
+  });
+
+  it('Emojis', () => {
+    const hast = mdxish(':smiley:');
+    const paragraph = hast.children[0] as Element;
+
+    expect(paragraph.type).toBe('element');
+    expect(paragraph.tagName).toBe('p');
+    // gemojiTransformer converts :smiley: to 😃
+    const textNode = paragraph.children[0];
+    expect(textNode.type).toBe('text');
+    expect('value' in textNode && textNode.value).toBe('😃');
+  });
+});

__tests__/compilers.test.ts

@@ -1,6 +1,7 @@
+import type { Element } from 'hast';
 import type { Root } from 'mdast';
 
-import { mdast, mdx } from '../../index';
+import { mdast, mdx, mdxish } from '../../index';
 
 describe('callouts compiler', () => {
   it('compiles callouts', () => {
@@ -156,3 +157,134 @@ describe('callouts compiler', () => {
     expect(mdx(mockAst as Root).trim()).toBe(markdown);
   });
 });
+
+describe('mdxish callout compiler', () => {
+  it('compiles callouts', () => {
+    const markdown = `> 🚧 It works!
+>
+> And, it no longer deletes your content!
+`;
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.theme).toBe('warn');
+    expect(callout.children).toHaveLength(2); // h3 and p
+  });
+
+  it('compiles callouts with no heading', () => {
+    const markdown = `> 🚧
+>
+> And, it no longer deletes your content!
+`;
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.empty).toBe('');
+    expect(callout.properties?.theme).toBe('warn');
+  });
+
+  it('compiles callouts with no heading or body', () => {
+    const markdown = `> 🚧
+`;
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.empty).toBe('');
+    expect(callout.properties?.theme).toBe('warn');
+  });
+
+  it('compiles callouts with no heading or body and no new line at the end', () => {
+    const markdown = '> ℹ️';
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('ℹ️');
+    expect(callout.properties?.empty).toBe('');
+    expect(callout.properties?.theme).toBe('info');
+  });
+
+  it('compiles callouts with markdown in the heading', () => {
+    const markdown = `> 🚧 It **works**!
+>
+> And, it no longer deletes your content!
+`;
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.theme).toBe('warn');
+
+    const heading = callout.children[0] as Element;
+    expect(heading.tagName).toBe('h3');
+    expect(heading.properties?.id).toBe('it-works');
+  });
+
+  it('compiles callouts with paragraphs', () => {
+    const markdown = `> 🚧 It **works**!
+>
+> And...
+>
+> it correctly compiles paragraphs. :grimace:
+`;
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.theme).toBe('warn');
+    expect(callout.children.length).toBeGreaterThan(1); // heading + multiple paragraphs
+  });
+
+  it('compiles callouts with icons + theme', () => {
+    const markdown = `
+<Callout icon="fad fa-wagon-covered" theme="warn">
+  test
+</Callout>`.trim();
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('fad fa-wagon-covered');
+    expect(callout.properties?.theme).toBe('warn');
+  });
+
+  it('compiles a callout with only a theme set', () => {
+    const markdown = '> 🚧 test';
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.theme).toBe('warn');
+
+    const heading = callout.children[0] as Element;
+    expect(heading.tagName).toBe('h3');
+  });
+
+  it('compiles a callout with only an icon set', () => {
+    const markdown = '> 🚧 test';
+
+    const hast = mdxish(markdown);
+    const callout = hast.children[0] as Element;
+
+    expect(callout.tagName).toBe('Callout');
+    expect(callout.properties?.icon).toBe('🚧');
+    expect(callout.properties?.theme).toBe('warn'); // defaults based on icon
+  });
+});

__tests__/compilers/callout.test.ts

@@ -1,4 +1,4 @@
-import { mdast, mdx } from '../../index';
+import { mdast, mdx, mdxish } from '../../index';
 
 describe('code-tabs compiler', () => {
   it('compiles code tabs', () => {
@@ -41,3 +41,65 @@ I should stay here
     expect(mdx(mdast(markdown))).toBe(markdown);
   });
 });
+
+describe('mdxish code-tabs compiler', () => {
+  it('compiles code tabs', () => {
+    const markdown = `\`\`\`
+const works = true;
+\`\`\`
+\`\`\`
+const cool = true;
+\`\`\`
+`;
+
+    const hast = mdxish(markdown);
+    // Code blocks should be grouped into CodeTabs
+    const firstChild = hast.children[0];
+    
+    expect(firstChild.type).toBe('element');
+    expect(firstChild.tagName).toBe('CodeTabs');
+    expect(firstChild.children).toHaveLength(2); // Two code blocks
+  });
+
+  it('compiles code tabs with metadata', () => {
+    const markdown = `\`\`\`js Testing
+const works = true;
+\`\`\`
+\`\`\`js
+const cool = true;
+\`\`\`
+`;
+
+    const hast = mdxish(markdown);
+    const firstChild = hast.children[0];
+    
+    expect(firstChild.type).toBe('element');
+    expect(firstChild.tagName).toBe('CodeTabs');
+    expect(firstChild.children).toHaveLength(2); // Two code blocks
+  });
+
+  it("doesnt't mess with joining other blocks", () => {
+    const markdown = `\`\`\`
+const works = true;
+\`\`\`
+\`\`\`
+const cool = true;
+\`\`\`
+
+## Hello!
+
+I should stay here
+`;
+
+    const hast = mdxish(markdown);
+    // CodeTabs should be first
+    const firstChild = hast.children[0];
+    expect(firstChild.type).toBe('element');
+    expect(firstChild.tagName).toBe('CodeTabs');
+    
+    // Then heading
+    const heading = hast.children.find(c => c.type === 'element' && c.tagName === 'h2');
+    expect(heading).toBeDefined();
+    expect(heading.tagName).toBe('h2');
+  });
+});

__tests__/compilers/code-tabs.test.js

@@ -1,11 +1,13 @@
+import type { CustomComponents } from '../../types';
+import type { Element } from 'hast';
+
 import fs from 'node:fs';
 
 import { render, screen } from '@testing-library/react';
-import React from 'react';
 
 import { vi } from 'vitest';
 
-import { mdx, compile, run } from '../../index';
+import { mdx, compile, run, mdxish } from '../../index';
 import { migrate } from '../helpers';
 
 describe('compatability with RDMD', () => {
@@ -507,3 +509,74 @@ ${JSON.stringify(
     `);
   });
 });
+
+describe('mdxish compatability with RDMD', () => {
+  it('processes Glossary component', () => {
+    const markdown = '<Glossary>parliament</Glossary>';
+
+    const hast = mdxish(markdown);
+    const paragraph = hast.children[0] as Element;
+    const glossary = paragraph.children[0] as Element;
+
+    expect(paragraph.type).toBe('element');
+    expect(paragraph.tagName).toBe('p');
+    expect(glossary.type).toBe('element');
+    expect(glossary.tagName).toBe('Glossary');
+    const textNode = glossary.children[0];
+    expect(textNode.type).toBe('text');
+    expect('value' in textNode && textNode.value).toBe('parliament');
+  });
+
+  it('processes Image component with attributes and caption', () => {
+    const markdown = `<Image align="center" width="300px" src="https://drastik.ch/wp-content/uploads/2023/06/blackcat.gif" border={true}>
+hello **cat**
+</Image>`;
+
+    const hast = mdxish(markdown.trim());
+    const image = hast.children[0] as Element;
+
+    expect(image.type).toBe('element');
+    expect(image.tagName).toBe('img');
+    expect(image.properties.align).toBe('center');
+    expect(image.properties.width).toBe('300px');
+    expect(image.properties.src).toBe('https://drastik.ch/wp-content/uploads/2023/06/blackcat.gif');
+    expect(image.properties.border).toBe('true');
+    // Caption text should be processed (but Image components don't support captions in mdxish)
+  });
+
+  it('processes Embed component with attributes', () => {
+    const markdown =
+      '<Embed url="https://cdn.shopify.com/s/files/1/0711/5132/1403/files/BRK0502-034178M.pdf" title="iframe" href="https://cdn.shopify.com/s/files/1/0711/5132/1403/files/BRK0502-034178M.pdf" typeOfEmbed="iframe" height="300px" width="100%" iframe="true" />';
+
+    const hast = mdxish(markdown);
+    const embed = hast.children[0] as Element;
+
+    expect(embed.type).toBe('element');
+    expect(embed.tagName).toBe('embed');
+    expect(embed.properties.url).toBe('https://cdn.shopify.com/s/files/1/0711/5132/1403/files/BRK0502-034178M.pdf');
+    expect(embed.properties.title).toBe('iframe');
+    expect(embed.properties.typeOfEmbed).toBe('iframe');
+    expect(embed.properties.height).toBe('300px');
+    expect(embed.properties.width).toBe('100%');
+    expect(embed.properties.iframe).toBe('true');
+  });
+
+  it('processes reusable content component', () => {
+    const markdown = '<Parliament />';
+
+    const hast = mdxish(markdown, {
+      components: {
+        Parliament: '# Parliament',
+      },
+    } as unknown as CustomComponents);
+
+    // Component is recognized and preserved in HAST
+    expect(hast.children.length).toBeGreaterThan(0);
+    const component = hast.children.find(
+      child => child.type === 'element' && (child as Element).tagName === 'Parliament',
+    ) as Element | undefined;
+    expect(component).toBeDefined();
+    expect(component?.type).toBe('element');
+    expect(component?.tagName).toBe('Parliament');
+  });
+});

__tests__/compilers/compatability.test.tsx

@@ -1,4 +1,4 @@
-import { mdast, mdx } from '../../index';
+import { mdast, mdx, mdxish } from '../../index';
 
 describe('escape compiler', () => {
   it('handles escapes', () => {
@@ -7,3 +7,17 @@ describe('escape compiler', () => {
     expect(mdx(mdast(txt))).toBe('\\¶\n');
   });
 });
+
+describe('mdxish escape compiler', () => {
+  it('handles escapes', () => {
+    const txt = '\\¶';
+
+    const hast = mdxish(txt);
+    const paragraph = hast.children[0];
+    
+    expect(paragraph.type).toBe('element');
+    expect(paragraph.tagName).toBe('p');
+    expect(paragraph.children[0].type).toBe('text');
+    expect(paragraph.children[0].value).toBe('¶');
+  });
+});