feat: stream support

- closes #31
- closes #35

Signed-off-by: Lexus Drumgold <[email protected]>

.codecov.yml

@@ -90,6 +90,6 @@ ignore:
 
 profiling:
   critical_files_paths:
-    - src/constructs/eof.ts
     - src/constructs/initialize.ts
     - src/lexer.ts
+    - src/preprocess.ts

.dictionary.txt

@@ -13,6 +13,7 @@ fbca
 ggshield
 gpgsign
 hmarr
+iife
 jchen
 kaisugi
 lcov
@@ -22,6 +23,8 @@ mlly
 nocheck
 nvmrc
 onreturn
+onsuccessfulcheck
+onsuccessfulconstruct
 pathe
 pkgs
 preid
@@ -33,3 +36,4 @@ vates
 vfile
 vitest
 yarnrc
+zwnj

.dprint.jsonc

@@ -24,7 +24,8 @@
     "CHANGELOG.md",
     "LICENSE.md",
     "RELEASE_NOTES.md",
-    "yarn.lock"
+    "yarn.lock",
+    "__fixtures__/markdown/*.md"
   ],
   "exec": {
     "commands": [

.markdownlintignore

@@ -5,3 +5,4 @@
 **/CHANGELOG.md
 **/LICENSE.md
 **/RELEASE_NOTES.md
+__fixtures__/markdown/*.md

__fixtures__/markdown/code-fenced.md

@@ -0,0 +1,31 @@
+```
+fenced code
+```
+
+```js
+fenced code with a language
+```
+
+```js line=1
+fenced code with meta
+```
+
+~~~
+fenced code with tildes
+~~~
+
+```not fenced code```
+
+~~~fenced code~~~
+asd
+~~~
+
+```
+
+asd
+```
+
+```
+asd
+
+```

__fixtures__/markdown/code-indented.md

@@ -0,0 +1,18 @@
+    indented code
+
+
+    more indented code
+Not indented code
+
+      more
+       indent
+
+Not code.
+
+	tabs
+  	and mixed with spaces
+	  extra spaces
+
+Not code.
+
+	a tab

__fixtures__/markdown/code-text.md

@@ -0,0 +1,8 @@
+A couple of code examples: `a`, ` b`, `c `, ` d `, ` e
+`, ` f
+`, `g
+`, `
+h`.
+
+And: `alpha bravo charlie
+	delta echo`.

__fixtures__/tt.ts

@@ -0,0 +1,16 @@
+/**
+ * @file Fixtures - tt
+ * @module fixtures/tt
+ */
+
+/**
+ * Token types.
+ *
+ * @enum {string}
+ */
+enum tt {
+  eof = 'eof',
+  typeMetadata = 'typeMetadata'
+}
+
+export default tt

__fixtures__/type-metadata.txt

@@ -0,0 +1,2 @@
+{{ id: string }}
+{string

__tests__/constructs/code-fenced.ts

@@ -0,0 +1,37 @@
+/**
+ * @file Test Constructs - codeFenced
+ * @module tests/constructs/codeFenced
+ */
+
+import { codes } from '#src/enums'
+import type { Construct, TokenizeContext } from '#src/interfaces'
+import type { Code, Tokenizer } from '#src/types'
+import * as micromark from 'micromark-core-commonmark'
+
+/**
+ * Fenced code construct.
+ *
+ * @const {Construct} codeFenced
+ */
+const codeFenced: Construct = {
+  name: micromark.codeFenced.name,
+  test,
+  tokenize: micromark.codeFenced.tokenize as unknown as Tokenizer
+}
+
+export default codeFenced
+
+/**
+ * Check if the current character `code` can start this construct.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode TokenizeContext}
+ *
+ * @this {TokenizeContext}
+ *
+ * @param {Code} code - Current character code
+ * @return {boolean} `true` if `code` can start construct
+ */
+function test(this: TokenizeContext, code: Code): boolean {
+  return code === codes.graveAccent || code === codes.tilde
+}

__tests__/constructs/code-text.ts

@@ -0,0 +1,39 @@
+/**
+ * @file Test Constructs - codeText
+ * @module tests/constructs/codeText
+ */
+
+import { codes } from '#src/enums'
+import type { Construct, TokenizeContext } from '#src/interfaces'
+import type { Code, Guard, Resolver, Tokenizer } from '#src/types'
+import * as micromark from 'micromark-core-commonmark'
+
+/**
+ * Inline code construct.
+ *
+ * @const {Construct} codeText
+ */
+const codeText: Construct = {
+  name: micromark.codeText.name,
+  previous: micromark.codeText.previous as unknown as Guard,
+  resolve: micromark.codeText.resolve as unknown as Resolver,
+  test,
+  tokenize: micromark.codeText.tokenize as unknown as Tokenizer
+}
+
+export default codeText
+
+/**
+ * Check if the current character `code` can start this construct.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode TokenizeContext}
+ *
+ * @this {TokenizeContext}
+ *
+ * @param {Code} code - Current character code
+ * @return {boolean} `true` if `code` can start construct
+ */
+function test(this: TokenizeContext, code: Code): boolean {
+  return code === codes.graveAccent
+}

__tests__/constructs/eof.ts

@@ -0,0 +1,90 @@
+/**
+ * @file Test Constructs - eof
+ * @module tests/constructs/eof
+ */
+
+import tt from '#fixtures/tt'
+import { codes, ev } from '#src/enums'
+import type { Construct, Effects, TokenizeContext } from '#src/interfaces'
+import type { Code, Event, State } from '#src/types'
+
+/**
+ * End of file construct.
+ *
+ * @const {Construct} eof
+ */
+const eof: Construct = { name: tt.eof, previous, resolveAll, tokenize }
+
+export default eof
+
+/**
+ * Check if the previous character `code` can come before this construct.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode TokenizeContext}
+ *
+ * @this {TokenizeContext}
+ *
+ * @param {Code} code - Previous character code
+ * @return {boolean} `true` if `code` allowed before construct
+ */
+function previous(this: TokenizeContext, code: Code): boolean {
+  return typeof code === 'number'
+}
+
+/**
+ * Resolve all events.
+ *
+ * @see {@linkcode Event}
+ * @see {@linkcode TokenizeContext}
+ *
+ * @param {Event[]} events - List of events
+ * @param {TokenizeContext} context - Tokenize context
+ * @return {Event[]} Changed events
+ */
+function resolveAll(events: Event[], context: TokenizeContext): Event[] {
+  for (const [event, token] of events) {
+    if (event === ev.enter && token.type !== tt.eof) {
+      token.value = context.sliceSerialize(token)
+    }
+  }
+
+  return events
+}
+
+/**
+ * Set up a state machine to handle character codes streaming in.
+ *
+ * @see {@linkcode Effects}
+ * @see {@linkcode State}
+ * @see {@linkcode TokenizeContext}
+ *
+ * @this {TokenizeContext}
+ *
+ * @param {Effects} effects - Context object to transition state machine
+ * @param {State} ok - Successful tokenization state
+ * @param {State} nok - Failed tokenization state
+ * @return {State} Initial state
+ */
+function tokenize(
+  this: TokenizeContext,
+  effects: Effects,
+  ok: State,
+  nok: State
+): State {
+  return eof
+
+  /**
+   * Tokenize end of file.
+   *
+   * @param {Code} code - Current character code
+   * @return {State | undefined} Next state
+   */
+  function eof(code: Code): State | undefined {
+    if (code !== codes.eof) return nok(code)
+    effects.enter(tt.eof)
+    effects.consume(code)
+    effects.exit(tt.eof)
+    return ok
+  }
+}

__tests__/constructs/html-flow.ts

@@ -0,0 +1,38 @@
+/**
+ * @file Test Constructs - htmlFlow
+ * @module tests/constructs/htmlFlow
+ */
+
+import { codes } from '#src/enums'
+import type { Construct, TokenizeContext } from '#src/interfaces'
+import type { Code, Resolver, Tokenizer } from '#src/types'
+import * as micromark from 'micromark-core-commonmark'
+
+/**
+ * HTML flow construct.
+ *
+ * @const {Construct} htmlFlow
+ */
+const htmlFlow: Construct = {
+  name: micromark.htmlFlow.name,
+  resolveTo: micromark.htmlFlow.resolveTo as unknown as Resolver,
+  test,
+  tokenize: micromark.htmlFlow.tokenize as unknown as Tokenizer
+}
+
+export default htmlFlow
+
+/**
+ * Check if the current character `code` can start this construct.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode TokenizeContext}
+ *
+ * @this {TokenizeContext}
+ *
+ * @param {Code} code - Current character code
+ * @return {boolean} `true` if `code` can start construct
+ */
+function test(this: TokenizeContext, code: Code): boolean {
+  return code === codes.lt
+}