Building the Super Tiny Compiler with Reason (part 1)

src/compiler.re

@@ -0,0 +1,61 @@
+type token =
+  | OpenParen
+  | CloseParen
+  | Number string
+  | String string
+  | Name string;
+
+let explode s => {
+  let rec exp i l =>
+    if (i < 0) {
+      l
+    } else {
+      exp (i - 1) [s.[i], ...l]
+    };
+  exp (String.length s - 1) []
+};
+
+let tokenizer input => {
+  let rec tok input current tokens =>
+    switch input {
+    | [] => List.rev tokens
+    | _ =>
+      let head = List.hd input;
+      let tail = List.tl input;
+      let next = tok tail;
+      switch (head, current, tokens) {
+      /* State: None */
+      | ('(', None, t) => next None [OpenParen, ...t]
+      | (')', None, t) => next None [CloseParen, ...t]
+      | (' ' | '\t' | '\r' | '\n', None, t) => next None t
+      | ('"', None, t) => next (Some (String "")) t
+      | ('0'..'9' as i, None, t) => next (Some (Number (String.make 1 i))) t
+      | ('a'..'z' as i, None, t) => next (Some (Name (String.make 1 i))) t
+      /* State: String */
+      | ('"', Some (String c), t) => next None [String c, ...t] /* TODO allow escaped double quotes :) */
+      | (i, Some (String c), t) => next (Some (String (c ^ String.make 1 i))) t
+      /* State: Number */
+      | ('0'..'9' as i, Some (Number c), t) => next (Some (Number (c ^ String.make 1 i))) t
+      | (')', Some (Number c), t) => next None [CloseParen, Number c, ...t]
+      | (' ', Some (Number c), t) => next None [Number c, ...t]
+      /* State: Name */
+      | ('a'..'z' as i, Some (Name c), t) => next (Some (Name (c ^ String.make 1 i))) t
+      | (')', Some (Name c), t) => next None [CloseParen, Name c, ...t]
+      | (' ', Some (Name c), t) => next None [Name c, ...t]
+      /* Errors */
+      | (_, _, t) => List.rev t /* TODO: handle errors */
+      }
+    };
+  tok (explode input) None []
+};
+
+let printToken token =>
+  switch token {
+  | OpenParen => "OpenParen"
+  | CloseParen => "CloseParen"
+  | Number s => "Number " ^ s
+  | String s => "String " ^ s
+  | Name s => "Name " ^ s
+  };
+
+List.iter (fun k => Js.log (printToken k)) (tokenizer "(add 2 (subtract 4 2))");