Lexer.fsl

{
open FSharp.Text.Lexing
open Parser

let lexeme lexbuf = LexBuffer<_>.LexemeString lexbuf
}

let TRUE  = "true"
let FALSE = "false"
let NULL  = "null"
let char  = ['a'-'z' 'A'-'Z']
let digit = [ '0'-'9' ]
let int   = digit+
let identifier = char(char|digit|'_')*
let strliteral = '"'[^'"']*'"'
let whitespace = [ ' ' '\t' ]
let newline = '\n' | 'r' | "\n\r"

rule tokenize = parse
| whitespace { tokenize lexbuf; }
| newline    { lexbuf. EndPos <- lexbuf.EndPos.NextLine; tokenize lexbuf; }
| '+'        { ADD }
| '-'        { SUB }
| ';'        { SEMICOLON }
| ','        { COMMA }
| '='        { EQ }
| "=="       { EQUAL }
| "!="       { NOTEQ }
| '<'        { LT }
| '>'        { GT }
| "<="       { LE }
| ">="       { GE }
| '('        { LPAR }
| ')'        { RPAR }
| '['        { LSQR }
| ']'        { RSQR }
| '{'        { LCUR }
| '}'        { RCUR }
| TRUE       { TrueTok }
| FALSE      { FalseTok }
| NULL       { NullTok }
| "let"      { LET }
| "fn"       { FN }
| int        { INT <| lexeme lexbuf }
| identifier { ID  <| lexeme lexbuf }
| strliteral { let s = lexeme lexbuf in s.[1..((String.length s) - 2)] |> STR }
| eof        { EOF }
| _          { INVALID }