scanner.go

package tokenizer

import (
	"bufio"
	"bytes"
	"io"
	"strings"
	"unicode"
)

///////////////////////////////////////////////////////////////////////////////
// TYPES

// Feature represents optional scanner features that can be enabled
type Feature uint

const (
	// HashComment enables # style comments (# comment until end of line)
	HashComment Feature = 1 << iota

	// LineComment enables // style comments (// comment until end of line)
	LineComment

	// BlockComment enables /* */ style comments (/* block comment */)
	BlockComment

	// UnderscoreToken emits underscores as separate Underscore tokens
	// When disabled, underscores are part of identifiers (hello_world)
	UnderscoreToken

	// NewlineToken emits newlines as separate Newline tokens instead of Space
	NewlineToken

	// NumberFloatToken enables parsing of floating point numbers (1.5, 3.14e10)
	// When disabled, "1.5" is parsed as NumberInteger("1") + Punkt(".") + NumberInteger("5")
	NumberFloatToken

	// SingleQuoteToken emits single quotes as separate tokens instead of parsing
	// single-quoted strings.
	SingleQuoteToken

	// DoubleQuoteToken emits double quotes as separate tokens instead of parsing
	// double-quoted strings.
	DoubleQuoteToken

	// HyphenIdentToken allows identifiers to include hyphens (e.g., "hello-world") instead
	// of treating hyphens as separate tokens.
	HyphenIdentToken
)

// Scanner represents a lexical scanner.
type Scanner struct {
	r            *bufio.Reader
	pos          Pos
	buf          []*Token
	pushback     []rune  // pushback buffer for multi-character lookahead
	lastFromPush bool    // was the last read from pushback?
	lastRune     rune    // the last rune read
	features     Feature // enabled features
	err          error
}

///////////////////////////////////////////////////////////////////////////////
// GLOBALS

const (
	tokDefaultCapacity = 20
)

var (
	hexDigits = &unicode.RangeTable{
		R16: []unicode.Range16{
			{Lo: 0x0030, Hi: 0x0039, Stride: 1}, // 0-9
			{Lo: 0x0041, Hi: 0x0046, Stride: 1}, // A-F
			{Lo: 0x0061, Hi: 0x0066, Stride: 1}, // a-f
		},
	}
	octalDigits = &unicode.RangeTable{
		R16: []unicode.Range16{
			{Lo: 0x0030, Hi: 0x0037, Stride: 1}, // 0-7
		},
	}
	binaryDigits = &unicode.RangeTable{
		R16: []unicode.Range16{
			{Lo: 0x0030, Hi: 0x0031, Stride: 1}, // 0-1
		},
	}
	numberPrefix = &unicode.RangeTable{
		R16: []unicode.Range16{
			{Lo: 0x002B, Hi: 0x002B, Stride: 1}, // +
			{Lo: 0x002D, Hi: 0x002E, Stride: 1}, // - .
		},
	}
)

///////////////////////////////////////////////////////////////////////////////
// LIFECYCLE

// NewScanner returns a new instance of Scanner with optional features.
// Features can be combined using bitwise OR: HashComment|LineComment|BlockComment
func NewScanner(r io.Reader, pos Pos, features ...Feature) *Scanner {
	var f Feature
	for _, feature := range features {
		f |= feature
	}
	return &Scanner{
		r:        bufio.NewReader(r),
		pos:      pos,
		buf:      make([]*Token, 0, tokDefaultCapacity),
		pushback: make([]rune, 0, 4),
		features: f,
	}
}

// hasFeature returns true if the scanner has the specified feature enabled
func (s *Scanner) hasFeature(f Feature) bool {
	return s.features&f != 0
}

///////////////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

// Tokens scans all remaining input and returns a slice of tokens.
// Scanning stops at EOF or when illegal input is encountered.
// Returns an error with positional information if illegal input is found.
func (s *Scanner) Tokens() ([]*Token, error) {
	result := make([]*Token, 0, tokDefaultCapacity)
	for {
		tok := s.next()
		if tok == nil {
			if s.err == nil {
				s.err = NewPosError(ErrBadParameter.With("Illegal input"), s.pos)
			}
			return nil, s.err
		} else if tok.Kind == EOF {
			break
		}
		result = append(result, tok)
	}
	// Return tokens
	return result, nil
}

// Next returns the next token and advances the scanner position.
// If the scanner has reached EOF, subsequent calls continue to return EOF.
// Use Peak() instead if you need to look ahead without consuming the token.
func (s *Scanner) Next() *Token {
	// Obtain the next token
	token := s.Peak()

	// If it's not an EOF token then remove it from the buffer
	if token != nil && token.Kind != EOF {
		s.buf = s.buf[1:]
	}

	// Return the token
	return token
}

// Peak returns the next token without advancing the scanner position.
// This allows looking ahead at upcoming tokens without consuming them.
// If the scanner has reached EOF, subsequent calls continue to return EOF.
// Note: The token is buffered, so multiple Peak() calls return the same token.
func (s *Scanner) Peak() *Token {
	// If there is already a token in the buffer, return it
	if len(s.buf) > 0 {
		return s.buf[0]
	}
	// Consume a token, add it to the buffer and return it
	token := s.next()
	if token == nil {
		if s.err == nil {
			s.err = NewPosError(ErrBadParameter.With("Illegal input"), s.pos)
		}
		token = NewToken(EOF, "", s.pos)
	}
	s.buf = append(s.buf, token)
	return token
}

// Err returns the first scanning error encountered by Peak, Next, or Tokens.
func (s *Scanner) Err() error {
	return s.err
}

// NewError wraps the given error with the scanner's current position.
// This is useful for creating error messages that include file, line,
// and column information indicating where the error occurred.
func (s *Scanner) NewError(err error) error {
	return NewPosError(err, s.pos)
}

///////////////////////////////////////////////////////////////////////////////
// PRIVATE METHODS

// Returns the next token and literal value.
func (s *Scanner) next() *Token {
	// Save position at start of token (before reading)
	startPos := s.pos

	// Read the next rune, and advance the position
	ch := s.read()

	// If we see whitespace then consume all contiguous whitespace.
	// If we see a letter then consume as an ident or reserved word.
	// If UnderscoreToken is disabled, underscore can also start an identifier.
	if unicode.IsSpace(ch) {
		s.unread()
		return s.scanWhitespace()
	} else if unicode.IsLetter(ch) || (ch == '_' && !s.hasFeature(UnderscoreToken)) {
		s.unread()
		return s.scanIdent()
	} else if unicode.IsDigit(ch) || unicode.Is(numberPrefix, ch) {
		s.unread()
		return s.scanNumber()
	} else if ch == '"' {
		if s.hasFeature(DoubleQuoteToken) {
			return NewToken(DoubleQuote, string(ch), startPos)
		}
		s.unread()
		return s.scanString()
	} else if ch == '\'' {
		if s.hasFeature(SingleQuoteToken) {
			return NewToken(SingleQuote, string(ch), startPos)
		}
		s.unread()
		return s.scanString()
	} else if ch == '#' && s.hasFeature(HashComment) {
		s.unread()
		return s.scanHashComment()
	} else if ch == '/' && s.hasFeature(LineComment|BlockComment) {
		s.unread()
		return s.scanSlashComment()
	}

	// Otherwise read the individual character
	if kind, exists := tokenKindMap[ch]; exists {
		return NewToken(kind, string(ch), startPos)
	} else {
		return nil
	}
}

// scanWhitespace consumes the current rune and all contiguous whitespace.
// If NewlineToken feature is enabled, newlines are emitted as separate Newline tokens.
func (s *Scanner) scanWhitespace() *Token {
	// Save position at start of token
	startPos := s.pos

	// Read the first character
	ch := s.read()

	// If NewlineToken is enabled and this is a newline, emit it separately
	if s.hasFeature(NewlineToken) && ch == '\n' {
		return NewToken(Newline, string(ch), startPos)
	}

	// Create a buffer and read the current character into it.
	var buf bytes.Buffer
	buf.WriteRune(ch)

	// Read every subsequent whitespace character into the buffer.
	// Non-whitespace characters and EOF will cause the loop to exit.
	// If NewlineToken is enabled, stop at newlines.
	for {
		if ch := s.read(); ch == eof {
			break
		} else if ch == '\n' && s.hasFeature(NewlineToken) {
			s.unread()
			break
		} else if !unicode.IsSpace(ch) {
			s.unread()
			break
		} else {
			buf.WriteRune(ch)
		}
	}

	return NewToken(Space, buf.String(), startPos)
}

// scanHashComment consumes the current rune and all contiguous runes until a newline
func (s *Scanner) scanHashComment() *Token {
	// Save position at start of token
	startPos := s.pos

	// Create a buffer and read the current character into it.
	var buf bytes.Buffer
	buf.WriteRune(s.read())

	// Read every subsequent character into the buffer.
	// Newlines and EOF will cause the loop to exit.
	for {
		if ch := s.read(); ch == eof {
			break
		} else if ch == '\n' {
			s.unread()
			break
		} else {
			buf.WriteRune(ch)
		}
	}

	return NewToken(Comment, buf.String(), startPos)
}

// scanSlashComment expects another slash and then consumes the current rune and all
// contiguous runes until a newline
func (s *Scanner) scanSlashComment() *Token {
	// Save position at start of token
	startPos := s.pos

	// Create a buffer and read the current character into it.
	var buf bytes.Buffer
	buf.WriteRune(s.read()) // Read the first '/'

	// Peek at the next character to determine comment type
	ch := s.read()
	if ch == '/' && s.hasFeature(LineComment) {
		// Single line comment
		buf.WriteRune(ch)
		for {
			ch = s.read()
			if ch == eof || ch == '\n' {
				if ch == '\n' {
					s.unread()
				}
				break
			}
			buf.WriteRune(ch)
		}
		return NewToken(Comment, buf.String(), startPos)
	} else if ch == '*' && s.hasFeature(BlockComment) {
		// Block comment
		buf.WriteRune(ch)
		for {
			ch = s.read()
			if ch == eof {
				// Unterminated block comment
				return nil
			}
			buf.WriteRune(ch)
			if ch == '*' {
				// Check if next char is '/'
				next := s.read()
				if next == '/' {
					buf.WriteRune(next)
					break
				} else if next != eof {
					s.unread()
				} else {
					// EOF after *
					return nil
				}
			}
		}
		return NewToken(Comment, buf.String(), startPos)
	} else {
		// Not a comment, just a single slash (divide operator)
		if ch != eof {
			s.unread()
		}
		return NewToken(Divide, buf.String(), startPos)
	}
}

// scanIdent consumes the current rune and all contiguous ident runes.
func (s *Scanner) scanIdent() *Token {
	// Save position at start of token
	startPos := s.pos

	// Create a buffer and read the current character into it.
	var buf bytes.Buffer
	buf.WriteRune(s.read())

	// Read every subsequent ident character into the buffer.
	// Non-ident characters and EOF will cause the loop to exit.
	// Underscores are only included if UnderscoreToken feature is disabled.
	// Hyphens are only included if HyphenIdentToken is enabled.
	for {
		if ch := s.read(); ch == eof {
			break
		} else if ch == '_' && s.hasFeature(UnderscoreToken) {
			s.unread()
			break
		} else if ch == '-' && !s.hasFeature(HyphenIdentToken) {
			s.unread()
			break
		} else if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' && !(ch == '-' && s.hasFeature(HyphenIdentToken)) {
			s.unread()
			break
		} else {
			_, _ = buf.WriteRune(ch)
		}
	}

	// Reserved words
	keyword := strings.ToLower(buf.String())
	if kind, exists := tokenKeywordMap[keyword]; exists {
		return NewToken(kind, buf.String(), startPos)
	}

	// Otherwise return as a regular identifier.
	return NewToken(Ident, buf.String(), startPos)
}

// scanString consumes a contiguous string of non-quote characters.
// Quote characters can be consumed if they're first escaped with a backslash.
func (s *Scanner) scanString() *Token {
	// Save position at start of token
	startPos := s.pos

	// Read the delimiter
	ending := s.read()

	// Create a buffer and read the current character into it.
	var buf bytes.Buffer

	// Read every subsequent character into the buffer.
	for {
		if ch := s.read(); ch == eof {
			// Return nil if the string is not terminated
			return nil
		} else if ch == ending {
			break
		} else if ch == '\\' {
			// If the next character is an escape then write the escaped char
			next := s.read()
			if next == eof {
				// Unterminated escape
				return nil
			} else if next == 'n' {
				buf.WriteRune('\n')
			} else if next == 'r' {
				buf.WriteRune('\r')
			} else if next == 't' {
				buf.WriteRune('\t')
			} else if next == '\\' {
				buf.WriteRune('\\')
			} else if next == '"' {
				buf.WriteRune('"')
			} else if next == '\'' {
				buf.WriteRune('\'')
			} else {
				// Invalid escape
				return nil
			}
		} else {
			buf.WriteRune(ch)
		}
	}

	// Return the string
	return NewToken(String, buf.String(), startPos)
}

// scanNumber consumes all kinds of numbers
func (s *Scanner) scanNumber() *Token {
	// Save position at start of token
	startPos := s.pos

	// Create a buffer
	var buf bytes.Buffer
	// Exponent is true if we've seen an E
	var Exponent bool
	// Set the default kind
	kind := NumberInteger

	// Read the first character
	ch := s.read()

	// Handle leading +, -, or .
	if ch == '+' || ch == '-' {
		// Peek at the next character
		next := s.read()
		if next == eof {
			// Just a + or - on its own
			if k, exists := tokenKindMap[ch]; exists {
				return NewToken(k, string(ch), startPos)
			}
			return nil
		}
		if next == '.' {
			if !s.hasFeature(NumberFloatToken) {
				// Don't parse +. or -. as float start
				s.pushbackRune(next) // push back the dot
				if k, exists := tokenKindMap[ch]; exists {
					return NewToken(k, string(ch), startPos)
				}
				return nil
			}
			// Check what comes after the dot
			afterDot := s.read()
			if afterDot == eof || !unicode.IsDigit(afterDot) {
				// -. or +. followed by non-digit or EOF
				// Return just the + or -, push back the dot (and afterDot if not EOF)
				if afterDot != eof {
					s.pushbackRune(afterDot)
				}
				s.pushbackRune(next) // push back the dot
				if k, exists := tokenKindMap[ch]; exists {
					return NewToken(k, string(ch), startPos)
				}
				return nil
			}
			// We have +.N or -.N - it's a float
			buf.WriteRune(ch)
			buf.WriteRune(next)
			buf.WriteRune(afterDot)
			kind = NumberFloat
		} else if !unicode.IsDigit(next) {
			// +e, -e, +x, -x etc - return just the + or -
			s.pushbackRune(next)
			if k, exists := tokenKindMap[ch]; exists {
				return NewToken(k, string(ch), startPos)
			}
			return nil
		} else {
			// We have +N or -N
			buf.WriteRune(ch)
			buf.WriteRune(next)
			// Check for octal prefix (leading 0) - but only if there are more digits
			if next == '0' {
				peek := s.read()
				if peek != eof {
					s.pushbackRune(peek)
					// Only treat as potential octal if followed by another digit or x/b
					if peek >= '0' && peek <= '9' || peek == 'x' || peek == 'X' || peek == 'b' || peek == 'B' {
						kind = NumberOctal
					}
				}
				// Otherwise keep it as NumberInteger (just "0" or "-0" or "+0")
			}
		}
	} else if ch == '.' {
		// Leading dot - must be followed by digit for float, and FloatToken must be enabled
		if !s.hasFeature(NumberFloatToken) {
			// Just a dot
			return NewToken(Punkt, string(ch), startPos)
		}
		next := s.read()
		if next == eof || !unicode.IsDigit(next) {
			if next != eof {
				s.pushbackRune(next)
			}
			// Just a dot
			return NewToken(Punkt, string(ch), startPos)
		}
		buf.WriteRune(ch)
		buf.WriteRune(next)
		kind = NumberFloat
	} else if unicode.IsDigit(ch) {
		buf.WriteRune(ch)
		if ch == '0' {
			// Peek to see if this is octal/hex/binary prefix
			peek := s.read()
			if peek != eof {
				s.pushbackRune(peek)
				// Only treat as potential octal if followed by another digit or x/b
				if peek >= '0' && peek <= '9' || peek == 'x' || peek == 'X' || peek == 'b' || peek == 'B' {
					kind = NumberOctal
				}
			}
			// Otherwise keep it as NumberInteger (just "0")
		}
	} else {
		// Not a number
		s.unread()
		return nil
	}

	// Read the rest of the number
FOR_LOOP:
	for {
		ch = s.read()
		switch {
		case ch == eof:
			break FOR_LOOP
		case kind == NumberOctal && (ch == 'x' || ch == 'X'):
			// Switch to hex
			str := buf.String()
			if str == "0" || str == "-0" || str == "+0" {
				kind = NumberHex
				buf.WriteRune(ch)
			} else {
				return nil
			}
		case kind == NumberOctal && (ch == 'b' || ch == 'B'):
			// Switch to binary
			str := buf.String()
			if str == "0" || str == "-0" || str == "+0" {
				kind = NumberBinary
				buf.WriteRune(ch)
			} else {
				return nil
			}
		case kind == NumberHex:
			if unicode.Is(hexDigits, ch) {
				buf.WriteRune(ch)
			} else {
				s.unread()
				break FOR_LOOP
			}
		case kind == NumberOctal:
			if unicode.Is(octalDigits, ch) {
				buf.WriteRune(ch)
			} else if ch == '.' && s.hasFeature(NumberFloatToken) {
				// Switch to float (e.g., 0.5)
				kind = NumberFloat
				buf.WriteRune(ch)
			} else if ch == '.' {
				// No float support - return integer
				s.unread()
				break FOR_LOOP
			} else if !unicode.IsDigit(ch) {
				s.unread()
				break FOR_LOOP
			} else {
				// Invalid octal digit (8 or 9)
				return nil
			}
		case kind == NumberBinary:
			if unicode.Is(binaryDigits, ch) {
				buf.WriteRune(ch)
			} else {
				s.unread()
				break FOR_LOOP
			}
		case kind == NumberInteger && ch == '.':
			if !s.hasFeature(NumberFloatToken) {
				// Don't parse as float - return integer and let dot be separate token
				s.unread()
				break FOR_LOOP
			}
			kind = NumberFloat
			buf.WriteRune(ch)
		case kind == NumberInteger && unicode.IsDigit(ch):
			buf.WriteRune(ch)
		case kind == NumberFloat && unicode.IsDigit(ch):
			buf.WriteRune(ch)
		case (kind == NumberFloat || kind == NumberInteger) && (ch == 'e' || ch == 'E'):
			if !s.hasFeature(NumberFloatToken) {
				// No float support - 'e' ends the number
				s.unread()
				break FOR_LOOP
			}
			if Exponent {
				return nil
			}
			Exponent = true
			kind = NumberFloat
			buf.WriteRune(ch)
			// Check for optional +/- after exponent
			next := s.read()
			if next == '+' || next == '-' {
				// Must be followed by a digit
				afterSign := s.read()
				if afterSign == eof || !unicode.IsDigit(afterSign) {
					return nil
				}
				buf.WriteRune(next)
				buf.WriteRune(afterSign)
			} else if unicode.IsDigit(next) {
				buf.WriteRune(next)
			} else {
				// E not followed by digit or sign
				return nil
			}
		default:
			s.unread()
			break FOR_LOOP
		}
	}

	// Validate we have a proper number
	str := buf.String()
	if len(str) == 0 {
		return nil
	}

	// Check hex/binary have actual digits after prefix
	if kind == NumberHex {
		// Must have at least one hex digit after 0x
		if len(str) <= 2 || (len(str) <= 3 && (str[0] == '+' || str[0] == '-')) {
			return nil
		}
	}
	if kind == NumberBinary {
		// Must have at least one binary digit after 0b
		if len(str) <= 2 || (len(str) <= 3 && (str[0] == '+' || str[0] == '-')) {
			return nil
		}
	}

	return NewToken(kind, str, s.pos)
}

// read the next rune from the buffered reader or pushback buffer.
// Returns the rune(0) if an error occurs (or io.EOF is returned).
func (s *Scanner) read() rune {
	// Check pushback buffer first (LIFO - last in, first out)
	if len(s.pushback) > 0 {
		ch := s.pushback[len(s.pushback)-1]
		s.pushback = s.pushback[:len(s.pushback)-1]
		// Track that this came from pushback
		s.lastFromPush = true
		s.lastRune = ch
		// Mark previous position and advance
		s.pos.x, s.pos.y = s.pos.Line, s.pos.Col
		if ch == '\n' {
			s.pos.Line++
			s.pos.Col = 0
		} else if ch != eof {
			s.pos.Col++
		}
		return ch
	}

	// Read a rune from the reader
	ch, _, err := s.r.ReadRune()
	if err != nil {
		s.lastFromPush = false
		s.lastRune = eof
		return eof
	}

	// Track that this came from reader
	s.lastFromPush = false
	s.lastRune = ch

	// Mark previous position
	s.pos.x, s.pos.y = s.pos.Line, s.pos.Col

	// Advance position
	if ch == '\n' {
		s.pos.Line++
		s.pos.Col = 0
	} else if ch != eof {
		s.pos.Col++
	}
	return ch
}

// unread places the previously read rune back for re-reading.
func (s *Scanner) unread() {
	if s.lastFromPush {
		// If last read was from pushback, push it back there
		s.pushback = append(s.pushback, s.lastRune)
	} else {
		// Otherwise use the underlying reader's UnreadRune
		_ = s.r.UnreadRune()
	}
	// Restore previous position
	s.pos.Line, s.pos.Col, s.pos.x, s.pos.y = s.pos.x, s.pos.y, 0, 0
}

// pushbackRune pushes a rune back onto the pushback buffer for re-reading.
// This supports multiple levels of pushback unlike unread().
func (s *Scanner) pushbackRune(ch rune) {
	s.pushback = append(s.pushback, ch)
	// Restore previous position
	s.pos.Line, s.pos.Col, s.pos.x, s.pos.y = s.pos.x, s.pos.y, 0, 0
}