package lexer import token "../token" import "core:fmt" import "core:log" import "core:reflect" import "core:strconv" import "core:testing" Lexer :: struct { input: string, position: int, read_position: int, ch: byte, } new :: proc(input: string) -> Lexer { l := Lexer{input, 0, 0, 0} read_char(&l) return l } read_char :: proc(l: ^Lexer) { if l.read_position >= len(l.input) { l.ch = 0 } else { l.ch = l.input[l.read_position] } l.position = l.read_position l.read_position += 1 } peek_char :: proc(l: ^Lexer) -> byte { if l.read_position >= len(l.input) { return 0 } else { return l.input[l.read_position] } } next_token :: proc(l: ^Lexer) -> token.Token { tok: token.Token skip_whitespace(l) switch l.ch { case '=': if peek_char(l) == '=' { tok = token.Equals{} read_char(l) } else { tok = token.Equal_Sign{} } read_char(l) case '+': tok = token.Plus{} read_char(l) case '-': tok = token.Minus{} read_char(l) case '!': if peek_char(l) == '=' { tok = token.Not_Equals{} read_char(l) } else { tok = token.Bang{} } read_char(l) case '/': tok = token.Forward_Slash{} read_char(l) case '*': tok = token.Asterisk{} read_char(l) case '<': tok = token.Less_Than{} read_char(l) case '>': tok = token.Greater_Than{} read_char(l) case ';': tok = token.Semicolon{} read_char(l) case ',': tok = token.Comma{} read_char(l) case '(': tok = token.Left_Parenthesis{} read_char(l) case ')': tok = token.Right_Parenthesis{} read_char(l) case '{': tok = token.Left_Brace{} read_char(l) case '}': tok = token.Right_Brace{} read_char(l) case 0: tok = token.EOF{} read_char(l) case: if is_letter(l.ch) { tok = read_identifier(l) } else if is_digit(l.ch) { tok = read_number(l) } else { tok = token.Illegal{} } } return tok } skip_whitespace :: proc(l: ^Lexer) { for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' { read_char(l) } } read_identifier :: proc(l: ^Lexer) -> token.Token { position := l.position for is_letter(l.ch) { read_char(l) } ident := l.input[position:l.position] return token.lookup_identifier(ident) } read_number :: proc(l: ^Lexer) -> token.Token { position := l.position for is_digit(l.ch) { read_char(l) } integer_raw := l.input[position:l.position] integer, ok := strconv.parse_int(integer_raw) if !ok { return token.Illegal{} } return token.Integer{integer} } is_letter :: proc(ch: byte) -> bool { return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' } is_digit :: proc(ch: byte) -> bool { return '0' <= ch && ch <= '9' } @(test) test_next_token :: proc(t: ^testing.T) { input := `let five = 5; let ten = 10; let add = fn(x, y) { x + y; }; let result = add(five, ten); !-/*5; 5 < 10 > 5; if (5 < 10) { return true; } else { return false; } 10 == 10; 10 != 9; ` tests := []token.Token { token.Let{}, token.Identifier{"five"}, token.Equal_Sign{}, token.Integer{5}, token.Semicolon{}, token.Let{}, token.Identifier{"ten"}, token.Equal_Sign{}, token.Integer{10}, token.Semicolon{}, token.Let{}, token.Identifier{"add"}, token.Equal_Sign{}, token.Function{}, token.Left_Parenthesis{}, token.Identifier{"x"}, token.Comma{}, token.Identifier{"y"}, token.Right_Parenthesis{}, token.Left_Brace{}, token.Identifier{"x"}, token.Plus{}, token.Identifier{"y"}, token.Semicolon{}, token.Right_Brace{}, token.Semicolon{}, token.Let{}, token.Identifier{"result"}, token.Equal_Sign{}, token.Identifier{"add"}, token.Left_Parenthesis{}, token.Identifier{"five"}, token.Comma{}, token.Identifier{"ten"}, token.Right_Parenthesis{}, token.Semicolon{}, token.Bang{}, token.Minus{}, token.Forward_Slash{}, token.Asterisk{}, token.Integer{5}, token.Semicolon{}, token.Integer{5}, token.Less_Than{}, token.Integer{10}, token.Greater_Than{}, token.Integer{5}, token.Semicolon{}, token.If{}, token.Left_Parenthesis{}, token.Integer{5}, token.Less_Than{}, token.Integer{10}, token.Right_Parenthesis{}, token.Left_Brace{}, token.Return{}, token.True{}, token.Semicolon{}, token.Right_Brace{}, token.Else{}, token.Left_Brace{}, token.Return{}, token.False{}, token.Semicolon{}, token.Right_Brace{}, token.Integer{10}, token.Equals{}, token.Integer{10}, token.Semicolon{}, token.Integer{10}, token.Not_Equals{}, token.Integer{9}, token.Semicolon{}, token.EOF{}, } l := new(input) for tt in tests { char := l.ch tok := next_token(&l) testing.expect_value(t, tok, tt) } }