From baa37871d9b71d3f94cc8acca8d375ee1232d7f0 Mon Sep 17 00:00:00 2001 From: Ibrahim Muftee Date: Tue, 14 Jul 2026 22:18:38 -0500 Subject: chapter 01 --- lexer/lexer.odin | 257 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 lexer/lexer.odin (limited to 'lexer') diff --git a/lexer/lexer.odin b/lexer/lexer.odin new file mode 100644 index 0000000..d4ad9b3 --- /dev/null +++ b/lexer/lexer.odin @@ -0,0 +1,257 @@ +package lexer + +import token "../token" +import "core:fmt" +import "core:log" +import "core:reflect" +import "core:strconv" +import "core:testing" + +Lexer :: struct { + input: string, + position: int, + read_position: int, + ch: byte, +} + +new :: proc(input: string) -> Lexer { + l := Lexer{input, 0, 0, 0} + read_char(&l) + return l +} + +read_char :: proc(l: ^Lexer) { + if l.read_position >= len(l.input) { + l.ch = 0 + } else { + l.ch = l.input[l.read_position] + } + l.position = l.read_position + l.read_position += 1 +} + +peek_char :: proc(l: ^Lexer) -> byte { + if l.read_position >= len(l.input) { + return 0 + } else { + return l.input[l.read_position] + } + +} + +next_token :: proc(l: ^Lexer) -> token.Token { + tok: token.Token + + skip_whitespace(l) + + switch l.ch { + case '=': + if peek_char(l) == '=' { + tok = token.Equals{} + read_char(l) + } else { + tok = token.Equal_Sign{} + } + read_char(l) + case '+': + tok = token.Plus{} + read_char(l) + case '-': + tok = token.Minus{} + read_char(l) + case '!': + if peek_char(l) == '=' { + tok = token.Not_Equals{} + read_char(l) + } else { + tok = token.Bang{} + } + read_char(l) + case '/': + tok = token.Forward_Slash{} + read_char(l) + case '*': + tok = token.Asterisk{} + read_char(l) + case '<': + tok = token.Less_Than{} + read_char(l) + case '>': + tok = token.Greater_Than{} + read_char(l) + case ';': + tok = token.Semicolon{} + read_char(l) + case ',': + tok = token.Comma{} + read_char(l) + case '(': + tok = token.Left_Parenthesis{} + read_char(l) + case ')': + tok = token.Right_Parenthesis{} + read_char(l) + case '{': + tok = token.Left_Brace{} + read_char(l) + case '}': + tok = token.Right_Brace{} + read_char(l) + case 0: + tok = token.EOF{} + read_char(l) + case: + if is_letter(l.ch) { + tok = read_identifier(l) + } else if is_digit(l.ch) { + tok = read_number(l) + } else { + tok = token.Illegal{} + } + } + + return tok +} + +skip_whitespace :: proc(l: ^Lexer) { + for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' { + read_char(l) + } +} + +read_identifier :: proc(l: ^Lexer) -> token.Token { + position := l.position + for is_letter(l.ch) { + read_char(l) + } + ident := l.input[position:l.position] + return token.lookup_identifier(ident) +} + +read_number :: proc(l: ^Lexer) -> token.Token { + position := l.position + for is_digit(l.ch) { + read_char(l) + } + integer_raw := l.input[position:l.position] + integer, ok := strconv.parse_int(integer_raw) + if !ok { + return token.Illegal{} + } + return token.Integer{integer} +} + +is_letter :: proc(ch: byte) -> bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' +} + +is_digit :: proc(ch: byte) -> bool { + return '0' <= ch && ch <= '9' +} + +@(test) +test_next_token :: proc(t: ^testing.T) { + input := `let five = 5; +let ten = 10; +let add = fn(x, y) { + x + y; +}; + +let result = add(five, ten); +!-/*5; +5 < 10 > 5; + +if (5 < 10) { + return true; +} else { + return false; +} + +10 == 10; +10 != 9; +` + + tests := []token.Token { + token.Let{}, + token.Identifier{"five"}, + token.Equal_Sign{}, + token.Integer{5}, + token.Semicolon{}, + token.Let{}, + token.Identifier{"ten"}, + token.Equal_Sign{}, + token.Integer{10}, + token.Semicolon{}, + token.Let{}, + token.Identifier{"add"}, + token.Equal_Sign{}, + token.Function{}, + token.Left_Parenthesis{}, + token.Identifier{"x"}, + token.Comma{}, + token.Identifier{"y"}, + token.Right_Parenthesis{}, + token.Left_Brace{}, + token.Identifier{"x"}, + token.Plus{}, + token.Identifier{"y"}, + token.Semicolon{}, + token.Right_Brace{}, + token.Semicolon{}, + token.Let{}, + token.Identifier{"result"}, + token.Equal_Sign{}, + token.Identifier{"add"}, + token.Left_Parenthesis{}, + token.Identifier{"five"}, + token.Comma{}, + token.Identifier{"ten"}, + token.Right_Parenthesis{}, + token.Semicolon{}, + token.Bang{}, + token.Minus{}, + token.Forward_Slash{}, + token.Asterisk{}, + token.Integer{5}, + token.Semicolon{}, + token.Integer{5}, + token.Less_Than{}, + token.Integer{10}, + token.Greater_Than{}, + token.Integer{5}, + token.Semicolon{}, + token.If{}, + token.Left_Parenthesis{}, + token.Integer{5}, + token.Less_Than{}, + token.Integer{10}, + token.Right_Parenthesis{}, + token.Left_Brace{}, + token.Return{}, + token.True{}, + token.Semicolon{}, + token.Right_Brace{}, + token.Else{}, + token.Left_Brace{}, + token.Return{}, + token.False{}, + token.Semicolon{}, + token.Right_Brace{}, + token.Integer{10}, + token.Equals{}, + token.Integer{10}, + token.Semicolon{}, + token.Integer{10}, + token.Not_Equals{}, + token.Integer{9}, + token.Semicolon{}, + token.EOF{}, + } + + l := new(input) + for tt in tests { + char := l.ch + tok := next_token(&l) + testing.expect_value(t, tok, tt) + } +} -- cgit v1.2.3