chapter 01

1 files changed, 257 insertions, 0 deletions

diff --git a/lexer/lexer.odin b/lexer/lexer.odin
new file mode 100644
index 0000000..d4ad9b3
--- /dev/null
+++ b/lexer/lexer.odin
@@ -0,0 +1,257 @@
+package lexer
+
+import token "../token"
+import "core:fmt"
+import "core:log"
+import "core:reflect"
+import "core:strconv"
+import "core:testing"
+
+Lexer :: struct {
+	input:         string,
+	position:      int,
+	read_position: int,
+	ch:            byte,
+}
+
+new :: proc(input: string) -> Lexer {
+	l := Lexer{input, 0, 0, 0}
+	read_char(&l)
+	return l
+}
+
+read_char :: proc(l: ^Lexer) {
+	if l.read_position >= len(l.input) {
+		l.ch = 0
+	} else {
+		l.ch = l.input[l.read_position]
+	}
+	l.position = l.read_position
+	l.read_position += 1
+}
+
+peek_char :: proc(l: ^Lexer) -> byte {
+	if l.read_position >= len(l.input) {
+		return 0
+	} else {
+		return l.input[l.read_position]
+	}
+
+}
+
+next_token :: proc(l: ^Lexer) -> token.Token {
+	tok: token.Token
+
+	skip_whitespace(l)
+
+	switch l.ch {
+	case '=':
+		if peek_char(l) == '=' {
+			tok = token.Equals{}
+			read_char(l)
+		} else {
+			tok = token.Equal_Sign{}
+		}
+		read_char(l)
+	case '+':
+		tok = token.Plus{}
+		read_char(l)
+	case '-':
+		tok = token.Minus{}
+		read_char(l)
+	case '!':
+		if peek_char(l) == '=' {
+			tok = token.Not_Equals{}
+			read_char(l)
+		} else {
+			tok = token.Bang{}
+		}
+		read_char(l)
+	case '/':
+		tok = token.Forward_Slash{}
+		read_char(l)
+	case '*':
+		tok = token.Asterisk{}
+		read_char(l)
+	case '<':
+		tok = token.Less_Than{}
+		read_char(l)
+	case '>':
+		tok = token.Greater_Than{}
+		read_char(l)
+	case ';':
+		tok = token.Semicolon{}
+		read_char(l)
+	case ',':
+		tok = token.Comma{}
+		read_char(l)
+	case '(':
+		tok = token.Left_Parenthesis{}
+		read_char(l)
+	case ')':
+		tok = token.Right_Parenthesis{}
+		read_char(l)
+	case '{':
+		tok = token.Left_Brace{}
+		read_char(l)
+	case '}':
+		tok = token.Right_Brace{}
+		read_char(l)
+	case 0:
+		tok = token.EOF{}
+		read_char(l)
+	case:
+		if is_letter(l.ch) {
+			tok = read_identifier(l)
+		} else if is_digit(l.ch) {
+			tok = read_number(l)
+		} else {
+			tok = token.Illegal{}
+		}
+	}
+
+	return tok
+}
+
+skip_whitespace :: proc(l: ^Lexer) {
+	for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+		read_char(l)
+	}
+}
+
+read_identifier :: proc(l: ^Lexer) -> token.Token {
+	position := l.position
+	for is_letter(l.ch) {
+		read_char(l)
+	}
+	ident := l.input[position:l.position]
+	return token.lookup_identifier(ident)
+}
+
+read_number :: proc(l: ^Lexer) -> token.Token {
+	position := l.position
+	for is_digit(l.ch) {
+		read_char(l)
+	}
+	integer_raw := l.input[position:l.position]
+	integer, ok := strconv.parse_int(integer_raw)
+	if !ok {
+		return token.Illegal{}
+	}
+	return token.Integer{integer}
+}
+
+is_letter :: proc(ch: byte) -> bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
+}
+
+is_digit :: proc(ch: byte) -> bool {
+	return '0' <= ch && ch <= '9'
+}
+
+@(test)
+test_next_token :: proc(t: ^testing.T) {
+	input := `let five = 5;
+let ten = 10;
+let add = fn(x, y) {
+  x + y;
+};
+
+let result = add(five, ten);
+!-/*5;
+5 < 10 > 5;
+
+if (5 < 10) {
+        return true;
+} else {
+        return false;
+}
+
+10 == 10;
+10 != 9;
+`
+
+	tests := []token.Token {
+		token.Let{},
+		token.Identifier{"five"},
+		token.Equal_Sign{},
+		token.Integer{5},
+		token.Semicolon{},
+		token.Let{},
+		token.Identifier{"ten"},
+		token.Equal_Sign{},
+		token.Integer{10},
+		token.Semicolon{},
+		token.Let{},
+		token.Identifier{"add"},
+		token.Equal_Sign{},
+		token.Function{},
+		token.Left_Parenthesis{},
+		token.Identifier{"x"},
+		token.Comma{},
+		token.Identifier{"y"},
+		token.Right_Parenthesis{},
+		token.Left_Brace{},
+		token.Identifier{"x"},
+		token.Plus{},
+		token.Identifier{"y"},
+		token.Semicolon{},
+		token.Right_Brace{},
+		token.Semicolon{},
+		token.Let{},
+		token.Identifier{"result"},
+		token.Equal_Sign{},
+		token.Identifier{"add"},
+		token.Left_Parenthesis{},
+		token.Identifier{"five"},
+		token.Comma{},
+		token.Identifier{"ten"},
+		token.Right_Parenthesis{},
+		token.Semicolon{},
+		token.Bang{},
+		token.Minus{},
+		token.Forward_Slash{},
+		token.Asterisk{},
+		token.Integer{5},
+		token.Semicolon{},
+		token.Integer{5},
+		token.Less_Than{},
+		token.Integer{10},
+		token.Greater_Than{},
+		token.Integer{5},
+		token.Semicolon{},
+		token.If{},
+		token.Left_Parenthesis{},
+		token.Integer{5},
+		token.Less_Than{},
+		token.Integer{10},
+		token.Right_Parenthesis{},
+		token.Left_Brace{},
+		token.Return{},
+		token.True{},
+		token.Semicolon{},
+		token.Right_Brace{},
+		token.Else{},
+		token.Left_Brace{},
+		token.Return{},
+		token.False{},
+		token.Semicolon{},
+		token.Right_Brace{},
+		token.Integer{10},
+		token.Equals{},
+		token.Integer{10},
+		token.Semicolon{},
+		token.Integer{10},
+		token.Not_Equals{},
+		token.Integer{9},
+		token.Semicolon{},
+		token.EOF{},
+	}
+
+	l := new(input)
+	for tt in tests {
+		char := l.ch
+		tok := next_token(&l)
+		testing.expect_value(t, tok, tt)
+	}
+}