From b91b7d3820deb7e0aed536141cd79e7a8c4ca8c9 Mon Sep 17 00:00:00 2001 From: martin dobrev Date: Sat, 8 Sep 2018 12:23:28 +0300 Subject: Add more tokens to Lexer --- JSImpl/src/Lexer.cpp | 15 +++++++++++---- JSImpl/src/Lexer.h | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) (limited to 'JSImpl/src') diff --git a/JSImpl/src/Lexer.cpp b/JSImpl/src/Lexer.cpp index 3e78275..49c7567 100644 --- a/JSImpl/src/Lexer.cpp +++ b/JSImpl/src/Lexer.cpp @@ -71,6 +71,10 @@ Tokenizer::Tokenizer(const char* code, const std::function& onError) m_KeyWordsTable["while"] = TokenType::While; m_KeyWordsTable["with"] = TokenType::With; m_KeyWordsTable["yield"] = TokenType::Yield; + m_KeyWordsTable["null"] = TokenType::Null; + m_KeyWordsTable["undefined"] = TokenType::Undefined; + m_KeyWordsTable["true"] = TokenType::True; + m_KeyWordsTable["false"] = TokenType::False; } IPLVector Tokenizer::Tokenize() @@ -167,10 +171,13 @@ Token Tokenizer::NextToken() case '!': m_Current++; return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictNotEqual) : ProduceToken(TokenType::BangEqual) : ProduceToken(TokenType::Bang); case '>': m_Current++; return Match('=') ? ProduceToken(TokenType::GreaterEqual) : Match('>') ? ProduceToken(TokenType::RightShift) : ProduceToken(TokenType::Greater); case '<': m_Current++; return Match('=') ? ProduceToken(TokenType::LessEqual) : Match('<') ? ProduceToken(TokenType::LeftShift) : ProduceToken(TokenType::Less); - case '&': m_Current++; return Match('&') ? ProduceToken(LogicalAnd) : ProduceToken(BitwiseAnd); - case '^': m_Current++; return ProduceToken(BitwiseXor); - case '|': m_Current++; return Match('|') ? ProduceToken(LogicalOr) : ProduceToken(BitwiseOr); - + case '&': m_Current++; return Match('&') ? ProduceToken(TokenType::LogicalAnd) : ProduceToken(TokenType::BitwiseAnd); + case '^': m_Current++; return ProduceToken(TokenType::BitwiseXor); + case '|': m_Current++; return Match('|') ? ProduceToken(TokenType::LogicalOr) : ProduceToken(TokenType::BitwiseOr); + case '?': m_Current++; return ProduceToken(TokenType::QuestionMark); + case ':': m_Current++; return ProduceToken(TokenType::Colon); + case '[': m_Current++; return ProduceToken(TokenType::LeftSquareBracket); + case ']': m_Current++; return ProduceToken(TokenType::RightSquareBracket); default: break; } diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h index 21c06cc..fd3d244 100644 --- a/JSImpl/src/Lexer.h +++ b/JSImpl/src/Lexer.h @@ -22,6 +22,10 @@ enum TokenType BitwiseAnd, BitwiseXor, BitwiseOr, + QuestionMark, + Colon, + LeftSquareBracket, + RightSquareBracket, // One or two character tokens Bang, @@ -40,6 +44,16 @@ enum TokenType RightShift, LogicalAnd, LogicalOr, + StarEqual, + DivideEqual, + ModuloEqual, + PlusEqual, + MinusEqual, + LeftShiftEqual, + RightShiftEqual, + BitwiseAndEqual, + BitwiseXorEqual, + BitwiseOrEqual, Identifier, String, @@ -79,6 +93,11 @@ enum TokenType While, With, Yield, + Null, + Undefined, + True, + False, + Eof, }; -- cgit v1.3.1-23-gd82d