diff options
1 files changed, 286 insertions, 111 deletions
| -rw-r--r-- | JSImpl/src/Lexer.cpp | 397 |
diff --git a/JSImpl/src/Lexer.cpp b/JSImpl/src/Lexer.cpp index 22ffab8..2e69087 100644 --- a/JSImpl/src/Lexer.cpp +++ b/JSImpl/src/Lexer.cpp @@ -1,42 +1,128 @@ #include "Lexer.h" +#include <utility> + +namespace +{ + inline bool IsUpperCase(char c) + { + return c >= 'A' && c <= 'Z'; + } + + inline bool IsLowerCase(char c) + { + return c >= 'a' && c <= 'z'; + } + + inline bool IsDigit(char c) + { + return c >= '0' && c <= '9'; + } + + inline bool IsValidIdentifierStartingChar(char c) + { + return IsUpperCase(c) || IsLowerCase(c) || c == '_'; + } + + inline bool IsValidIdentifierChar(char c) + { + return IsValidIdentifierStartingChar(c) || IsDigit(c); + } + + inline bool IsStringBound(char c) + { + return c == '\'' || c == '"'; + } + + inline bool IsWhiteSpace(char c) + { + return c == ' ' || c == '\t'; + } + + inline bool IsNewLine(char c) + { + return c == '\n'; + } + + inline bool IsEnd(char c) + { + return c == '\0'; + } +} + +struct Keyword +{ + TokenType Type; + IPLString Content; +}; + +using Identifier = Keyword; + +enum class State: unsigned char +{ + Success, + Fail, + Error +}; + +#define RETURN_SUCCESS(ret) m_GenerationState = State::Success; return ret; +#define RETURN_FAIL(ret) m_GenerationState = State::Fail; return ret; +#define RETURN_ERROR(ret) m_GenerationState = State::Error; return ret; class Tokenizer { public: - Tokenizer(const char* code, const std::function<void()>& onError); - IPLVector<Token> Tokenize(); + Tokenizer(const char* code); + LexerResult Tokenize(IPLVector<Token>& tokens); private: Token NextToken(); - Token ProduceToken(TokenType type); - Token ProduceToken(TokenType type, IPLString lexeme); + inline Token ProduceToken(TokenType type); + inline Token ProduceToken(TokenType type, IPLString lexeme, double number); + inline Token ProduceErrorToken(); + double ParseNumber(); IPLString ParseString(); + Keyword ParseKeyword(); + Identifier ParseIdentifier(); + + inline bool IsStateSuccess() const; + inline bool IsStateError() const; + + void SetError(const IPLString& what); + bool SkipWhiteSpaces(); bool SkipNewLine(); bool Match(const char c); - std::function<void()> OnError; + inline void NextSymbol(); + inline void NextLine(); + unsigned m_Line; + unsigned m_Column; unsigned m_Current; const char* m_Code; + IPLError m_Error; + State m_GenerationState; + IPLUnorderedMap<IPLString, TokenType> m_KeyWordsTable; }; -IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError) +LexerResult Tokenize(const char* code, IPLVector<Token>& tokens) { - Tokenizer tokenizer(code, onError); + Tokenizer tokenizer(code); - return tokenizer.Tokenize(); + return tokenizer.Tokenize(tokens); } -Tokenizer::Tokenizer(const char* code, const std::function<void()>& onError) - : OnError(onError) - , m_Line(0) +Tokenizer::Tokenizer(const char* code) + : m_Line(0) + , m_Column(0) , m_Current(0) , m_Code(code) + , m_Error() + , m_GenerationState(State::Success) { m_KeyWordsTable["break"] = TokenType::Break; m_KeyWordsTable["case"] = TokenType::Case; @@ -77,23 +163,30 @@ Tokenizer::Tokenizer(const char* code, const std::function<void()>& onError) m_KeyWordsTable["false"] = TokenType::False; } -IPLVector<Token> Tokenizer::Tokenize() +LexerResult Tokenizer::Tokenize(IPLVector<Token>& tokens) { - IPLVector<Token> result; do { - result.push_back(NextToken()); - } while (result.back().Type != TokenType::Eof); + const auto& token = NextToken(); + + if (m_GenerationState == State::Error) + { + return LexerResult{ false, IPLError(m_Error) }; + } + + tokens.emplace_back(token); - return result; + } while (tokens.back().Type != TokenType::Eof); + + return LexerResult{ true, IPLError() }; } bool Tokenizer::SkipWhiteSpaces() { const auto old = m_Current; - while (m_Code[m_Current] == ' ' || m_Code[m_Current] == '\t') + while (IsWhiteSpace(m_Code[m_Current])) { - m_Current++; + NextSymbol(); } return old != m_Current; } @@ -101,10 +194,9 @@ bool Tokenizer::SkipWhiteSpaces() bool Tokenizer::SkipNewLine() { const auto old = m_Current; - while (m_Code[m_Current] == '\n') + while (IsNewLine(m_Code[m_Current])) { - m_Current++; - m_Line++; + NextLine(); } return old != m_Current; } @@ -113,40 +205,160 @@ bool Tokenizer::Match(const char c) { if (c == m_Code[m_Current]) { - ++m_Current; + if (!IsNewLine(m_Code[m_Current])) + { + NextSymbol(); + } + else + { + NextLine(); + } + return true; } return false; } -Token Tokenizer::ProduceToken(TokenType type) +inline void Tokenizer::NextSymbol() { - return { type, m_Line, "", 0.0 }; + ++m_Current; + ++m_Column; } -Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme) +inline void Tokenizer::NextLine() { - return{ type, m_Line, lexeme, 0.0 }; + ++m_Current; + ++m_Line; + m_Column = 0; } -namespace +inline Token Tokenizer::ProduceToken(TokenType type) +{ + return Token{ type, m_Line, "", 0.0 }; +} + +inline Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme, double number = 0.0) +{ + return Token{ type, m_Line, lexeme, number }; +} + +inline Token Tokenizer::ProduceErrorToken() +{ + return Token{ TokenType::Eof, m_Line, "", 0.0 }; +} + +double Tokenizer::ParseNumber() +{ + if (!IsDigit(m_Code[m_Current])) + { + RETURN_FAIL(0.0); + } + + size_t parsedBytes = 0; + double number = std::stod(m_Code + m_Current, &parsedBytes); + m_Current += static_cast<unsigned>(parsedBytes); + m_Column += static_cast<unsigned>(parsedBytes); + + RETURN_SUCCESS(number); +} + +IPLString Tokenizer::ParseString() +{ + if (!IsStringBound(m_Code[m_Current])) + { + RETURN_FAIL(IPLString()); + } + + char bound = m_Code[m_Current]; + auto start = m_Current; + + // skip first " or ' + NextSymbol(); + while (m_Code[m_Current] != bound && !IsEnd(m_Code[m_Current]) && !IsNewLine(m_Code[m_Current])) + { + NextSymbol(); + } + + if (IsEnd(m_Code[m_Current]) || IsNewLine(m_Code[m_Current])) + { + SetError("\"\" string literal contains an unescaped line break"); + RETURN_ERROR(IPLString()); + } + + // skip second " or ' + NextSymbol(); + + RETURN_SUCCESS(IPLString(m_Code + start, m_Code + m_Current)); +} + +Keyword Tokenizer::ParseKeyword() +{ + if (!IsLowerCase(m_Code[m_Current])) + { + RETURN_FAIL(Keyword()); + } + + auto start = m_Current; + + NextSymbol(); + while (IsLowerCase(m_Code[m_Current])) + { + NextSymbol(); + } + + auto key = IPLString(m_Code + start, m_Code + m_Current); + auto keyword = m_KeyWordsTable.find(key); + + if (keyword == m_KeyWordsTable.end()) + { + // It's not a keyword so we must revert current counter + m_Current = start; + m_Column = start; + + RETURN_FAIL(Keyword()); + } + + RETURN_SUCCESS((Keyword{ keyword->second, keyword->first })); +} + +Identifier Tokenizer::ParseIdentifier() { -inline bool IsValidIdentifierStartingChar(char c) + if (!IsValidIdentifierStartingChar(m_Code[m_Current])) + { + RETURN_FAIL(Identifier()); + } + + auto start = m_Current; + + NextSymbol(); + while (IsValidIdentifierChar(m_Code[m_Current])) + { + NextSymbol(); + } + + RETURN_SUCCESS((Identifier{ TokenType::Identifier, IPLString(m_Code + start, m_Code + m_Current) })); +} + +inline bool Tokenizer::IsStateSuccess() const { - return (c >= 'A' && c <= 'z') || c == '_'; + return m_GenerationState == State::Success; } -inline bool IsValidIdentifierChar(char c) +inline bool Tokenizer::IsStateError() const { - return IsValidIdentifierStartingChar(c) || (c >= '0' && c <= '9'); + return m_GenerationState == State::Error; } + +void Tokenizer::SetError(const IPLString & what) +{ + m_Error = IPLError{ m_Line, m_Column, "", what }; } Token Tokenizer::NextToken() { - if (m_Code[m_Current] == '\0') + if (IsEnd(m_Code[m_Current])) { - return Token{ TokenType::Eof, 0, "", 0.0 }; + ProduceToken(TokenType::Eof); } while (SkipWhiteSpaces() || SkipNewLine()); @@ -154,98 +366,61 @@ Token Tokenizer::NextToken() // Single Char switch (m_Code[m_Current]) { - case '(': m_Current++; return ProduceToken(TokenType::LeftParen); - case ')': m_Current++; return ProduceToken(TokenType::RightParen); - case '{': m_Current++; return ProduceToken(TokenType::LeftBrace); - case '}': m_Current++; return ProduceToken(TokenType::RightBrace); - case ',': m_Current++; return ProduceToken(TokenType::Comma); - case '.': m_Current++; return ProduceToken(TokenType::Dot); - case '-': m_Current++; return Match('-') ? ProduceToken(TokenType::MinusMinus) : ProduceToken(TokenType::Minus); - case '+': m_Current++; return Match('+') ? ProduceToken(TokenType::PlusPlus) : ProduceToken(TokenType::Plus); - case ';': m_Current++; return ProduceToken(TokenType::Semicolon); - case '*': m_Current++; return ProduceToken(TokenType::Star); - case '/': m_Current++; return ProduceToken(TokenType::Division); - case '%': m_Current++; return ProduceToken(TokenType::Modulo); - case '~': m_Current++; return ProduceToken(TokenType::BitwiseNot); - case '=': m_Current++; return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictEqual) : ProduceToken(TokenType::EqualEqual) : ProduceToken(TokenType::Equal); - case '!': m_Current++; return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictNotEqual) : ProduceToken(TokenType::BangEqual) : ProduceToken(TokenType::Bang); - case '>': m_Current++; return Match('=') ? ProduceToken(TokenType::GreaterEqual) : Match('>') ? ProduceToken(TokenType::RightShift) : ProduceToken(TokenType::Greater); - case '<': m_Current++; return Match('=') ? ProduceToken(TokenType::LessEqual) : Match('<') ? ProduceToken(TokenType::LeftShift) : ProduceToken(TokenType::Less); - case '&': m_Current++; return Match('&') ? ProduceToken(TokenType::LogicalAnd) : ProduceToken(TokenType::BitwiseAnd); - case '^': m_Current++; return ProduceToken(TokenType::BitwiseXor); - case '|': m_Current++; return Match('|') ? ProduceToken(TokenType::LogicalOr) : ProduceToken(TokenType::BitwiseOr); - case '?': m_Current++; return ProduceToken(TokenType::QuestionMark); - case ':': m_Current++; return ProduceToken(TokenType::Colon); - case '[': m_Current++; return ProduceToken(TokenType::LeftSquareBracket); - case ']': m_Current++; return ProduceToken(TokenType::RightSquareBracket); + case '(': NextSymbol(); return ProduceToken(TokenType::LeftParen); + case ')': NextSymbol(); return ProduceToken(TokenType::RightParen); + case '{': NextSymbol(); return ProduceToken(TokenType::LeftBrace); + case '}': NextSymbol(); return ProduceToken(TokenType::RightBrace); + case ',': NextSymbol(); return ProduceToken(TokenType::Comma); + case '.': NextSymbol(); return ProduceToken(TokenType::Dot); + case '-': NextSymbol(); return Match('-') ? ProduceToken(TokenType::MinusMinus) : ProduceToken(TokenType::Minus); + case '+': NextSymbol(); return Match('+') ? ProduceToken(TokenType::PlusPlus) : ProduceToken(TokenType::Plus); + case ';': NextSymbol(); return ProduceToken(TokenType::Semicolon); + case '*': NextSymbol(); return ProduceToken(TokenType::Star); + case '/': NextSymbol(); return ProduceToken(TokenType::Division); + case '%': NextSymbol(); return ProduceToken(TokenType::Modulo); + case '~': NextSymbol(); return ProduceToken(TokenType::BitwiseNot); + case '=': NextSymbol(); return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictEqual) : ProduceToken(TokenType::EqualEqual) : ProduceToken(TokenType::Equal); + case '!': NextSymbol(); return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictNotEqual) : ProduceToken(TokenType::BangEqual) : ProduceToken(TokenType::Bang); + case '>': NextSymbol(); return Match('=') ? ProduceToken(TokenType::GreaterEqual) : Match('>') ? ProduceToken(TokenType::RightShift) : ProduceToken(TokenType::Greater); + case '<': NextSymbol(); return Match('=') ? ProduceToken(TokenType::LessEqual) : Match('<') ? ProduceToken(TokenType::LeftShift) : ProduceToken(TokenType::Less); + case '&': NextSymbol(); return Match('&') ? ProduceToken(TokenType::LogicalAnd) : ProduceToken(TokenType::BitwiseAnd); + case '^': NextSymbol(); return ProduceToken(TokenType::BitwiseXor); + case '|': NextSymbol(); return Match('|') ? ProduceToken(TokenType::LogicalOr) : ProduceToken(TokenType::BitwiseOr); + case '?': NextSymbol(); return ProduceToken(TokenType::QuestionMark); + case ':': NextSymbol(); return ProduceToken(TokenType::Colon); + case '[': NextSymbol(); return ProduceToken(TokenType::LeftSquareBracket); + case ']': NextSymbol(); return ProduceToken(TokenType::RightSquareBracket); default: break; } - // Number - const char c = m_Code[m_Current]; - if (c >= '0' && c <= '9') + + const auto& number = ParseNumber(); + if (IsStateSuccess()) { - size_t parsedBytes = 0; - const double number = std::stod(m_Code + m_Current, &parsedBytes); - m_Current += (unsigned)parsedBytes; - return Token{ TokenType::Number, m_Line, "", number }; + return ProduceToken(TokenType::Number, "", number); } - // String - if (c == '"') + const auto& string = ParseString(); + if (IsStateSuccess()) { - auto start = m_Current; - m_Current++; // skip first '"' - while (m_Code[m_Current] != '"') - { - ++m_Current; - } - m_Current++; // skip second '"' - return Token{ TokenType::String, m_Line, IPLString(m_Code + start, m_Code + m_Current), 0.0 }; + return ProduceToken(TokenType::String, string); } - - if (c == '\'') + else if (IsStateError()) { - auto start = m_Current; - m_Current++; // skip first ' - while (m_Code[m_Current] != '\'') - { - ++m_Current; - } - m_Current++; // skip second ' - return Token{ TokenType::String, m_Line, IPLString(m_Code + start, m_Code + m_Current), 0.0 }; + return ProduceErrorToken(); } - // Key words - if (c >= 'a' && c <= 'z') + const auto& keyword = ParseKeyword(); + if (IsStateSuccess()) { - auto start = m_Current; - m_Current++; - while (m_Code[m_Current] >= 'a' && m_Code[m_Current] <= 'z') - { - ++m_Current; - } - auto key = IPLString(m_Code + start, m_Code + m_Current); - auto word = m_KeyWordsTable.find(key); - if (word != m_KeyWordsTable.end()) - { - return Token{ word->second, m_Line, key, 0.0 }; - } - // It's not a key word so we must revert current counter - m_Current = start; + return ProduceToken(keyword.Type, keyword.Content); } - // Identifier - if (IsValidIdentifierStartingChar(c)) + const auto& identifier = ParseIdentifier(); + if (IsStateSuccess()) { - auto start = m_Current; - m_Current++; - while (IsValidIdentifierChar(m_Code[m_Current])) - { - ++m_Current; - } - return Token{ TokenType::Identifier, m_Line, IPLString(m_Code + start, m_Code + m_Current), 0.0 }; + return ProduceToken(identifier.Type, identifier.Content); } - // TODO: Error reporting - return Token{ TokenType::Eof, m_Line, "", 0.0 }; + + return ProduceErrorToken(); } |