Lexer whitespace and commens tokens. (#13)

* Lexer can create whitespace tokens

* Parse whitespaces, comments and error fixes

* Brace error fix

* Less explicit default settings tokenizer.

3 files changed, 340 insertions, 30 deletions

diff --git a/JSImpl/src/Lexer.cpp b/JSImpl/src/Lexer.cpp
index 0602f1a..5d85570 100644
--- a/JSImpl/src/Lexer.cpp
+++ b/JSImpl/src/Lexer.cpp
@@ -57,7 +57,7 @@ struct Keyword
 
 using Identifier = Keyword;
 
-enum class State: unsigned char
+enum class State : unsigned char
 {
 	Success,
 	Fail,
@@ -71,16 +71,19 @@ enum class State: unsigned char
 class Tokenizer
 {
 public:
-	Tokenizer(const char* code);
+	Tokenizer(const char* code, const LexerSettings& settings);
 	LexerResult Tokenize(IPLVector<Token>& tokens);
 
 private:
 	Token NextToken();
 
+	inline Token ProduceSkipToken();
 	inline Token ProduceToken(TokenType type);
 	inline Token ProduceToken(TokenType type, IPLString lexeme, double number);
 	inline Token ProduceErrorToken();
 
+	IPLString ParseComment();
+	IPLString ParseWhitespaces();
 	double ParseNumber();
 	IPLString ParseString();
 	Keyword ParseKeyword();
@@ -96,6 +99,7 @@ private:
 	bool Match(const char c);
 
 	inline void NextSymbol();
+	inline void PreviousSymbol();
 	inline void NextLine();
 
 	unsigned m_Line;
@@ -106,23 +110,33 @@ private:
 	IPLError m_Error;
 	State m_GenerationState;
 
+	LexerSettings m_Settings;
+	bool m_SkipNextToken;
+
 	IPLUnorderedMap<IPLString, TokenType> m_KeyWordsTable;
 };
 
-LexerResult Tokenize(const char* code, IPLVector<Token>& tokens)
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings)
 {
-	Tokenizer tokenizer(code);
+	Tokenizer tokenizer(code, settings);
 
 	return tokenizer.Tokenize(tokens);
 }
 
-Tokenizer::Tokenizer(const char* code)
+LexerResult Tokenize(const char * code, IPLVector<Token>& tokens)
+{
+	return Tokenize(code, tokens, { false, false });
+}
+
+Tokenizer::Tokenizer(const char* code, const LexerSettings& settings)
 	: m_Line(0)
 	, m_Column(0)
 	, m_Current(0)
 	, m_Code(code)
 	, m_Error()
 	, m_GenerationState(State::Success)
+	, m_Settings(settings)
+	, m_SkipNextToken(false)
 {
 	m_KeyWordsTable["break"] = TokenType::Break;
 	m_KeyWordsTable["case"] = TokenType::Case;
@@ -153,6 +167,7 @@ Tokenizer::Tokenizer(const char* code)
 	m_KeyWordsTable["try"] = TokenType::Try;
 	m_KeyWordsTable["typeof"] = TokenType::Typeof;
 	m_KeyWordsTable["var"] = TokenType::Var;
+	m_KeyWordsTable["let"] = TokenType::Let;
 	m_KeyWordsTable["void"] = TokenType::Void;
 	m_KeyWordsTable["while"] = TokenType::While;
 	m_KeyWordsTable["with"] = TokenType::With;
@@ -167,7 +182,11 @@ LexerResult Tokenizer::Tokenize(IPLVector<Token>& tokens)
 {
 	do
 	{
-		const auto& token = NextToken();
+		Token token;
+		do
+		{
+			token = NextToken();
+		} while (m_SkipNextToken);
 
 		if (m_GenerationState == State::Error)
 		{
@@ -175,7 +194,6 @@ LexerResult Tokenizer::Tokenize(IPLVector<Token>& tokens)
 		}
 
 		tokens.emplace_back(token);
-
 	} while (tokens.back().Type != TokenType::Eof);
 
 	return LexerResult{ true, IPLError() };
@@ -225,6 +243,12 @@ inline void Tokenizer::NextSymbol()
 	++m_Column;
 }
 
+inline void Tokenizer::PreviousSymbol()
+{
+	--m_Current;
+	--m_Column;
+}
+
 inline void Tokenizer::NextLine()
 {
 	++m_Current;
@@ -232,19 +256,164 @@ inline void Tokenizer::NextLine()
 	m_Column = 0;
 }
 
-inline Token Tokenizer::ProduceToken(TokenType type)
+inline Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme, double number = 0.0)
 {
-	return Token{ type, m_Line, "", 0.0 };
+	return Token{ type, m_Line, lexeme, number };
 }
 
-inline Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme, double number = 0.0)
+inline Token Tokenizer::ProduceSkipToken()
 {
-	return Token{ type, m_Line, lexeme, number };
+	return ProduceToken(TokenType::Invalid, "");
+}
+
+inline Token Tokenizer::ProduceToken(TokenType type)
+{
+	switch (type)
+	{
+	case TokenType::RightSquareBracket:
+		return ProduceToken(type, "]");
+	case TokenType::LeftSquareBracket:
+		return ProduceToken(type, "[");
+	case TokenType::Colon:
+		return ProduceToken(type, ":");
+	case TokenType::QuestionMark:
+		return ProduceToken(type, "?");
+	case TokenType::BitwiseXor:
+		return ProduceToken(type, "^");
+	case TokenType::LogicalOr:
+		return ProduceToken(type, "||");
+	case TokenType::BitwiseOr:
+		return ProduceToken(type, "|");
+	case TokenType::LogicalAnd:
+		return ProduceToken(type, "&&");
+	case TokenType::BitwiseAnd:
+		return ProduceToken(type, "&");
+	case TokenType::LessEqual:
+		return ProduceToken(type, "<=");
+	case TokenType::LeftShift:
+		return ProduceToken(type, "<<");
+	case TokenType::Less:
+		return ProduceToken(type, "<");
+	case TokenType::GreaterEqual:
+		return ProduceToken(type, ">=");
+	case TokenType::RightShift:
+		return ProduceToken(type, ">>");
+	case TokenType::Greater:
+		return ProduceToken(type, ">");
+	case TokenType::StrictNotEqual:
+		return ProduceToken(type, "!==");
+	case TokenType::BangEqual:
+		return ProduceToken(type, "!=");
+	case TokenType::Bang:
+		return ProduceToken(type, "!");
+	case TokenType::StrictEqual:
+		return ProduceToken(type, "===");
+	case TokenType::EqualEqual:
+		return ProduceToken(type, "==");
+	case TokenType::Equal:
+		return ProduceToken(type, "=");
+	case TokenType::BitwiseNot:
+		return ProduceToken(type, "~");
+	case TokenType::Modulo:
+		return ProduceToken(type, "%");
+	case TokenType::Division:
+		return ProduceToken(type, "/");
+	case TokenType::LeftParen:
+		return ProduceToken(type, "(");
+	case TokenType::RightParen:
+		return ProduceToken(type, ")");
+	case TokenType::LeftBrace:
+		return ProduceToken(type, "{");
+	case TokenType::RightBrace:
+		return ProduceToken(type, "}");
+	case TokenType::Comma:
+		return ProduceToken(type, ",");
+	case TokenType::Dot:
+		return ProduceToken(type, ".");
+	case TokenType::Minus:
+		return ProduceToken(type, "-");
+	case TokenType::MinusMinus:
+		return ProduceToken(type, "--");
+	case TokenType::Plus:
+		return ProduceToken(type, "+");
+	case TokenType::PlusPlus:
+		return ProduceToken(type, "++");
+	case TokenType::Semicolon:
+		return ProduceToken(type, ";");
+	case TokenType::Star:
+		return ProduceToken(type, "*");
+	default: // EoF Invalid
+		break;
+	}
+	return ProduceToken(type, "");
 }
 
 inline Token Tokenizer::ProduceErrorToken()
 {
-	return Token{ TokenType::Eof, m_Line, "", 0.0 };
+	return ProduceToken(TokenType::Invalid, "");
+}
+
+IPLString Tokenizer::ParseComment()
+{
+	auto start = m_Current;
+
+	if (!Match('/'))
+	{
+		RETURN_FAIL(IPLString());
+	}
+
+	if (Match('/'))
+	{
+		while (!IsEnd(m_Code[m_Current]) && !IsNewLine(m_Code[m_Current]))
+		{
+			NextSymbol();
+		}
+		RETURN_SUCCESS(IPLString(m_Code + start, m_Code + m_Current))
+	}
+	else if (!Match('*'))
+	{
+		PreviousSymbol();
+		RETURN_FAIL(IPLString());
+	}
+
+	while (!IsEnd(m_Code[m_Current]))
+	{
+		if (Match('*') && Match('/'))
+		{
+			RETURN_SUCCESS(IPLString(m_Code + start, m_Code + m_Current));
+		}
+		NextSymbol();
+	}
+
+	SetError("unterminated comment");
+	RETURN_ERROR(IPLString());
+}
+
+IPLString Tokenizer::ParseWhitespaces()
+{
+	if (!IsWhiteSpace(m_Code[m_Current]) && !IsNewLine(m_Code[m_Current]))
+	{
+		RETURN_FAIL(IPLString());
+	}
+
+	auto start = m_Current;
+	while (1)
+	{
+		if (IsWhiteSpace(m_Code[m_Current]))
+		{
+			NextSymbol();
+		}
+		else if (IsNewLine(m_Code[m_Current]))
+		{
+			NextLine();
+		}
+		else
+		{
+			break;
+		}
+	}
+
+	RETURN_SUCCESS(IPLString(m_Code + start, m_Code + m_Current));
 }
 
 double Tokenizer::ParseNumber()
@@ -279,7 +448,7 @@ IPLString Tokenizer::ParseString()
 		NextSymbol();
 	}
 
-	if (IsEnd(m_Code[m_Current])  || IsNewLine(m_Code[m_Current]))
+	if (IsEnd(m_Code[m_Current]) || IsNewLine(m_Code[m_Current]))
 	{
 		SetError("\"\" string literal contains an unescaped line break");
 		RETURN_ERROR(IPLString());
@@ -312,8 +481,8 @@ Keyword Tokenizer::ParseKeyword()
 	if (keyword == m_KeyWordsTable.end())
 	{
 		// It's not a keyword so we must revert current counter
+		m_Column -= m_Current - start;
 		m_Current = start;
-		m_Column = start;
 
 		RETURN_FAIL(Keyword());
 	}
@@ -349,19 +518,45 @@ inline bool Tokenizer::IsStateError() const
 	return m_GenerationState == State::Error;
 }
 
-void Tokenizer::SetError(const IPLString & what)
+void Tokenizer::SetError(const IPLString& what)
 {
-	m_Error = IPLError{ m_Line, m_Column, "", what };
+	m_Error = IPLError{ m_Line, m_Column, "", "Syntax error: " + what };
 }
 
 Token Tokenizer::NextToken()
 {
+	m_SkipNextToken = false;
+
 	if (IsEnd(m_Code[m_Current]))
 	{
-		ProduceToken(TokenType::Eof);
+		return ProduceToken(TokenType::Eof);
+	}
+
+	const auto& comment = ParseComment();
+	if (IsStateError())
+	{
+		return ProduceErrorToken();
+	}
+	if (IsStateSuccess())
+	{
+		if (!m_Settings.CreateCommentTokens)
+		{
+			m_SkipNextToken = true;
+			return ProduceSkipToken();
+		}
+		return ProduceToken(TokenType::Comment, comment);
 	}
 
-	while (SkipWhiteSpaces() || SkipNewLine());
+	const auto& spaces = ParseWhitespaces();
+	if (IsStateSuccess())
+	{
+		if (!m_Settings.CreateWhitespaceTokens)
+		{
+			m_SkipNextToken = true;
+			return ProduceSkipToken();
+		}
+		return ProduceToken(TokenType::Whitespace, spaces);
+	}
 
 	// Single Char
 	switch (m_Code[m_Current])
diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h
index fb3e0a1..e17270d 100644
--- a/JSImpl/src/Lexer.h
+++ b/JSImpl/src/Lexer.h
@@ -88,6 +88,7 @@ enum class TokenType
 	Try,
 	Typeof,
 	Var,
+	Let,
 	Void,
 	While,
 	With,
@@ -97,6 +98,9 @@ enum class TokenType
 	True,
 	False,
 
+	Whitespace,
+	Comment,
+
 	Eof,
 	Invalid
 };
@@ -115,4 +119,11 @@ struct LexerResult
 	IPLError Error;
 };
 
-LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
+struct LexerSettings
+{
+	bool CreateWhitespaceTokens;
+	bool CreateCommentTokens;
+};
+
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings);
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
\ No newline at end of file
diff --git a/JSImpl/src/main.cpp b/JSImpl/src/main.cpp
index cf9bf70..3950569 100644
--- a/JSImpl/src/main.cpp
+++ b/JSImpl/src/main.cpp
@@ -85,9 +85,106 @@ void TestVariableDeclaration()
 void TestStringError()
 {
 	IPLVector<Token> tokens;
-	const auto& res = Tokenize("\"aaaa", tokens);
+	const auto& res = Tokenize("\" aaaa", tokens);
 
-	CHECK(!res.IsSuccessful && res.Error.Row == 0 && res.Error.Column == 5);
+	CHECK(!res.IsSuccessful && res.Error.Row == 0 && res.Error.Column == 6);
+}
+
+void TestWhiteSpaceTokens()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize(" 1\n2  abc\n \n", tokens, { true, false });
+	CHECK(res.IsSuccessful
+		&& tokens.size() == 8
+		&& tokens[0].Type == TokenType::Whitespace
+		&& tokens[0].Lexeme == " "
+		&& tokens[1].Type == TokenType::Number
+		&& tokens[1].Number == 1
+		&& tokens[2].Type == TokenType::Whitespace
+		&& tokens[2].Lexeme == "\n"
+		&& tokens[3].Type == TokenType::Number
+		&& tokens[3].Number == 2
+		&& tokens[4].Type == TokenType::Whitespace
+		&& tokens[4].Lexeme == "  "
+		&& tokens[5].Type == TokenType::Identifier
+		&& tokens[5].Lexeme == "abc"
+		&& tokens[6].Type == TokenType::Whitespace
+		&& tokens[6].Lexeme == "\n \n"
+		&& tokens[7].Type == TokenType::Eof);
+}
+
+void TestSimpleComment()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize("//comment", tokens, { false, true });
+	CHECK(res.IsSuccessful
+		&& tokens.size() == 2
+		&& tokens[0].Type == TokenType::Comment
+		&& tokens[0].Lexeme == "//comment"
+		&& tokens[1].Type == TokenType::Eof);
+}
+
+void TestSimpleMultiLineComment()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize("/*comment\ncomment*/", tokens, { false, true });
+	CHECK(res.IsSuccessful
+		&& tokens.size() == 2
+		&& tokens[0].Type == TokenType::Comment
+		&& tokens[0].Lexeme == "/*comment\ncomment*/"
+		&& tokens[1].Type == TokenType::Eof);
+}
+
+void TestConsecutiveMultiLineComment()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize("/*comment1\ncomment1*//*comment2\ncomment2*/", tokens, { false, true });
+	CHECK(res.IsSuccessful
+		&& tokens.size() == 3
+		&& tokens[0].Type == TokenType::Comment
+		&& tokens[0].Lexeme == "/*comment1\ncomment1*/"
+		&& tokens[1].Type == TokenType::Comment
+		&& tokens[1].Lexeme == "/*comment2\ncomment2*/"
+		&& tokens[2].Type == TokenType::Eof);
+}
+
+void TestConsecutiveMultiLineCommentWithSpace()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize("/*comment1\ncomment1*/\n\n\n/*comment2\ncomment2*/", tokens, { false, true });
+	CHECK(res.IsSuccessful
+		&& tokens.size() == 3
+		&& tokens[0].Type == TokenType::Comment
+		&& tokens[0].Lexeme == "/*comment1\ncomment1*/"
+		&& tokens[1].Type == TokenType::Comment
+		&& tokens[1].Lexeme == "/*comment2\ncomment2*/"
+		&& tokens[2].Type == TokenType::Eof);
+}
+
+void TestConsecutiveMultiLineCommentWithSpaceWithTokens()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize("//comment1\n/*comment2\ncomment2*/\n\n\n/*comment3\ncomment3*/", tokens, { true, true });
+	CHECK(res.IsSuccessful
+		&& tokens.size() == 6
+		&& tokens[0].Type == TokenType::Comment
+		&& tokens[0].Lexeme == "//comment1"
+		&& tokens[1].Type == TokenType::Whitespace
+		&& tokens[1].Lexeme == "\n"
+		&& tokens[2].Type == TokenType::Comment
+		&& tokens[2].Lexeme == "/*comment2\ncomment2*/"
+		&& tokens[3].Type == TokenType::Whitespace
+		&& tokens[3].Lexeme == "\n\n\n"
+		&& tokens[4].Type == TokenType::Comment
+		&& tokens[4].Lexeme == "/*comment3\ncomment3*/"
+		&& tokens[5].Type == TokenType::Eof);
+}
+
+void TestCommentError()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize(" aa\n /*", tokens, { false, true });
+	CHECK(!res.IsSuccessful && res.Error.Row == 1 && res.Error.Column == 3);
 }
 
 // Parser Tests
@@ -120,15 +217,22 @@ void TestScientificNotationNumber()
 
 int main()
 {
-	//EXECUTE_TEST(TestLess);
-	//EXECUTE_TEST(TestNumber);
-	//EXECUTE_TEST(TestNumberStartWithNine);
-	//EXECUTE_TEST(TestNumberStartWithZero);
-	//EXECUTE_TEST(TestString);
-	//EXECUTE_TEST(TestSpaceNewLineSpace);
-	//EXECUTE_TEST(TestStringSingleQuotedStrings);
-	//EXECUTE_TEST(TestKeyWord);
-	//EXECUTE_TEST(TestVariableDeclaration);
+	EXECUTE_TEST(TestLess);
+	EXECUTE_TEST(TestNumber);
+	EXECUTE_TEST(TestNumberStartWithNine);
+	EXECUTE_TEST(TestNumberStartWithZero);
+	EXECUTE_TEST(TestString);
+	EXECUTE_TEST(TestSpaceNewLineSpace);
+	EXECUTE_TEST(TestStringSingleQuotedStrings);
+	EXECUTE_TEST(TestKeyWord);
+	EXECUTE_TEST(TestVariableDeclaration);
+	EXECUTE_TEST(TestWhiteSpaceTokens);
+	EXECUTE_TEST(TestSimpleComment);
+	EXECUTE_TEST(TestSimpleMultiLineComment);
+	EXECUTE_TEST(TestConsecutiveMultiLineComment);
+	EXECUTE_TEST(TestConsecutiveMultiLineCommentWithSpace);
+	EXECUTE_TEST(TestConsecutiveMultiLineCommentWithSpaceWithTokens);
+	EXECUTE_TEST(TestCommentError);
 	EXECUTE_TEST(TestHexNumber);
 	EXECUTE_TEST(TestScientificNotationNumber);
 	// TestParseUnaryExpr();