Merge branch 'omalinov-lexer-errors-handling'

6 files changed, 340 insertions, 128 deletions

diff --git a/JSImpl/src/CommonTypes.h b/JSImpl/src/CommonTypes.h
index fb5ef2a..e9c1f1b 100644
--- a/JSImpl/src/CommonTypes.h
+++ b/JSImpl/src/CommonTypes.h
@@ -21,3 +21,11 @@ inline IPLSharedPtr<T> IPLMakeSharePtr(Args&&... args) {
 
 template< typename T>
 using IPLEnableShared = std::enable_shared_from_this<T>;
+
+struct IPLError
+{
+	unsigned Row;
+	unsigned Column;
+	IPLString File;
+	IPLString What;
+};
\ No newline at end of file
diff --git a/JSImpl/src/Lexer.cpp b/JSImpl/src/Lexer.cpp
index 22ffab8..2e69087 100644
--- a/JSImpl/src/Lexer.cpp
+++ b/JSImpl/src/Lexer.cpp
@@ -1,42 +1,128 @@
 #include "Lexer.h"
+#include <utility>
+
+namespace
+{
+	inline bool IsUpperCase(char c)
+	{
+		return c >= 'A' && c <= 'Z';
+	}
+
+	inline bool IsLowerCase(char c)
+	{
+		return c >= 'a' && c <= 'z';
+	}
+
+	inline bool IsDigit(char c)
+	{
+		return c >= '0' && c <= '9';
+	}
+
+	inline bool IsValidIdentifierStartingChar(char c)
+	{
+		return IsUpperCase(c) || IsLowerCase(c) || c == '_';
+	}
+
+	inline bool IsValidIdentifierChar(char c)
+	{
+		return IsValidIdentifierStartingChar(c) || IsDigit(c);
+	}
+
+	inline bool IsStringBound(char c)
+	{
+		return c == '\'' || c == '"';
+	}
+
+	inline bool IsWhiteSpace(char c)
+	{
+		return c == ' ' || c == '\t';
+	}
+
+	inline bool IsNewLine(char c)
+	{
+		return c == '\n';
+	}
+
+	inline bool IsEnd(char c)
+	{
+		return c == '\0';
+	}
+}
+
+struct Keyword
+{
+	TokenType Type;
+	IPLString Content;
+};
+
+using Identifier = Keyword;
+
+enum class State: unsigned char
+{
+	Success,
+	Fail,
+	Error
+};
+
+#define RETURN_SUCCESS(ret) m_GenerationState = State::Success; return ret;
+#define RETURN_FAIL(ret) m_GenerationState = State::Fail; return ret;
+#define RETURN_ERROR(ret) m_GenerationState = State::Error; return ret;
 
 class Tokenizer
 {
 public:
-	Tokenizer(const char* code, const std::function<void()>& onError);
-	IPLVector<Token> Tokenize();
+	Tokenizer(const char* code);
+	LexerResult Tokenize(IPLVector<Token>& tokens);
 
 private:
 	Token NextToken();
 
-	Token ProduceToken(TokenType type);
-	Token ProduceToken(TokenType type, IPLString lexeme);
+	inline Token ProduceToken(TokenType type);
+	inline Token ProduceToken(TokenType type, IPLString lexeme, double number);
+	inline Token ProduceErrorToken();
+
 	double ParseNumber();
 	IPLString ParseString();
+	Keyword ParseKeyword();
+	Identifier ParseIdentifier();
+
+	inline bool IsStateSuccess() const;
+	inline bool IsStateError() const;
+
+	void SetError(const IPLString& what);
+
 	bool SkipWhiteSpaces();
 	bool SkipNewLine();
 	bool Match(const char c);
 
-	std::function<void()> OnError;
+	inline void NextSymbol();
+	inline void NextLine();
+
 	unsigned m_Line;
+	unsigned m_Column;
 	unsigned m_Current;
 	const char* m_Code;
 
+	IPLError m_Error;
+	State m_GenerationState;
+
 	IPLUnorderedMap<IPLString, TokenType> m_KeyWordsTable;
 };
 
-IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError)
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens)
 {
-	Tokenizer tokenizer(code, onError);
+	Tokenizer tokenizer(code);
 
-	return tokenizer.Tokenize();
+	return tokenizer.Tokenize(tokens);
 }
 
-Tokenizer::Tokenizer(const char* code, const std::function<void()>& onError)
-	: OnError(onError)
-	, m_Line(0)
+Tokenizer::Tokenizer(const char* code)
+	: m_Line(0)
+	, m_Column(0)
 	, m_Current(0)
 	, m_Code(code)
+	, m_Error()
+	, m_GenerationState(State::Success)
 {
 	m_KeyWordsTable["break"] = TokenType::Break;
 	m_KeyWordsTable["case"] = TokenType::Case;
@@ -77,23 +163,30 @@ Tokenizer::Tokenizer(const char* code, const std::function<void()>& onError)
 	m_KeyWordsTable["false"] = TokenType::False;
 }
 
-IPLVector<Token> Tokenizer::Tokenize()
+LexerResult Tokenizer::Tokenize(IPLVector<Token>& tokens)
 {
-	IPLVector<Token> result;
 	do
 	{
-		result.push_back(NextToken());
-	} while (result.back().Type != TokenType::Eof);
+		const auto& token = NextToken();
+
+		if (m_GenerationState == State::Error)
+		{
+			return LexerResult{ false, IPLError(m_Error) };
+		}
+
+		tokens.emplace_back(token);
 
-	return result;
+	} while (tokens.back().Type != TokenType::Eof);
+
+	return LexerResult{ true, IPLError() };
 }
 
 bool Tokenizer::SkipWhiteSpaces()
 {
 	const auto old = m_Current;
-	while (m_Code[m_Current] == ' ' || m_Code[m_Current] == '\t')
+	while (IsWhiteSpace(m_Code[m_Current]))
 	{
-		m_Current++;
+		NextSymbol();
 	}
 	return old != m_Current;
 }
@@ -101,10 +194,9 @@ bool Tokenizer::SkipWhiteSpaces()
 bool Tokenizer::SkipNewLine()
 {
 	const auto old = m_Current;
-	while (m_Code[m_Current] == '\n')
+	while (IsNewLine(m_Code[m_Current]))
 	{
-		m_Current++;
-		m_Line++;
+		NextLine();
 	}
 	return old != m_Current;
 }
@@ -113,40 +205,160 @@ bool Tokenizer::Match(const char c)
 {
 	if (c == m_Code[m_Current])
 	{
-		++m_Current;
+		if (!IsNewLine(m_Code[m_Current]))
+		{
+			NextSymbol();
+		}
+		else
+		{
+			NextLine();
+		}
+
 		return true;
 	}
 	return false;
 }
 
-Token Tokenizer::ProduceToken(TokenType type)
+inline void Tokenizer::NextSymbol()
 {
-	return { type, m_Line, "", 0.0 };
+	++m_Current;
+	++m_Column;
 }
 
-Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme)
+inline void Tokenizer::NextLine()
 {
-	return{ type, m_Line, lexeme, 0.0 };
+	++m_Current;
+	++m_Line;
+	m_Column = 0;
 }
 
-namespace
+inline Token Tokenizer::ProduceToken(TokenType type)
+{
+	return Token{ type, m_Line, "", 0.0 };
+}
+
+inline Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme, double number = 0.0)
+{
+	return Token{ type, m_Line, lexeme, number };
+}
+
+inline Token Tokenizer::ProduceErrorToken()
+{
+	return Token{ TokenType::Eof, m_Line, "", 0.0 };
+}
+
+double Tokenizer::ParseNumber()
+{
+	if (!IsDigit(m_Code[m_Current]))
+	{
+		RETURN_FAIL(0.0);
+	}
+
+	size_t parsedBytes = 0;
+	double number = std::stod(m_Code + m_Current, &parsedBytes);
+	m_Current += static_cast<unsigned>(parsedBytes);
+	m_Column += static_cast<unsigned>(parsedBytes);
+
+	RETURN_SUCCESS(number);
+}
+
+IPLString Tokenizer::ParseString()
+{
+	if (!IsStringBound(m_Code[m_Current]))
+	{
+		RETURN_FAIL(IPLString());
+	}
+
+	char bound = m_Code[m_Current];
+	auto start = m_Current;
+
+	// skip first " or '
+	NextSymbol();
+	while (m_Code[m_Current] != bound && !IsEnd(m_Code[m_Current]) && !IsNewLine(m_Code[m_Current]))
+	{
+		NextSymbol();
+	}
+
+	if (IsEnd(m_Code[m_Current])  || IsNewLine(m_Code[m_Current]))
+	{
+		SetError("\"\" string literal contains an unescaped line break");
+		RETURN_ERROR(IPLString());
+	}
+
+	// skip second " or '
+	NextSymbol();
+
+	RETURN_SUCCESS(IPLString(m_Code + start, m_Code + m_Current));
+}
+
+Keyword Tokenizer::ParseKeyword()
+{
+	if (!IsLowerCase(m_Code[m_Current]))
+	{
+		RETURN_FAIL(Keyword());
+	}
+
+	auto start = m_Current;
+
+	NextSymbol();
+	while (IsLowerCase(m_Code[m_Current]))
+	{
+		NextSymbol();
+	}
+
+	auto key = IPLString(m_Code + start, m_Code + m_Current);
+	auto keyword = m_KeyWordsTable.find(key);
+
+	if (keyword == m_KeyWordsTable.end())
+	{
+		// It's not a keyword so we must revert current counter
+		m_Current = start;
+		m_Column = start;
+
+		RETURN_FAIL(Keyword());
+	}
+
+	RETURN_SUCCESS((Keyword{ keyword->second, keyword->first }));
+}
+
+Identifier Tokenizer::ParseIdentifier()
 {
-inline bool IsValidIdentifierStartingChar(char c)
+	if (!IsValidIdentifierStartingChar(m_Code[m_Current]))
+	{
+		RETURN_FAIL(Identifier());
+	}
+
+	auto start = m_Current;
+
+	NextSymbol();
+	while (IsValidIdentifierChar(m_Code[m_Current]))
+	{
+		NextSymbol();
+	}
+
+	RETURN_SUCCESS((Identifier{ TokenType::Identifier, IPLString(m_Code + start, m_Code + m_Current) }));
+}
+
+inline bool Tokenizer::IsStateSuccess() const
 {
-	return (c >= 'A' && c <= 'z') || c == '_';
+	return m_GenerationState == State::Success;
 }
 
-inline bool IsValidIdentifierChar(char c)
+inline bool Tokenizer::IsStateError() const
 {
-	return IsValidIdentifierStartingChar(c) || (c >= '0' && c <= '9');
+	return m_GenerationState == State::Error;
 }
+
+void Tokenizer::SetError(const IPLString & what)
+{
+	m_Error = IPLError{ m_Line, m_Column, "", what };
 }
 
 Token Tokenizer::NextToken()
 {
-	if (m_Code[m_Current] == '\0')
+	if (IsEnd(m_Code[m_Current]))
 	{
-		return Token{ TokenType::Eof, 0, "", 0.0 };
+		ProduceToken(TokenType::Eof);
 	}
 
 	while (SkipWhiteSpaces() || SkipNewLine());
@@ -154,98 +366,61 @@ Token Tokenizer::NextToken()
 	// Single Char
 	switch (m_Code[m_Current])
 	{
-	case '(': m_Current++; return ProduceToken(TokenType::LeftParen);
-	case ')': m_Current++; return ProduceToken(TokenType::RightParen);
-	case '{': m_Current++; return ProduceToken(TokenType::LeftBrace);
-	case '}': m_Current++; return ProduceToken(TokenType::RightBrace);
-	case ',': m_Current++; return ProduceToken(TokenType::Comma);
-	case '.': m_Current++; return ProduceToken(TokenType::Dot);
-	case '-': m_Current++; return Match('-') ? ProduceToken(TokenType::MinusMinus) : ProduceToken(TokenType::Minus);
-	case '+': m_Current++; return Match('+') ? ProduceToken(TokenType::PlusPlus) : ProduceToken(TokenType::Plus);
-	case ';': m_Current++; return ProduceToken(TokenType::Semicolon);
-	case '*': m_Current++; return ProduceToken(TokenType::Star);
-	case '/': m_Current++; return ProduceToken(TokenType::Division);
-	case '%': m_Current++; return ProduceToken(TokenType::Modulo);
-	case '~': m_Current++; return ProduceToken(TokenType::BitwiseNot);
-	case '=': m_Current++; return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictEqual) : ProduceToken(TokenType::EqualEqual) : ProduceToken(TokenType::Equal);
-	case '!': m_Current++; return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictNotEqual) : ProduceToken(TokenType::BangEqual) : ProduceToken(TokenType::Bang);
-	case '>': m_Current++; return Match('=') ? ProduceToken(TokenType::GreaterEqual) : Match('>') ? ProduceToken(TokenType::RightShift) : ProduceToken(TokenType::Greater);
-	case '<': m_Current++; return Match('=') ? ProduceToken(TokenType::LessEqual) : Match('<') ? ProduceToken(TokenType::LeftShift) : ProduceToken(TokenType::Less);
-	case '&': m_Current++; return Match('&') ? ProduceToken(TokenType::LogicalAnd) : ProduceToken(TokenType::BitwiseAnd);
-	case '^': m_Current++; return ProduceToken(TokenType::BitwiseXor);
-	case '|': m_Current++; return Match('|') ? ProduceToken(TokenType::LogicalOr) : ProduceToken(TokenType::BitwiseOr);
-	case '?': m_Current++; return ProduceToken(TokenType::QuestionMark);
-	case ':': m_Current++; return ProduceToken(TokenType::Colon);
-	case '[': m_Current++; return ProduceToken(TokenType::LeftSquareBracket);
-	case ']': m_Current++; return ProduceToken(TokenType::RightSquareBracket);
+	case '(': NextSymbol(); return ProduceToken(TokenType::LeftParen);
+	case ')': NextSymbol(); return ProduceToken(TokenType::RightParen);
+	case '{': NextSymbol(); return ProduceToken(TokenType::LeftBrace);
+	case '}': NextSymbol(); return ProduceToken(TokenType::RightBrace);
+	case ',': NextSymbol(); return ProduceToken(TokenType::Comma);
+	case '.': NextSymbol(); return ProduceToken(TokenType::Dot);
+	case '-': NextSymbol(); return Match('-') ? ProduceToken(TokenType::MinusMinus) : ProduceToken(TokenType::Minus);
+	case '+': NextSymbol(); return Match('+') ? ProduceToken(TokenType::PlusPlus) : ProduceToken(TokenType::Plus);
+	case ';': NextSymbol(); return ProduceToken(TokenType::Semicolon);
+	case '*': NextSymbol(); return ProduceToken(TokenType::Star);
+	case '/': NextSymbol(); return ProduceToken(TokenType::Division);
+	case '%': NextSymbol(); return ProduceToken(TokenType::Modulo);
+	case '~': NextSymbol(); return ProduceToken(TokenType::BitwiseNot);
+	case '=': NextSymbol(); return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictEqual) : ProduceToken(TokenType::EqualEqual) : ProduceToken(TokenType::Equal);
+	case '!': NextSymbol(); return Match('=') ? Match('=') ? ProduceToken(TokenType::StrictNotEqual) : ProduceToken(TokenType::BangEqual) : ProduceToken(TokenType::Bang);
+	case '>': NextSymbol(); return Match('=') ? ProduceToken(TokenType::GreaterEqual) : Match('>') ? ProduceToken(TokenType::RightShift) : ProduceToken(TokenType::Greater);
+	case '<': NextSymbol(); return Match('=') ? ProduceToken(TokenType::LessEqual) : Match('<') ? ProduceToken(TokenType::LeftShift) : ProduceToken(TokenType::Less);
+	case '&': NextSymbol(); return Match('&') ? ProduceToken(TokenType::LogicalAnd) : ProduceToken(TokenType::BitwiseAnd);
+	case '^': NextSymbol(); return ProduceToken(TokenType::BitwiseXor);
+	case '|': NextSymbol(); return Match('|') ? ProduceToken(TokenType::LogicalOr) : ProduceToken(TokenType::BitwiseOr);
+	case '?': NextSymbol(); return ProduceToken(TokenType::QuestionMark);
+	case ':': NextSymbol(); return ProduceToken(TokenType::Colon);
+	case '[': NextSymbol(); return ProduceToken(TokenType::LeftSquareBracket);
+	case ']': NextSymbol(); return ProduceToken(TokenType::RightSquareBracket);
 	default:
 		break;
 	}
-	// Number
-	const char c = m_Code[m_Current];
-	if (c >= '0' && c <= '9')
+
+	const auto& number = ParseNumber();
+	if (IsStateSuccess())
 	{
-		size_t parsedBytes = 0;
-		const double number = std::stod(m_Code + m_Current, &parsedBytes);
-		m_Current += (unsigned)parsedBytes;
-		return Token{ TokenType::Number, m_Line, "", number };
+		return ProduceToken(TokenType::Number, "", number);
 	}
 
-	// String
-	if (c == '"')
+	const auto& string = ParseString();
+	if (IsStateSuccess())
 	{
-		auto start = m_Current;
-		m_Current++; // skip first '"'
-		while (m_Code[m_Current] != '"')
-		{
-			++m_Current;
-		}
-		m_Current++; // skip second '"'
-		return Token{ TokenType::String, m_Line, IPLString(m_Code + start, m_Code + m_Current), 0.0 };
+		return ProduceToken(TokenType::String, string);
 	}
-
-	if (c == '\'')
+	else if (IsStateError())
 	{
-		auto start = m_Current;
-		m_Current++; // skip first '
-		while (m_Code[m_Current] != '\'')
-		{
-			++m_Current;
-		}
-		m_Current++; // skip second '
-		return Token{ TokenType::String, m_Line, IPLString(m_Code + start, m_Code + m_Current), 0.0 };
+		return ProduceErrorToken();
 	}
 
-	// Key words
-	if (c >= 'a' && c <= 'z')
+	const auto& keyword = ParseKeyword();
+	if (IsStateSuccess())
 	{
-		auto start = m_Current;
-		m_Current++;
-		while (m_Code[m_Current] >= 'a' && m_Code[m_Current] <= 'z')
-		{
-			++m_Current;
-		}
-		auto key = IPLString(m_Code + start, m_Code + m_Current);
-		auto word = m_KeyWordsTable.find(key);
-		if (word != m_KeyWordsTable.end())
-		{
-			return Token{ word->second, m_Line, key, 0.0 };
-		}
-		// It's not a key word so we must revert current counter
-		m_Current = start;
+		return ProduceToken(keyword.Type, keyword.Content);
 	}
 
-	// Identifier
-	if (IsValidIdentifierStartingChar(c))
+	const auto& identifier = ParseIdentifier();
+	if (IsStateSuccess())
 	{
-		auto start = m_Current;
-		m_Current++;
-		while (IsValidIdentifierChar(m_Code[m_Current]))
-		{
-			++m_Current;
-		}
-		return Token{ TokenType::Identifier, m_Line, IPLString(m_Code + start, m_Code + m_Current), 0.0 };
+		return ProduceToken(identifier.Type, identifier.Content);
 	}
-	// TODO: Error reporting
-	return Token{ TokenType::Eof, m_Line, "", 0.0 };
+
+	return ProduceErrorToken();
 }
diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h
index 914b6ed..fb3e0a1 100644
--- a/JSImpl/src/Lexer.h
+++ b/JSImpl/src/Lexer.h
@@ -1,9 +1,8 @@
 #pragma once
 
 #include "CommonTypes.h"
-#include <functional>
 
-enum TokenType
+enum class TokenType
 {
 	// Single-character tokens.
 	LeftParen,
@@ -98,7 +97,6 @@ enum TokenType
 	True,
 	False,
 
-
 	Eof,
 	Invalid
 };
@@ -111,4 +109,10 @@ struct Token
 	double Number;
 };
 
-IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError = {});
+struct LexerResult
+{
+	bool IsSuccessful;
+	IPLError Error;
+};
+
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
diff --git a/JSImpl/src/Parser.cpp b/JSImpl/src/Parser.cpp
index 09166db..a2a46e2 100644
--- a/JSImpl/src/Parser.cpp
+++ b/JSImpl/src/Parser.cpp
@@ -344,7 +344,7 @@ ExpressionPtr Parser::Unary()
 	//	| -UnaryExpressionnormal
 	//	| ~UnaryExpressionnormal
 	//	| !UnaryExpressionnormal
-	if (MatchOneOf({ 
+	if (MatchOneOf({
 		TokenType::Delete,
 		TokenType::MinusMinus,
 		TokenType::PlusPlus,
diff --git a/JSImpl/src/Parser.h b/JSImpl/src/Parser.h
index 380224c..33890cb 100644
--- a/JSImpl/src/Parser.h
+++ b/JSImpl/src/Parser.h
@@ -3,5 +3,6 @@
 #include "CommonTypes.h"
 #include "Expression.h"
 #include "Lexer.h"
+#include <functional>
 
 ExpressionPtr Parse(const IPLVector<Token>&, const std::function<void()>& onError = {});
diff --git a/JSImpl/src/main.cpp b/JSImpl/src/main.cpp
index 586eef1..bd57ce7 100644
--- a/JSImpl/src/main.cpp
+++ b/JSImpl/src/main.cpp
@@ -4,36 +4,44 @@
 #include <iostream>
 #define LOG(msg) std::cout << msg << std::endl
 #define EXECUTE_TEST(test) std::cout << #test << " "; test();
+
 #define CHECK(cond)	if (cond) {LOG("PASS");} else {LOG(#cond " FAIL");}
 
 // Lexer Tests
 void TestLess()
 {
-	auto tokens = Tokenize("<");
+	IPLVector<Token> tokens;
+	Tokenize("<", tokens);
+
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::Less);
 }
 
 void TestNumber()
 {
-	auto tokens = Tokenize("213434.24");
+	IPLVector<Token> tokens;
+	Tokenize("213434.24", tokens);
+
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::Number && tokens[0].Number == 213434.24);
 }
 
 void TestNumberStartWithNine()
 {
-	auto tokens = Tokenize("999");
+	IPLVector<Token> tokens;
+	Tokenize("999", tokens);
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::Number && tokens[0].Number == 999);
 }
 
 void TestNumberStartWithZero()
 {
-	auto tokens = Tokenize("0999");
+	IPLVector<Token> tokens;
+	Tokenize("0999", tokens);
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::Number && tokens[0].Number == 999);
 }
 
 void TestSpaceNewLineSpace()
 {
-	auto tokens = Tokenize(" \n var a = 4;");
+	IPLVector<Token> tokens;
+	Tokenize(" \n var a = 4; ", tokens);
 	CHECK(tokens.size() == 6 && tokens[0].Type == TokenType::Var &&
 		tokens[1].Type == TokenType::Identifier &&
 		tokens[2].Type == TokenType::Equal &&
@@ -43,36 +51,53 @@ void TestSpaceNewLineSpace()
 
 void TestString()
 {
-	auto tokens = Tokenize("\"alabala\"");
+	IPLVector<Token> tokens;
+	Tokenize("\"alabala\"", tokens);
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::String && tokens[0].Lexeme == "\"alabala\"");
 }
 
 void TestStringSingleQuotedStrings()
 {
-	auto tokens = Tokenize("'alabala'");
+	IPLVector<Token> tokens;
+	Tokenize("'alabala'", tokens);
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::String && tokens[0].Lexeme == "'alabala'");
 }
 
 void TestKeyWord()
 {
-	auto tokens = Tokenize("for");
+	IPLVector<Token> tokens;
+	Tokenize("for", tokens);
 	CHECK(tokens.size() == 2 && tokens[0].Type == TokenType::For);
 }
 
 void TestVariableDeclaration()
 {
-	auto tokens = Tokenize("var pesho = 10");
+	IPLVector<Token> tokens;
+	Tokenize("var pesho = 10", tokens);
+
 	CHECK(tokens.size() == 5 && tokens[0].Type == TokenType::Var
 		&& tokens[1].Type == TokenType::Identifier
 		&& tokens[2].Type == TokenType::Equal
 		&& tokens[3].Type == TokenType::Number
 		&& tokens[4].Type == TokenType::Eof);
 }
+
+void TestStringError()
+{
+	IPLVector<Token> tokens;
+	const auto& res = Tokenize("\"aaaa", tokens);
+
+	CHECK(!res.IsSuccessful && res.Error.Row == 0 && res.Error.Column == 5);
+}
+
 // Parser Tests
 void TestParseUnaryExpr()
 {
+	IPLVector<Token> tokens;
+	Tokenize("function pesho() { var a = 0; return a; }", tokens);
+
 	// TODO make actual test :D
-	auto expr = Parse(Tokenize("function pesho(v1) { var a = 0; return a; }"));
+	auto expr = Parse(tokens);
 	ASTPrinter p(std::cout);
 	expr->Accept(p);
 }
@@ -89,10 +114,9 @@ int main()
 	//EXECUTE_TEST(TestKeyWord);
 	//EXECUTE_TEST(TestVariableDeclaration);
 
-
 	TestParseUnaryExpr();
 #if defined(_WIN32)
-	//std::system("pause");
+	std::system("pause");
 #endif
 	return 0;
 }