From a7575e08c67ab6e4f3c5a48b302c42a17d2500c2 Mon Sep 17 00:00:00 2001
From: martin dobrev <martindobrev0@gmail.com>
Date: Mon, 12 Oct 2020 22:56:16 +0300
Subject: Add support for new lines and tabs in tokenizer. Improve tests
 verbosity. Replace white spaces with tabs.

---
 JSImpl/src/Lexer.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'JSImpl/src/Lexer.h')

diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h
index 3c0c785..00acc7a 100644
--- a/JSImpl/src/Lexer.h
+++ b/JSImpl/src/Lexer.h
@@ -100,6 +100,8 @@ enum class TokenType
 	False,
 
 	Whitespace,
+	Tab,
+	NewLine,
 	Comment,
 
 	Eof,
@@ -119,6 +121,7 @@ struct LexerResult
 {
 	bool IsSuccessful;
 	IPLError Error;
+	IPLVector<Token> tokens;
 };
 
 struct LexerSettings
@@ -127,5 +130,5 @@ struct LexerSettings
 	bool CreateCommentTokens;
 };
 
-LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings);
-LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
\ No newline at end of file
+LexerResult Tokenize(const char* code, const LexerSettings& settings);
+LexerResult Tokenize(const char* code);
\ No newline at end of file
-- 
cgit v1.3.1-23-gd82d