From 7509dce0686d9a418975f73b3721ecaa2d534f16 Mon Sep 17 00:00:00 2001
From: omalinov <ognyan.m@coherent-labs.com>
Date: Sun, 21 Oct 2018 15:20:12 +0300
Subject: Lexer whitespace and commens tokens. (#13)

* Lexer can create whitespace tokens

* Parse whitespaces, comments and error fixes

* Brace error fix

* Less explicit default settings tokenizer.
---
 JSImpl/src/Lexer.h | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'JSImpl/src/Lexer.h')

diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h
index fb3e0a1..e17270d 100644
--- a/JSImpl/src/Lexer.h
+++ b/JSImpl/src/Lexer.h
@@ -88,6 +88,7 @@ enum class TokenType
 	Try,
 	Typeof,
 	Var,
+	Let,
 	Void,
 	While,
 	With,
@@ -97,6 +98,9 @@ enum class TokenType
 	True,
 	False,
 
+	Whitespace,
+	Comment,
+
 	Eof,
 	Invalid
 };
@@ -115,4 +119,11 @@ struct LexerResult
 	IPLError Error;
 };
 
-LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
+struct LexerSettings
+{
+	bool CreateWhitespaceTokens;
+	bool CreateCommentTokens;
+};
+
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings);
+LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
\ No newline at end of file
-- 
cgit v1.3.1-23-gd82d