diff options
1 files changed, 12 insertions, 1 deletions
| -rw-r--r-- | JSImpl/src/Lexer.h | 13 |
diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h index fb3e0a1..e17270d 100644 --- a/JSImpl/src/Lexer.h +++ b/JSImpl/src/Lexer.h @@ -88,6 +88,7 @@ enum class TokenType Try, Typeof, Var, + Let, Void, While, With, @@ -97,6 +98,9 @@ enum class TokenType True, False, + Whitespace, + Comment, + Eof, Invalid }; @@ -115,4 +119,11 @@ struct LexerResult IPLError Error; }; -LexerResult Tokenize(const char* code, IPLVector<Token>& tokens); +struct LexerSettings +{ + bool CreateWhitespaceTokens; + bool CreateCommentTokens; +}; + +LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings); +LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
\ No newline at end of file |