diff options
| Commit | 7509dce0686d9a418975f73b3721ecaa2d534f16 (patch) | |
|---|---|---|
| Author | 2018-10-21 15:20:12 +0300 | |
| Committer | 2018-10-21 15:20:12 +0300 | |
| Files | 0dc9528a6bb780560af29b660550a05f91d7f91a /JSImpl/src/Lexer.h | |
| IPL-7509dce0686d9a418975f73b3721ecaa2d534f16.tar IPL-7509dce0686d9a418975f73b3721ecaa2d534f16.tar.gz IPL-7509dce0686d9a418975f73b3721ecaa2d534f16.zip | ||
| Parent | 3075429adc93c18c72a77a311056debe89342aca (diff) | |
Lexer whitespace and commens tokens. (#13)
* Lexer can create whitespace tokens
* Parse whitespaces, comments and error fixes
* Brace error fix
* Less explicit default settings tokenizer.
1 files changed, 12 insertions, 1 deletions
| -rw-r--r-- | JSImpl/src/Lexer.h | 13 |
diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h index fb3e0a1..e17270d 100644 --- a/JSImpl/src/Lexer.h +++ b/JSImpl/src/Lexer.h @@ -88,6 +88,7 @@ enum class TokenType Try, Typeof, Var, + Let, Void, While, With, @@ -97,6 +98,9 @@ enum class TokenType True, False, + Whitespace, + Comment, + Eof, Invalid }; @@ -115,4 +119,11 @@ struct LexerResult IPLError Error; }; -LexerResult Tokenize(const char* code, IPLVector<Token>& tokens); +struct LexerSettings +{ + bool CreateWhitespaceTokens; + bool CreateCommentTokens; +}; + +LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings); +LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
\ No newline at end of file |