diff options
| Commit | a7575e08c67ab6e4f3c5a48b302c42a17d2500c2 (patch) | |
|---|---|---|
| Author | 2020-10-12 22:56:16 +0300 | |
| Committer | 2020-10-12 22:56:16 +0300 | |
| Files | ab3caa138c0048a0140af070d0dbbdd0d686f15c /JSImpl/src/Lexer.h | |
| IPL-a7575e08c67ab6e4f3c5a48b302c42a17d2500c2.tar IPL-a7575e08c67ab6e4f3c5a48b302c42a17d2500c2.tar.gz IPL-a7575e08c67ab6e4f3c5a48b302c42a17d2500c2.zip | ||
| Parent | d9183dd25685297d0b9278591b910bb753bc93ed (diff) | |
Add support for new lines and tabs in tokenizer.
Improve tests verbosity.
Replace white spaces with tabs.
1 files changed, 5 insertions, 2 deletions
| -rw-r--r-- | JSImpl/src/Lexer.h | 7 |
diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h index 3c0c785..00acc7a 100644 --- a/JSImpl/src/Lexer.h +++ b/JSImpl/src/Lexer.h @@ -100,6 +100,8 @@ enum class TokenType False, Whitespace, + Tab, + NewLine, Comment, Eof, @@ -119,6 +121,7 @@ struct LexerResult { bool IsSuccessful; IPLError Error; + IPLVector<Token> tokens; }; struct LexerSettings @@ -127,5 +130,5 @@ struct LexerSettings bool CreateCommentTokens; }; -LexerResult Tokenize(const char* code, IPLVector<Token>& tokens, const LexerSettings& settings); -LexerResult Tokenize(const char* code, IPLVector<Token>& tokens);
\ No newline at end of file +LexerResult Tokenize(const char* code, const LexerSettings& settings); +LexerResult Tokenize(const char* code);
\ No newline at end of file |