1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
#pragma once
#include "CommonTypes.h"
#include <functional>
enum TokenType
{
// Single-character tokens.
LeftParen,
RightParen,
LeftBrace,
RightBrace,
Comma,
Dot,
Minus,
Plus,
Semicolon,
Star,
Division,
Modulo,
BitwiseNot,
BitwiseAnd,
BitwiseXor,
BitwiseOr,
// One or two character tokens
Bang,
BangEqual,
Equal,
EqualEqual,
StrictEqual,
StrictNotEqual,
Greater,
GreaterEqual,
Less,
LessEqual,
MinusMinus,
PlusPlus,
LeftShift,
RightShift,
LogicalAnd,
LogicalOr,
Identifier,
String,
Number,
// Keywords. ECMAScript 2015
Break,
Case,
Catch,
Class,
Const,
Continue,
Debugger,
Default,
Delete,
Do,
Else,
Export,
Extends,
Finally,
For,
Function,
If,
Import,
In,
Instanceof,
New,
Return,
Super,
Switch,
This,
Throw,
Try,
Typeof,
Var,
Void,
While,
With,
Yield,
Eof,
};
struct Token
{
TokenType Type;
unsigned Line;
IPLString Lexeme;
double Number;
};
IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError = {});
|
