1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
#pragma once
#include "CommonTypes.h"
#include <functional>
enum TokenType
{
// Single-character tokens.
LeftParen,
RightParen,
LeftBrace,
RightBrace,
Comma,
Dot,
Minus,
Plus,
Semicolon,
Star,
Division,
Modulo,
BitwiseNot,
BitwiseAnd,
BitwiseXor,
BitwiseOr,
QuestionMark,
Colon,
LeftSquareBracket,
RightSquareBracket,
// One or two character tokens
Bang,
BangEqual,
Equal,
EqualEqual,
StrictEqual,
StrictNotEqual,
Greater,
GreaterEqual,
Less,
LessEqual,
MinusMinus,
PlusPlus,
LeftShift,
RightShift,
LogicalAnd,
LogicalOr,
StarEqual,
DivideEqual,
ModuloEqual,
PlusEqual,
MinusEqual,
LeftShiftEqual,
RightShiftEqual,
BitwiseAndEqual,
BitwiseXorEqual,
BitwiseOrEqual,
Identifier,
String,
Number,
// Keywords. ECMAScript 2015
Break,
Case,
Catch,
Class,
Const,
Continue,
Debugger,
Default,
Delete,
Do,
Else,
Export,
Extends,
Finally,
For,
Function,
If,
Import,
In,
Instanceof,
New,
Return,
Super,
Switch,
This,
Throw,
Try,
Typeof,
Var,
Void,
While,
With,
Yield,
Null,
Undefined,
True,
False,
Eof,
};
struct Token
{
TokenType Type;
unsigned Line;
IPLString Lexeme;
double Number = 0.0;
};
IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError = {});
|
