feat!: Add basic lexer components

2 files changed, 34 insertions, 0 deletions

diff --git a/src/Token/IToken.cl b/src/Token/IToken.cl
new file mode 100644
index 0000000..05282ab
--- /dev/null
+++ b/src/Token/IToken.cl
@@ -0,0 +1,24 @@
+class IToken inherits System {
+---
+	name() : String {
+		not_implemented_retstr("name")
+	};
+
+	value() : String {
+		not_implemented_retstr("value")
+	};
+
+---
+	id: Int;
+
+	id() : Int {
+		id
+	};
+
+	init(iid: Int) : Object {{
+		assert(iid = 0, "init", "ID 0 is not allowed for a token!");
+		assert(not (id = 0), "init", "token object '".concat(name()).concat("' is already initialized!"));
+		id <- iid;
+		void;
+	}};
+};
diff --git a/src/Token/WhitespaceToken.cl b/src/Token/WhitespaceToken.cl
new file mode 100644
index 0000000..e0d70c0
--- /dev/null
+++ b/src/Token/WhitespaceToken.cl
@@ -0,0 +1,10 @@
+class WhitespaceToken inherits IToken {
+	name() : String {
+		"WHITESPACE"
+	};
+
+	-- We'll never use the whitespace value
+	value() : String {
+		""
+	};
+};