feat: Output token list

1 files changed, 6 insertions, 0 deletions

diff --git a/src/StringConsumer/StringConsumerEngine.cl b/src/StringConsumer/StringConsumerEngine.cl
index 6fe4403..6f07d2e 100644
--- a/src/StringConsumer/StringConsumerEngine.cl
+++ b/src/StringConsumer/StringConsumerEngine.cl
@@ -4,6 +4,7 @@ class StringConsumerEngine inherits System {
 		;
 
 	str: String;
+	tl: TokenList <- new TokenList;
 
 	init(code: String) : Object {{
 		assert(not (code = ""), "init", "input code cannot be empty!");
@@ -23,6 +24,7 @@ class StringConsumerEngine inherits System {
 				case consumers.find(str) of
 					c: IStringConsumer => {
 						str <- str.substr(c.consumed_length(), str.length() - c.consumed_length());
+						tl.push(c.token());
 						true;
 					};
 					o: Object => false;
@@ -37,4 +39,8 @@ class StringConsumerEngine inherits System {
 	empty() : Bool {
 		str = ""
 	};
+
+	generated_tokens() : TokenList {
+		tl
+	};
 };