Merge commit '6e19bb560d474736d45f5be011ff0830c7019991' as 'spasm'

1 files changed, 51 insertions, 0 deletions

diff --git a/spasm/src/asm/tokenizer.cpp b/spasm/src/asm/tokenizer.cpp
new file mode 100644
index 0000000..8694ccf
--- /dev/null
+++ b/spasm/src/asm/tokenizer.cpp
@@ -0,0 +1,51 @@
+#include <iostream>
+#include <memory>
+#include <new>
+
+#include "lexer.hpp"
+#include "token.hpp"
+#include "tokenizer.hpp"
+
+namespace SpasmImpl
+{
+namespace ASM
+{
+namespace Lexer
+{
+Tokenizer::Tokenizer(std::istream& istr)
+    : _lexer(new Lexer(istr, 1024)), end_input(false)
+{
+    _lexer->buffer_init();
+}
+
+void Tokenizer::push_token(const Token& token)
+{
+    _tokens.push(token);
+    if (token.type() == Token::endinput)
+        end_input = true;
+}
+
+Token Tokenizer::next_token()
+{
+    Token t(Token::endinput);
+    if (!_tokens.empty())
+    {
+        t = _tokens.front();
+        _tokens.pop();
+    }
+    else if (!end_input)
+    {
+        while (_tokens.empty())
+        {
+            _lexer->read();
+            _lexer->tokenize(*this);
+        }
+        t = _tokens.front();
+        _tokens.pop();
+    }
+    return t;
+}
+
+}  // namespace Lexer
+}  // namespace ASM
+}  // namespace SpasmImpl