From 6e19bb560d474736d45f5be011ff0830c7019991 Mon Sep 17 00:00:00 2001
From: Dimitar Trendafilov <dimitar@coherent-labs.com>
Date: Mon, 5 Nov 2018 23:34:00 +0200
Subject: Squashed 'spasm/' content from commit 08ee95d

git-subtree-dir: spasm
git-subtree-split: 08ee95d89054ba0d33697d613799a0a013aedf89
---
 src/asm/tokenizer.cpp | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 src/asm/tokenizer.cpp

(limited to 'src/asm/tokenizer.cpp')

diff --git a/src/asm/tokenizer.cpp b/src/asm/tokenizer.cpp
new file mode 100644
index 0000000..8694ccf
--- /dev/null
+++ b/src/asm/tokenizer.cpp
@@ -0,0 +1,51 @@
+#include <iostream>
+#include <memory>
+#include <new>
+
+#include "lexer.hpp"
+#include "token.hpp"
+#include "tokenizer.hpp"
+
+namespace SpasmImpl
+{
+namespace ASM
+{
+namespace Lexer
+{
+Tokenizer::Tokenizer(std::istream& istr)
+    : _lexer(new Lexer(istr, 1024)), end_input(false)
+{
+    _lexer->buffer_init();
+}
+
+void Tokenizer::push_token(const Token& token)
+{
+    _tokens.push(token);
+    if (token.type() == Token::endinput)
+        end_input = true;
+}
+
+Token Tokenizer::next_token()
+{
+    Token t(Token::endinput);
+    if (!_tokens.empty())
+    {
+        t = _tokens.front();
+        _tokens.pop();
+    }
+    else if (!end_input)
+    {
+        while (_tokens.empty())
+        {
+            _lexer->read();
+            _lexer->tokenize(*this);
+        }
+        t = _tokens.front();
+        _tokens.pop();
+    }
+    return t;
+}
+
+}  // namespace Lexer
+}  // namespace ASM
+}  // namespace SpasmImpl
-- 
cgit v1.3.1-23-gd82d