Squashed 'spasm/' content from commit 08ee95d

git-subtree-dir: spasm
git-subtree-split: 08ee95d89054ba0d33697d613799a0a013aedf89

1 files changed, 35 insertions, 0 deletions

diff --git a/src/asm/tokenizer.hpp b/src/asm/tokenizer.hpp
new file mode 100644
index 0000000..1155db7
--- /dev/null
+++ b/src/asm/tokenizer.hpp
@@ -0,0 +1,35 @@
+#ifndef TOKENIZER_HPP
+#define TOKENIZER_HPP
+
+#include <iostream>
+#include <memory>
+#include <queue>
+
+#include "lexer.hpp"
+#include "token.hpp"
+
+namespace SpasmImpl
+{
+namespace ASM
+{
+namespace Lexer
+{
+class Tokenizer : public TokenStream
+{
+   public:
+    Tokenizer(std::istream&);
+
+    virtual void push_token(const Token&);
+    Token next_token();
+
+   private:
+    std::unique_ptr<Lexer> _lexer;
+    std::queue<Token> _tokens;
+    bool end_input;
+};  // class Tokenizer
+}  // namespace Lexer
+
+}  // namespace ASM
+}  // namespace SpasmImpl
+
+#endif