From 4fb889400b4f4e478bea030d1484becc89a1025c Mon Sep 17 00:00:00 2001
From: martin dobrev <martindobrev0@gmail.com>
Date: Fri, 7 Sep 2018 12:13:57 +0300
Subject: Implement identifier tokenization.

---
 JSImpl/src/Lexer.cpp | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

(limited to 'JSImpl/src/Lexer.cpp')

diff --git a/JSImpl/src/Lexer.cpp b/JSImpl/src/Lexer.cpp
index 797af26..d17f593 100644
--- a/JSImpl/src/Lexer.cpp
+++ b/JSImpl/src/Lexer.cpp
@@ -125,6 +125,19 @@ Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme)
 	return{ type, m_Line, lexeme };
 }
 
+namespace
+{
+inline bool IsValidIdentifierStartingChar(char c)
+{
+	return (c >= 'A' && c <= 'z') || c == '_';
+}
+
+inline bool IsValidIdentifierChar(char c)
+{
+	return IsValidIdentifierStartingChar(c) || (c >= '0' && c <= '9');
+}
+}
+
 Token Tokenizer::NextToken()
 {
 	if (m_Code[m_Current] == '\0')
@@ -192,7 +205,20 @@ Token Tokenizer::NextToken()
 		{
 			return Token{ word->second, m_Line, key };
 		}
+		// It's not a key word so we must revert current counter
+		m_Current = start;
+	}
 
+	// Identifier
+	if (IsValidIdentifierStartingChar(c))
+	{
+		auto start = m_Current;
+		m_Current++;
+		while (IsValidIdentifierChar(m_Code[m_Current]))
+		{
+			++m_Current;
+		}
+		return Token{ TokenType::Identifier, m_Line, IPLString(m_Code + start, m_Code + m_Current) };
 	}
 	return Token{ TokenType::Eof, m_Line, "" };
 }
\ No newline at end of file
-- 
cgit v1.3.1-23-gd82d