Create solution for sample vm implementation.

Implement basic Lexer.

7 files changed, 533 insertions, 0 deletions

diff --git a/JSImpl/solution/JSImpl.sln b/JSImpl/solution/JSImpl.sln
new file mode 100644
index 0000000..bee6a3e
--- /dev/null
+++ b/JSImpl/solution/JSImpl.sln
@@ -0,0 +1,28 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25420.1
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "JSImpl", "JSImpl.vcxproj", "{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Debug|x64.ActiveCfg = Debug|x64
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Debug|x64.Build.0 = Debug|x64
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Debug|x86.ActiveCfg = Debug|Win32
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Debug|x86.Build.0 = Debug|Win32
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Release|x64.ActiveCfg = Release|x64
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Release|x64.Build.0 = Release|x64
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Release|x86.ActiveCfg = Release|Win32
+		{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}.Release|x86.Build.0 = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
diff --git a/JSImpl/solution/JSImpl.vcxproj b/JSImpl/solution/JSImpl.vcxproj
new file mode 100644
index 0000000..6da736c
--- /dev/null
+++ b/JSImpl/solution/JSImpl.vcxproj
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{EE0F8C6F-3619-4A46-BC19-FAEAD4167EE4}</ProjectGuid>
+    <RootNamespace>JSImpl</RootNamespace>
+    <WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup />
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+    <Link>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+    <Link>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="..\src\CommonTypes.h" />
+    <ClInclude Include="..\src\Lexer.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\src\Lexer.cpp" />
+    <ClCompile Include="..\src\main.cpp" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/JSImpl/solution/JSImpl.vcxproj.filters b/JSImpl/solution/JSImpl.vcxproj.filters
new file mode 100644
index 0000000..238e847
--- /dev/null
+++ b/JSImpl/solution/JSImpl.vcxproj.filters
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\src\CommonTypes.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="..\src\Lexer.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\src\Lexer.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\src\main.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+</Project>
\ No newline at end of file
diff --git a/JSImpl/src/CommonTypes.h b/JSImpl/src/CommonTypes.h
new file mode 100644
index 0000000..b3ceb0b
--- /dev/null
+++ b/JSImpl/src/CommonTypes.h
@@ -0,0 +1,11 @@
+#pragma once
+#include <string>
+#include <vector>
+#include <unordered_map>
+
+template<typename T>
+using IPLVector = std::vector<T>;
+using IPLString = std::string;
+
+template <typename Key, typename T>
+using IPLUnorderedMap = std::unordered_map<Key, T>;
diff --git a/JSImpl/src/Lexer.cpp b/JSImpl/src/Lexer.cpp
new file mode 100644
index 0000000..797af26
--- /dev/null
+++ b/JSImpl/src/Lexer.cpp
@@ -0,0 +1,198 @@
+#include "Lexer.h"
+
+class Tokenizer
+{
+public:
+	Tokenizer(const char* code, const std::function<void()>& onError);
+	IPLVector<Token> Tokenize();
+
+private:
+	Token NextToken();
+
+	Token ProduceToken(TokenType type);
+	Token ProduceToken(TokenType type, IPLString lexeme);
+	double ParseNumber();
+	IPLString ParseString();
+	bool SkipWhiteSpaces();
+	bool SkipNewLine();
+	bool Match(const char c);
+
+	std::function<void()> OnError;
+	unsigned m_Line;
+	unsigned m_Current;
+	const char* m_Code;
+
+	IPLUnorderedMap<IPLString, TokenType> m_KeyWordsTable;
+};
+
+IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError)
+{
+	Tokenizer tokenizer(code, onError);
+
+	return tokenizer.Tokenize();
+}
+
+Tokenizer::Tokenizer(const char* code, const std::function<void()>& onError)
+	: OnError(onError)
+	, m_Line(0)
+	, m_Current(0)
+	, m_Code(code)
+{
+	m_KeyWordsTable["break"] = TokenType::Break;
+	m_KeyWordsTable["case"] = TokenType::Case;
+	m_KeyWordsTable["catch"] = TokenType::Catch;
+	m_KeyWordsTable["class"] = TokenType::Class;
+	m_KeyWordsTable["const"] = TokenType::Const;
+	m_KeyWordsTable["continue"] = TokenType::Continue;
+	m_KeyWordsTable["debugger"] = TokenType::Debugger;
+	m_KeyWordsTable["default"] = TokenType::Default;
+	m_KeyWordsTable["delete"] = TokenType::Delete;
+	m_KeyWordsTable["do"] = TokenType::Do;
+	m_KeyWordsTable["else"] = TokenType::Else;
+	m_KeyWordsTable["export"] = TokenType::Export;
+	m_KeyWordsTable["extends"] = TokenType::Extends;
+	m_KeyWordsTable["finally"] = TokenType::Finally;
+	m_KeyWordsTable["for"] = TokenType::For;
+	m_KeyWordsTable["function"] = TokenType::Function;
+	m_KeyWordsTable["if"] = TokenType::If;
+	m_KeyWordsTable["import"] = TokenType::Import;
+	m_KeyWordsTable["in"] = TokenType::In;
+	m_KeyWordsTable["instanceof"] = TokenType::Instanceof;
+	m_KeyWordsTable["new"] = TokenType::New;
+	m_KeyWordsTable["return"] = TokenType::Return;
+	m_KeyWordsTable["super"] = TokenType::Super;
+	m_KeyWordsTable["switch"] = TokenType::Switch;
+	m_KeyWordsTable["this"] = TokenType::This;
+	m_KeyWordsTable["throw"] = TokenType::Throw;
+	m_KeyWordsTable["try"] = TokenType::Try;
+	m_KeyWordsTable["typeof"] = TokenType::Typeof;
+	m_KeyWordsTable["var"] = TokenType::Var;
+	m_KeyWordsTable["void"] = TokenType::Void;
+	m_KeyWordsTable["while"] = TokenType::While;
+	m_KeyWordsTable["with"] = TokenType::With;
+	m_KeyWordsTable["yield"] = TokenType::Yield;
+}
+
+IPLVector<Token> Tokenizer::Tokenize()
+{
+	IPLVector<Token> result;
+	do
+	{
+		result.push_back(NextToken());
+	} while (result.back().Type != TokenType::Eof);
+
+	return result;
+}
+
+bool Tokenizer::SkipWhiteSpaces()
+{
+	const auto old = m_Current;
+	while (m_Code[m_Current] == ' ' || m_Code[m_Current] == '\t')
+	{
+		m_Current++;
+	}
+	return old != m_Current;
+}
+
+bool Tokenizer::SkipNewLine()
+{
+	const auto old = m_Current;
+	while (m_Code[m_Current] == '\n')
+	{
+		m_Current++;
+		m_Line++;
+	}
+	return old != m_Current;
+}
+
+bool Tokenizer::Match(const char c)
+{
+	if (c == m_Code[m_Current])
+	{
+		++m_Current;
+		return true;
+	}
+	return false;
+}
+
+Token Tokenizer::ProduceToken(TokenType type)
+{
+	return { type, m_Line, "" };
+}
+
+Token Tokenizer::ProduceToken(TokenType type, IPLString lexeme)
+{
+	return{ type, m_Line, lexeme };
+}
+
+Token Tokenizer::NextToken()
+{
+	if (m_Code[m_Current] == '\0')
+	{
+		return Token{ TokenType::Eof, 0, "" };
+	}
+
+	while (SkipWhiteSpaces() || SkipNewLine());
+
+	// Single Char
+	switch (m_Code[m_Current])
+	{
+	case '(': m_Current++; return ProduceToken(TokenType::LeftParen);
+	case ')': m_Current++; return ProduceToken(TokenType::RightParen);
+	case '{': m_Current++; return ProduceToken(TokenType::LeftBrace);
+	case '}': m_Current++; return ProduceToken(TokenType::RightBrace);
+	case ',': m_Current++; return ProduceToken(TokenType::Comma);
+	case '.': m_Current++; return ProduceToken(TokenType::Dot);
+	case '-': m_Current++; return ProduceToken(TokenType::Minums);
+	case '+': m_Current++; return ProduceToken(TokenType::Plus);
+	case ';': m_Current++; return ProduceToken(TokenType::Semicolon);
+	case '*': m_Current++; return ProduceToken(TokenType::Star);
+	case '=': m_Current++; return Match('=') ? ProduceToken(TokenType::EqualEqual) : ProduceToken(TokenType::Equal);
+	case '!': m_Current++; return Match('=') ? ProduceToken(TokenType::BangEqual) : ProduceToken(TokenType::Bang);
+	case '>': m_Current++; return Match('=') ? ProduceToken(TokenType::GreaterEqual) : ProduceToken(TokenType::Greater);
+	case '<': m_Current++; return Match('=') ? ProduceToken(TokenType::LessEqual) : ProduceToken(TokenType::Less);
+	default:
+		break;
+	}
+	// Number
+	const char c = m_Code[m_Current];
+	if (c >= '0' && c < '9')
+	{
+		size_t parsedBytes = 0;
+		const double number = std::stod(m_Code + m_Current, &parsedBytes);
+		m_Current += (unsigned)parsedBytes;
+		return Token{ TokenType::Number, m_Line, "", number };
+	}
+
+	// String
+	if (c == '"')
+	{
+		auto start = m_Current;
+		m_Current++; // skip first '"'
+		while (m_Code[m_Current] != '"')
+		{
+			++m_Current;
+		}
+		m_Current++; // skip second '"'
+		return Token{ TokenType::String, m_Line, IPLString(m_Code + start, m_Code + m_Current) };
+	}
+
+	// Key words
+	if (c >= 'a' && c <= 'z')
+	{
+		auto start = m_Current;
+		m_Current++;
+		while (m_Code[m_Current] >= 'a' && m_Code[m_Current] <= 'z')
+		{
+			++m_Current;
+		}
+		auto key = IPLString(m_Code + start, m_Code + m_Current);
+		auto word = m_KeyWordsTable.find(key);
+		if (word != m_KeyWordsTable.end())
+		{
+			return Token{ word->second, m_Line, key };
+		}
+
+	}
+	return Token{ TokenType::Eof, m_Line, "" };
+}
\ No newline at end of file
diff --git a/JSImpl/src/Lexer.h b/JSImpl/src/Lexer.h
new file mode 100644
index 0000000..6090759
--- /dev/null
+++ b/JSImpl/src/Lexer.h
@@ -0,0 +1,81 @@
+#pragma once
+
+#include "CommonTypes.h"
+#include <functional>
+
+enum TokenType
+{
+	// Single-character tokens.
+	LeftParen,
+	RightParen,
+	LeftBrace,
+	RightBrace,
+	Comma,
+	Dot,
+	Minums,
+	Plus,
+	Semicolon,
+	Slash,
+	Star,
+
+	// One or two character tokens
+	Bang,
+	BangEqual,
+	Equal,
+	EqualEqual,
+	Greater,
+	GreaterEqual,
+	Less,
+	LessEqual,
+
+	Identifier,
+	String,
+	Number,
+
+	// Keywords.
+	Break,
+	Case,
+	Catch,
+	Class,
+	Const,
+	Continue,
+	Debugger,
+	Default,
+	Delete,
+	Do,
+	Else,
+	Export,
+	Extends,
+	Finally,
+	For,
+	Function,
+	If,
+	Import,
+	In,
+	Instanceof,
+	New,
+	Return,
+	Super,
+	Switch,
+	This,
+	Throw,
+	Try,
+	Typeof,
+	Var,
+	Void,
+	While,
+	With,
+	Yield,
+
+	Eof,
+};
+
+struct Token
+{
+	TokenType Type;
+	unsigned Line;
+	IPLString Lexeme;
+	double Number;
+};
+
+IPLVector<Token> Tokenize(const char* code, const std::function<void()>& onError = {});
diff --git a/JSImpl/src/main.cpp b/JSImpl/src/main.cpp
new file mode 100644
index 0000000..8452e72
--- /dev/null
+++ b/JSImpl/src/main.cpp
@@ -0,0 +1,63 @@
+#include "Lexer.h"
+#include <iostream>
+#define LOG(msg) std::cout << msg << std::endl
+void TestLess()
+{
+	auto tokens = Tokenize("<");
+	if (tokens.size() == 2 && tokens[0].Type == TokenType::Less)
+	{
+		LOG("PASS");
+	}
+	else
+	{
+		LOG("FAIL");
+	}
+}
+
+void TestNumber()
+{
+	auto tokens = Tokenize("213434.24");
+	if (tokens.size() == 2 && tokens[0].Type == TokenType::Number && tokens[0].Number == 213434.24)
+	{
+		LOG("PASS");
+	}
+	else
+	{
+		LOG("FAIL");
+	}
+}
+
+void TestString()
+{
+	auto tokens = Tokenize("\"alabala\"");
+	if (tokens.size() == 2 && tokens[0].Type == TokenType::String && tokens[0].Lexeme == "\"alabala\"")
+	{
+		LOG("PASS");
+	}
+	else
+	{
+		LOG("FAIL");
+	}
+}
+
+void TestKeyWord()
+{
+	auto tokens = Tokenize("for");
+	if (tokens.size() == 2 && tokens[0].Type == TokenType::For)
+	{
+		LOG("PASS");
+	}
+	else
+	{
+		LOG("FAIL");
+	}
+}
+
+int main()
+{
+	TestLess();
+	TestNumber();
+	TestString();
+	TestKeyWord();
+	return 0;
+}
\ No newline at end of file