diff options
| Commit | 47e8c6f331525f01f150ba80a3557a18618b0cf5 (patch) | |
|---|---|---|
| Author | 2018-11-07 13:05:43 +0200 | |
| Committer | 2018-11-07 13:05:43 +0200 | |
| Files | 3fde3f4fc002e4d10c49436d85188d4f3b02e33f /homework/81709/JSCompiler.Core | |
| IPL-47e8c6f331525f01f150ba80a3557a18618b0cf5.tar IPL-47e8c6f331525f01f150ba80a3557a18618b0cf5.tar.gz IPL-47e8c6f331525f01f150ba80a3557a18618b0cf5.zip | ||
| Parent | 5a4403ef23e17a3da84f1ec9feb3ab294d23c198 (diff) | |
Homework 1 - C# Lexer and Syntax Highlighter for Javascript (#34)
* Initial lexer structure
* Added IInputStream interface for the Lexer and its base implementation in FileInputStream not implemented yet.
* Extend InputStream interface
* Implement FileInputStream not tested
* Lexer almost ready - comments need to be parsed.
InputStream ready
* Added some TODOs
* Add error token type and almost parse comments
* Parse multiline comments and fix string parsing to not include the quotes.
* Added Syntax Highlighter, added support for more js reserved words, removed error tokens to not break the actual word tokens, fixed comments and string token representation to include its special characters
* Initial lexer structure
* Added IInputStream interface for the Lexer and its base implementation in FileInputStream not implemented yet.
* Extend InputStream interface
* Implement FileInputStream not tested
* Lexer almost ready - comments need to be parsed.
InputStream ready
* Added some TODOs
* Add error token type and almost parse comments
* Parse multiline comments and fix string parsing to not include the quotes.
* Added Syntax Highlighter, added support for more js reserved words, removed error tokens to not break the actual word tokens, fixed comments and string token representation to include its special characters
* Improve project structure and add tests
* Fix physical path structure of the projects
* Write tests, fix tokenizer bugs
* Fix README format, fix AssemblyInfo name
* Escape slashes in readme
11 files changed, 868 insertions, 0 deletions
| -rw-r--r-- | homework/81709/JSCompiler.Core/BaseInputStream.cs | 91 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/IInputStream.cs | 17 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/JSCompiler.Core.csproj | 56 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/Lexer.cs | 276 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/NumToken.cs | 42 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/Properties/AssemblyInfo.cs | 36 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/ReservedStrings.cs | 82 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/Token.cs | 48 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/TokenType.cs | 134 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/Utils.cs | 41 | ||||
| -rw-r--r-- | homework/81709/JSCompiler.Core/WordToken.cs | 45 |
diff --git a/homework/81709/JSCompiler.Core/BaseInputStream.cs b/homework/81709/JSCompiler.Core/BaseInputStream.cs new file mode 100644 index 0000000..7145ce8 --- /dev/null +++ b/homework/81709/JSCompiler.Core/BaseInputStream.cs @@ -0,0 +1,91 @@ +using System; +using System.IO; +using System.Collections; +using System.Collections.Generic; + +namespace JSCompiler.Core +{ + public class BaseInputStream : IInputStream + { + private char[][] buffs; + + private int index; + + private int activeBuffer; + + private StreamReader reader; + + public BaseInputStream(Stream stream, int buffLength = 4096) + { + this.Line = 0; + this.index = 0; + this.activeBuffer = 0; + this.buffs = new char[2][]; + this.buffs[0] = new char[buffLength]; + this.buffs[1] = new char[buffLength]; + this.reader = new StreamReader(stream); + + ReadStream(buffs[0], buffLength); + } + + public BaseInputStream(string path, int buffLength = 4096) + : this(new FileStream(path, FileMode.Open), buffLength) { } + + public int Line { get; private set; } + + public void Close() + { + reader.Close(); + } + + public bool IsEndOfStream() + { + return Peek() == 0; + } + + public char Next() + { + if (IsEndOfStream()) throw new EndOfStreamException(); + + char cur = Peek(); + + if (cur == '\n') + { + Line++; + } + + index++; + + if (index >= buffs[activeBuffer].Length) + { + SwitchBuffers(); + } + + return cur; + } + + public char Peek() + { + return buffs[activeBuffer][index]; + } + + private void SwitchBuffers() + { + index = 0; + activeBuffer = activeBuffer == 0 ? 1 : 0; + ReadStream(buffs[activeBuffer], buffs[activeBuffer].Length); + } + + private int ReadStream(char[] buffer, int length) + { + int bytesRead = reader.Read(buffer, 0, length); + + if (bytesRead < length) + { + buffer[bytesRead] = (char) 0; + } + + return bytesRead; + } + } +} diff --git a/homework/81709/JSCompiler.Core/IInputStream.cs b/homework/81709/JSCompiler.Core/IInputStream.cs new file mode 100644 index 0000000..ced31d3 --- /dev/null +++ b/homework/81709/JSCompiler.Core/IInputStream.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public interface IInputStream + { + void Close(); + bool IsEndOfStream(); + char Next(); + char Peek(); + int Line { get; } + } +} diff --git a/homework/81709/JSCompiler.Core/JSCompiler.Core.csproj b/homework/81709/JSCompiler.Core/JSCompiler.Core.csproj new file mode 100644 index 0000000..7b10dd1 --- /dev/null +++ b/homework/81709/JSCompiler.Core/JSCompiler.Core.csproj @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> + <PropertyGroup> + <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> + <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> + <ProjectGuid>{C8DD3D07-6AE9-444B-8BA0-22DC633A9432}</ProjectGuid> + <OutputType>Library</OutputType> + <AppDesignerFolder>Properties</AppDesignerFolder> + <RootNamespace>JSCompiler.Core</RootNamespace> + <AssemblyName>JSCompiler.Core</AssemblyName> + <TargetFrameworkVersion>v4.7.1</TargetFrameworkVersion> + <FileAlignment>512</FileAlignment> + <Deterministic>true</Deterministic> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> + <DebugSymbols>true</DebugSymbols> + <DebugType>full</DebugType> + <Optimize>false</Optimize> + <OutputPath>bin\Debug\</OutputPath> + <DefineConstants>DEBUG;TRACE</DefineConstants> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> + <DebugType>pdbonly</DebugType> + <Optimize>true</Optimize> + <OutputPath>bin\Release\</OutputPath> + <DefineConstants>TRACE</DefineConstants> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <ItemGroup> + <Reference Include="System" /> + <Reference Include="System.Core" /> + <Reference Include="System.Xml.Linq" /> + <Reference Include="System.Data.DataSetExtensions" /> + <Reference Include="Microsoft.CSharp" /> + <Reference Include="System.Data" /> + <Reference Include="System.Net.Http" /> + <Reference Include="System.Xml" /> + </ItemGroup> + <ItemGroup> + <Compile Include="IInputStream.cs" /> + <Compile Include="BaseInputStream.cs" /> + <Compile Include="Lexer.cs" /> + <Compile Include="NumToken.cs" /> + <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="ReservedStrings.cs" /> + <Compile Include="Token.cs" /> + <Compile Include="TokenType.cs" /> + <Compile Include="Utils.cs" /> + <Compile Include="WordToken.cs" /> + </ItemGroup> + <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> +</Project>
\ No newline at end of file diff --git a/homework/81709/JSCompiler.Core/Lexer.cs b/homework/81709/JSCompiler.Core/Lexer.cs new file mode 100644 index 0000000..18df998 --- /dev/null +++ b/homework/81709/JSCompiler.Core/Lexer.cs @@ -0,0 +1,276 @@ +using JSCompiler.Core; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public class Lexer + { + public static readonly string UNCLOSED_MULTILINE_COMMENT = "Unclosed mulitline comment"; + + public static readonly string UNCLOSED_STRING = "Unclosed string"; + + public static readonly string UNCLOSED_STRING_NEWLINE = "Unclosed string newline"; + + public static readonly string UNIDENTIFIED_SYMBOL = "Unidentified symbol"; + + + private Dictionary<string, TokenType> keywords; + + private Dictionary<string, TokenType> otherReserved; + + private IInputStream stream; + + public Lexer(IInputStream stream) + { + this.stream = stream; + this.keywords = ReservedStrings.KeywordsAsMap(); + this.otherReserved = ReservedStrings.OthersAsMap(); + } + + public List<Token> Tokenize(Action<int, string> onError) + { + List<Token> tokens = new List<Token>(); + do + { + Token next = NextToken(onError); + if(next.Type != TokenType.Skip) + { + tokens.Add(next); + } + } while (!stream.IsEndOfStream()); + tokens.Add(new Token(TokenType.EOF)); + return tokens; + } + + public Token NextToken(Action<int, string> onError) + { + char cur = stream.Next(); + + if (Utils.IsCarriageReturn(cur)) + return new Token(TokenType.Skip); + + if (Utils.IsWhiteSpace(cur)) + return new Token(TokenType.Whitespace); + + if (Utils.IsTab(cur)) + return new Token(TokenType.Tab); + + if (Utils.IsNewLine(cur)) + return new Token(TokenType.NewLine); + + switch (cur) + { + case '=': + return Match('=') ? Match('=') ? new WordToken("===", TokenType.StrictEqual) + : new WordToken("==", TokenType.EqualEqual) + : new Token(TokenType.Equal); + case '!': + return Match('=') ? Match('=') ? new WordToken("!==", TokenType.StrictNotEqual) + : new WordToken("!=", TokenType.NotEqual) + : new Token(TokenType.Bang); + case '>': + return Match('=') ? new WordToken(">=", TokenType.GreaterEqual) + : Match('>') ? Match('=') ? new WordToken(">>=", TokenType.RightShiftEqual) + : new WordToken(">>", TokenType.RightShift) + : new Token(TokenType.Greater); + case '<': + return Match('=') ? new WordToken("<=", TokenType.LessEqual) + : Match('<') ? Match('=') ? new WordToken("<<=", TokenType.LeftShiftEqual) + : new WordToken("<<", TokenType.LeftShift) + : new Token(TokenType.Less); + case '+': + return Match('+') ? new WordToken("++", TokenType.PlusPlus) + : Match('=') ? new WordToken("+=", TokenType.PlusEqual) + : new Token(TokenType.Plus); + case '-': + return Match('-') ? new WordToken("--", TokenType.MinusMinus) + : Match('=') ? new WordToken("-=", TokenType.MinusEqual) + : new Token(TokenType.Minus); + case '*': + return Match('=') ? new WordToken("*=", TokenType.StarEqual) + : new Token(TokenType.Star); + case '/': + return Match('/') ? ParseOneLineComment() + : Match('*') ? ParseMultilineToken(onError) + : Match('=') ? new WordToken("/=", TokenType.DivideEqual) + : new Token(TokenType.Division); + case '%': + return Match('=') ? new WordToken("%=", TokenType.ModuloEqual) + : new Token(TokenType.Modulo); + case '^': + return Match('=') ? new WordToken("^=", TokenType.BitwiseXorEqual) + : new Token(TokenType.BitwiseXor); + case '|': + return Match('=') ? new WordToken("|=", TokenType.BitwiseOrEqual) + : Match('|') ? new WordToken("||", TokenType.LogicalOr) + : new Token(TokenType.BitwiseOr); + case '&': + return Match('=') ? new WordToken("&=", TokenType.BitwiseAndEqual) + : Match('&') ? new WordToken("&&", TokenType.LogicalAnd) + : new Token(TokenType.BitwiseAnd); + default: + break; + } + + if (cur < 127 && Enum.IsDefined(typeof(TokenType), (int)cur)) + return new Token((TokenType)cur); + + if(Utils.IsStringSingleQuote(cur) || Utils.IsStringDoubleQuote(cur)) + { + return ParseSingleQuoteString(cur, onError); + } + + if (char.IsDigit(cur)) + { + return ParseNumber(cur); + } + + if (char.IsLetter(cur)) + { + return ParseReservedOrIdentifier(cur); + } + + onError(stream.Line, UNIDENTIFIED_SYMBOL); + return new WordToken(cur.ToString(), TokenType.Error); + } + + private bool Match(char ch) + { + if(stream.Peek() == ch) + { + stream.Next(); + return true; + } + return false; + } + + private Token ParseOneLineComment() + { + string lexeme = "//"; + char peek = stream.Peek(); + while(!Utils.IsNewLine(peek) && !Utils.IsCarriageReturn(peek) && !stream.IsEndOfStream()) + { + lexeme += stream.Next(); + peek = stream.Peek(); + } + return new WordToken(lexeme, TokenType.Comment); + } + + private Token ParseMultilineToken(Action<int, string> onError) + { + string lexeme = "/*"; + int lineBeg = stream.Line + 1; + while (!stream.IsEndOfStream()) + { + while (stream.Peek() != '*') + { + if (stream.IsEndOfStream()) + { + onError(lineBeg, UNCLOSED_MULTILINE_COMMENT); + return new WordToken(lexeme, TokenType.Comment); + } + + lexeme += stream.Next(); + }; + lexeme += stream.Next(); + if (stream.Peek() == '/') + { + lexeme += stream.Next(); + return new WordToken(lexeme, TokenType.Comment); + } + } + onError(lineBeg, UNCLOSED_MULTILINE_COMMENT); + return new WordToken(lexeme, TokenType.Comment); + } + + private Token ParseSingleQuoteString(char quote, Action<int, string> onError) + { + string lexeme = "" + quote; + int startLine = stream.Line + 1; + char peek = stream.Peek(); + while(peek != quote) + { + if (Utils.IsNewLine(peek) || Utils.IsCarriageReturn(peek)) + { + onError(startLine, UNCLOSED_STRING_NEWLINE); + return new WordToken(lexeme, TokenType.Error); + } + + if (stream.IsEndOfStream()) + { + onError(startLine, UNCLOSED_STRING); + return new WordToken(lexeme, TokenType.Error); + } + + lexeme += peek; + stream.Next(); + + if(peek == '\\') + { + char next = stream.Peek(); + + if (next == quote || next == '\\') + { + lexeme += next; + stream.Next(); + } + else + { + if (Utils.IsCarriageReturn(next)) //Skip Carriage return + { + lexeme += stream.Next(); + next = stream.Peek(); + } + if (Utils.IsNewLine(next)) + { + lexeme += stream.Next(); + } + } + } + + peek = stream.Peek(); + } + + lexeme += stream.Next(); + return new WordToken(lexeme, TokenType.String); + } + + private Token ParseNumber(char cur) + { + long num = cur - '0'; + char peek = stream.Peek(); + while (char.IsDigit(peek) && !stream.IsEndOfStream()) + { + num = num * 10 + (stream.Next() - '0'); + peek = stream.Peek(); + } + return new NumToken(num); + } + + private Token ParseReservedOrIdentifier(char ch) + { + string lexeme = "" + ch; + + while (char.IsLetterOrDigit(stream.Peek()) && !stream.IsEndOfStream()) + { + lexeme += stream.Next(); + } + + TokenType reservedType; + + keywords.TryGetValue(lexeme, out reservedType); + if(reservedType != 0) + return new WordToken(lexeme, reservedType); + + otherReserved.TryGetValue(lexeme, out reservedType); + if(reservedType != 0) + return new WordToken(lexeme, reservedType); + + return new WordToken(lexeme, TokenType.Identifier); + } + } +} diff --git a/homework/81709/JSCompiler.Core/NumToken.cs b/homework/81709/JSCompiler.Core/NumToken.cs new file mode 100644 index 0000000..53af034 --- /dev/null +++ b/homework/81709/JSCompiler.Core/NumToken.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public class NumToken : Token, IEquatable<NumToken> + { + public NumToken(long num) : base(TokenType.Number) + { + Num = num; + } + + public long Num { get; private set; } + + public bool Equals(NumToken other) + { + return other.Num == Num && base.Equals((Token)other); + } + + public override bool Equals(object obj) + { + var item = obj as NumToken; + + if (item == null) return false; + + return Equals(item); + } + + public override int GetHashCode() + { + return Num.GetHashCode() * 17 + base.GetHashCode(); + } + + public override string ToString() + { + return Num.ToString(); + } + } +} diff --git a/homework/81709/JSCompiler.Core/Properties/AssemblyInfo.cs b/homework/81709/JSCompiler.Core/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..ead40a3 --- /dev/null +++ b/homework/81709/JSCompiler.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("JSCompiler.Core")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("JSCompiler.Core")] +[assembly: AssemblyCopyright("Copyright © 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("c8dd3d07-6ae9-444b-8ba0-22dc633a9432")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/homework/81709/JSCompiler.Core/ReservedStrings.cs b/homework/81709/JSCompiler.Core/ReservedStrings.cs new file mode 100644 index 0000000..261e444 --- /dev/null +++ b/homework/81709/JSCompiler.Core/ReservedStrings.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public static class ReservedStrings + { + public static Dictionary<string, TokenType> KeywordsAsMap() + { + return new Dictionary<string, TokenType> + { + {"break", TokenType.Break }, + {"case", TokenType.Case }, + {"catch", TokenType.Catch }, + {"class", TokenType.Class }, + {"const", TokenType.Const }, + {"continue", TokenType.Continue }, + {"debugger", TokenType.Debugger }, + {"default", TokenType.Default }, + {"delete", TokenType.Delete }, + {"do" , TokenType.Do }, + {"else", TokenType.Else }, + {"export", TokenType.Export }, + {"extends", TokenType.Extends }, + {"finally", TokenType.Finally }, + {"for", TokenType.For }, + {"function", TokenType.Function }, + {"if", TokenType.If }, + {"import", TokenType.Import }, + {"in", TokenType.In }, + {"instance", TokenType.Instanceof }, + {"new", TokenType.New }, + {"return", TokenType.Return }, + {"super", TokenType.Super }, + {"switch", TokenType.Switch }, + {"this", TokenType.This }, + {"throw", TokenType.Throw }, + {"try", TokenType.Try }, + {"typeof", TokenType.Typeof }, + {"var", TokenType.Var }, + {"let", TokenType.Let }, + {"void", TokenType.Void }, + {"while", TokenType.While }, + {"with", TokenType.With }, + {"yield", TokenType.Yield }, + {"null", TokenType.Null }, + {"true", TokenType.True }, + { "false", TokenType.False } + }; + } + + public static Dictionary<string, TokenType> OthersAsMap() + { + return new Dictionary<string, TokenType> + { + {"Array", TokenType.Array }, + {"Date", TokenType.Date }, + {"eval", TokenType.Eval }, + {"hasOwnProperty", TokenType.HasOwnProperty }, + {"Infinity", TokenType.Infinity }, + {"isFinite", TokenType.IsFinite }, + {"isNaN", TokenType.IsNaN }, + {"isPrototypeOf", TokenType.IsPrototypeOf }, + {"length", TokenType.Length }, + {"Math" , TokenType.Math }, + {"NaN", TokenType.NaN }, + {"name", TokenType.Name }, + {"String", TokenType.StringObj }, + {"Number", TokenType.NumberObj }, + {"toString", TokenType.ToString }, + {"Object", TokenType.Object }, + {"undefined", TokenType.Undefined }, + {"prototype", TokenType.Prototype }, + {"valueOf", TokenType.ValueOf } + }; + } + + } +} diff --git a/homework/81709/JSCompiler.Core/Token.cs b/homework/81709/JSCompiler.Core/Token.cs new file mode 100644 index 0000000..efa2dd8 --- /dev/null +++ b/homework/81709/JSCompiler.Core/Token.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public class Token : IEquatable<Token> + { + public Token(TokenType type) + { + Type = type; + } + + public TokenType Type { get; private set; } + + public override bool Equals(object obj) + { + var item = obj as Token; + + if(item == null) + { + return false; + } + + return Equals(item); + } + + public bool Equals(Token other) + { + return Type == other.Type; + } + + public override int GetHashCode() + { + return this.Type.GetHashCode(); + } + + public override string ToString() + { + return ((char)Type).ToString(); + } + } + + + +} diff --git a/homework/81709/JSCompiler.Core/TokenType.cs b/homework/81709/JSCompiler.Core/TokenType.cs new file mode 100644 index 0000000..a65f7a2 --- /dev/null +++ b/homework/81709/JSCompiler.Core/TokenType.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public enum TokenType + { + // Single-character token types mapped to ASCII codes + LeftParen = 40, // ( + RightParen = 41, // ) + LeftBrace = 123, // { + RightBrace = 125, // } + Comma = 44, // , + Dot = 46, // . + Minus = 45, // - + Plus = 43, // + + Semicolon = 59, // ; + Star = 42, // * + Division = 47, // / + Modulo = 37, // % + BitwiseNot = 126, // ~ + BitwiseAnd = 38, // & + BitwiseXor = 94, // ^ + BitwiseOr = 124, // | + QuestionMark = 63, // ? + Colon = 58, // : + LeftSquareBracket = 91, // [ + RightSquareBracket = 93, // ] + Bang = 33, // ! + Equal = 61, // = + Greater = 62, // > + Less = 60, // < + Backslash = 92, // \ + + Invalid = 127, + NewLine, + Tab, + Whitespace, + Comment, + EOF, + Skip, + + NotEqual, // != + EqualEqual, // == + StrictEqual, // === + StrictNotEqual, // !== + GreaterEqual, // >= + LessEqual, // <= + MinusMinus, // -- + PlusPlus, // ++ + LeftShift, // << + RightShift, // >> + LogicalAnd, // && + LogicalOr, // || + StarEqual, // *= + DivideEqual, // /= + ModuloEqual, // %= + PlusEqual, // += + MinusEqual, // -= + LeftShiftEqual, // <<= + RightShiftEqual, // >>= + BitwiseAndEqual, // &= + BitwiseXorEqual, // ^= + BitwiseOrEqual, // |= + + Identifier, + String, + Number, + Error, + + // Keywords. ECMAScript 2015 + Break, + Case, + Catch, + Class, + Const, + Continue, + Debugger, + Default, + Delete, + Do, + Else, + Export, + Extends, + Finally, + For, + Function, + If, + Import, + In, + Instanceof, + New, + Return, + Super, + Switch, + This, + Throw, + Try, + Typeof, + Var, + Let, + Void, + While, + With, + Yield, + Null, + True, + False, + + //Objects and other + Array, + Date, + Eval, + HasOwnProperty, + Infinity, + IsFinite, + IsNaN, + IsPrototypeOf, + Length, + Math, + NaN, + Name, + NumberObj, + Object, + Prototype, + StringObj, + ToString, + Undefined, + ValueOf + } +} diff --git a/homework/81709/JSCompiler.Core/Utils.cs b/homework/81709/JSCompiler.Core/Utils.cs new file mode 100644 index 0000000..17210bc --- /dev/null +++ b/homework/81709/JSCompiler.Core/Utils.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public static class Utils + { + public static bool IsWhiteSpace(char ch) + { + return ch == ' '; + } + + public static bool IsCarriageReturn(char ch) + { + return ch == '\r'; + } + + public static bool IsNewLine(char ch) + { + return ch == '\n'; + } + + public static bool IsTab(char ch) + { + return ch == '\t'; + } + + public static bool IsStringSingleQuote(char ch) + { + return ch == '\''; + } + + public static bool IsStringDoubleQuote(char ch) + { + return ch == '"'; + } + } +} diff --git a/homework/81709/JSCompiler.Core/WordToken.cs b/homework/81709/JSCompiler.Core/WordToken.cs new file mode 100644 index 0000000..cee419e --- /dev/null +++ b/homework/81709/JSCompiler.Core/WordToken.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JSCompiler.Core +{ + public class WordToken : Token, IEquatable<WordToken> + { + public WordToken(string lexeme, TokenType type) : base(type) + { + Lexeme = lexeme; + } + + public string Lexeme { get; private set; } + + public override bool Equals(object obj) + { + var item = obj as WordToken; + + if(item == null) + { + return false; + } + + return Equals(item); + } + + public bool Equals(WordToken token) + { + return base.Equals((Token) token) && string.Equals(Lexeme, token.Lexeme); + } + + public override int GetHashCode() + { + return Lexeme.GetHashCode() * 17 + base.GetHashCode(); + } + + public override string ToString() + { + return Lexeme; + } + } +} |
