From b0798cf5d8e83f7211409369fbbdd51531364a2b Mon Sep 17 00:00:00 2001 From: Syndamia Date: Wed, 20 Aug 2025 06:18:16 +0300 Subject: feat(blog/sane-eplanation-of-compilers): Rework part 1's code blocks and folder structure Mostly moved a lot of code into distinct files, which are included in our markup. --- src/blog.du | 2 +- src/blog/index.du | 2 +- .../img/tree-building-1.svg | 26 - .../img/tree-building-2.svg | 53 - .../img/tree-building-3.svg | 53 - .../img/tree-building-ex1.svg | 39 - .../img/tree-building-ex2.svg | 67 - .../img/tree-building-ex3.svg | 53 - .../img/tree-building-ex4.svg | 81 - .../img/tree-cycle.svg | 240 --- .../sane-explanation-of-compilers/img/tree-if.svg | 179 -- .../img/tree-simple.svg | 81 - src/blog/sane-explanation-of-compilers/index.du | 9 +- src/blog/sane-explanation-of-compilers/part-1.du | 1364 --------------- .../part-1/example_lex/lexer.l | 9 + .../part-1/example_lex/numbers_html.l | 19 + .../part-1/example_yacc/lexer.l | 13 + .../part-1/example_yacc/parser.y | 24 + .../part-1/grammar_trace_1.txt | 26 + .../part-1/grammar_trace_2.txt | 38 + .../part-1/img/tree-building-1.svg | 26 + .../part-1/img/tree-building-2.svg | 53 + .../part-1/img/tree-building-3.svg | 53 + .../part-1/img/tree-building-ex1.svg | 39 + .../part-1/img/tree-building-ex2.svg | 67 + .../part-1/img/tree-building-ex3.svg | 53 + .../part-1/img/tree-building-ex4.svg | 81 + .../part-1/img/tree-cycle.svg | 240 +++ .../part-1/img/tree-if.svg | 179 ++ .../part-1/img/tree-simple.svg | 81 + .../sane-explanation-of-compilers/part-1/index.du | 1071 ++++++++++++ .../part-1/proper/lex.yy.c | 1771 ++++++++++++++++++++ .../part-1/proper/lexer.l | 18 + .../part-1/proper/parser.tab.c | 1298 ++++++++++++++ .../part-1/proper/parser.tab.h | 90 + .../part-1/proper/parser.y | 45 + .../part-1/proper/test.txt | 1 + .../part-1/proper/test_output.txt | 9 + .../part-1/proper/tree.c | 24 + .../part-1/proper/tree.h | 17 + .../part-1/regex_parser_simple.cpp | 16 + .../part-1/template.l | 13 + .../part-1/template.y | 13 + 43 files changed, 5396 insertions(+), 2240 deletions(-) delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-1.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-2.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-3.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-ex1.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-ex2.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-ex3.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-building-ex4.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-cycle.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-if.svg delete mode 100644 src/blog/sane-explanation-of-compilers/img/tree-simple.svg delete mode 100644 src/blog/sane-explanation-of-compilers/part-1.du create mode 100644 src/blog/sane-explanation-of-compilers/part-1/example_lex/lexer.l create mode 100644 src/blog/sane-explanation-of-compilers/part-1/example_lex/numbers_html.l create mode 100644 src/blog/sane-explanation-of-compilers/part-1/example_yacc/lexer.l create mode 100644 src/blog/sane-explanation-of-compilers/part-1/example_yacc/parser.y create mode 100644 src/blog/sane-explanation-of-compilers/part-1/grammar_trace_1.txt create mode 100644 src/blog/sane-explanation-of-compilers/part-1/grammar_trace_2.txt create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-1.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-2.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-3.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex1.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex2.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex3.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex4.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-cycle.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-if.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/img/tree-simple.svg create mode 100644 src/blog/sane-explanation-of-compilers/part-1/index.du create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/lex.yy.c create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/lexer.l create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.c create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.h create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/parser.y create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/test.txt create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/test_output.txt create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/tree.c create mode 100644 src/blog/sane-explanation-of-compilers/part-1/proper/tree.h create mode 100644 src/blog/sane-explanation-of-compilers/part-1/regex_parser_simple.cpp create mode 100644 src/blog/sane-explanation-of-compilers/part-1/template.l create mode 100644 src/blog/sane-explanation-of-compilers/part-1/template.y (limited to 'src') diff --git a/src/blog.du b/src/blog.du index 3ee201c..76b3929 100644 --- a/src/blog.du +++ b/src/blog.du @@ -50,7 +50,7 @@ Articles detailing personal experiences or exploration of certain topics. ; [$gb] 25 Apr   [url /blog/managing-apps-with-adb-without-root Managing Android apps with ADB (and without root)] : `[Android]` | `[ADB]` | `[Apps]` `[Linux]` `[Windows]` -; [$gb] 18 Apr   [url /blog/sane-explanation-of-compilers/part-1.html Sane explanation of compilers: Part 1] +; [$gb] 18 Apr   [url /blog/sane-explanation-of-compilers/part-1 Sane explanation of compilers: Part 1] : `[Deep dive]` | `[Compilers]` | `[C]` `[yacc]` `[bison]` `[math]` `[Trees]` `[Regex]` `[Grammar]` `[Languages]` ; [$gb] 13 Jan   [url /blog/plan9-in-linux-mouse-menus Plan 9 in Linux: Mouse menus] diff --git a/src/blog/index.du b/src/blog/index.du index 3ee201c..76b3929 100644 --- a/src/blog/index.du +++ b/src/blog/index.du @@ -50,7 +50,7 @@ Articles detailing personal experiences or exploration of certain topics. ; [$gb] 25 Apr   [url /blog/managing-apps-with-adb-without-root Managing Android apps with ADB (and without root)] : `[Android]` | `[ADB]` | `[Apps]` `[Linux]` `[Windows]` -; [$gb] 18 Apr   [url /blog/sane-explanation-of-compilers/part-1.html Sane explanation of compilers: Part 1] +; [$gb] 18 Apr   [url /blog/sane-explanation-of-compilers/part-1 Sane explanation of compilers: Part 1] : `[Deep dive]` | `[Compilers]` | `[C]` `[yacc]` `[bison]` `[math]` `[Trees]` `[Regex]` `[Grammar]` `[Languages]` ; [$gb] 13 Jan   [url /blog/plan9-in-linux-mouse-menus Plan 9 in Linux: Mouse menus] diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-1.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-1.svg deleted file mode 100644 index 174f003..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-1.svg +++ /dev/null @@ -1,26 +0,0 @@ - - #background: #fdf6e3 - [2] - - - - - - - - - - - - - - 2 - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-2.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-2.svg deleted file mode 100644 index 48e2636..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-2.svg +++ /dev/null @@ -1,53 +0,0 @@ - - #background: #fdf6e3 - [*]->[2] - [*]->[<abstract>...] - - - - - - - - - - - - - - - - - - * - - - - - - - - - - - 2 - - - - - - - - - - - ... - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-3.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-3.svg deleted file mode 100644 index ea526df..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-3.svg +++ /dev/null @@ -1,53 +0,0 @@ - - #background: #fdf6e3 - [*]->[2] - [*]->[()] - - - - - - - - - - - - - - - - - - * - - - - - - - - - - - 2 - - - - - - - - - - - () - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-ex1.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-ex1.svg deleted file mode 100644 index d5addd2..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-ex1.svg +++ /dev/null @@ -1,39 +0,0 @@ - - #background: #fdf6e3 - [()]->[<abstract>...] - - - - - - - - - - - - - - - - () - - - - - - - - - - - ... - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-ex2.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-ex2.svg deleted file mode 100644 index fcf84a1..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-ex2.svg +++ /dev/null @@ -1,67 +0,0 @@ - - #background: #fdf6e3 - [*]->[()] - [*]->[<abstract>....] - [()]->[<abstract>...] - - - - - - - - - - - - - - - - - - - - * - - - - - - - - - - - () - - - - - - - - - - - .... - - - - - - - - - - - ... - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-ex3.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-ex3.svg deleted file mode 100644 index d15267e..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-ex3.svg +++ /dev/null @@ -1,53 +0,0 @@ - - #background: #fdf6e3 - [+]->[1] - [+]->[2] - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - 1 - - - - - - - - - - - 2 - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-building-ex4.svg b/src/blog/sane-explanation-of-compilers/img/tree-building-ex4.svg deleted file mode 100644 index 7642a42..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-building-ex4.svg +++ /dev/null @@ -1,81 +0,0 @@ - - #background: #fdf6e3 - [+]->[1] - [+]->[*] - [*]->[2] - [*]->[<abstract>...] - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - 1 - - - - - - - - - - - * - - - - - - - - - - - 2 - - - - - - - - - - - ... - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-cycle.svg b/src/blog/sane-explanation-of-compilers/img/tree-cycle.svg deleted file mode 100644 index 9a7371c..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-cycle.svg +++ /dev/null @@ -1,240 +0,0 @@ - - - - - #background: #fdf6e3 - [A]->[B] - [B]->[C] - [C]->[A] - - - - - - - - - - - - - - - - - - - - A - - - - - - - - - - B - - - - - - - - - - C - - - - - - - - - - - diff --git a/src/blog/sane-explanation-of-compilers/img/tree-if.svg b/src/blog/sane-explanation-of-compilers/img/tree-if.svg deleted file mode 100644 index 06ddcf6..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-if.svg +++ /dev/null @@ -1,179 +0,0 @@ - - #background: #fdf6e3 - [if]->[>] - [>]->[a] - [>]->[b] - [if]->[body when true] - [body when true]->[=] - [=]->[c] - [=]->[5] - [if]->[body when false] - [body when false]->[+=] - [+=]->[d] - [+=]->[10] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if - - - - - - - - - - - > - - - - - - - - - - - a - - - - - - - - - - - b - - - - - - - - - - - body when true - - - - - - - - - - - = - - - - - - - - - - - c - - - - - - - - - - - 5 - - - - - - - - - - - body when false - - - - - - - - - - - += - - - - - - - - - - - d - - - - - - - - - - - 10 - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/img/tree-simple.svg b/src/blog/sane-explanation-of-compilers/img/tree-simple.svg deleted file mode 100644 index 9f978a7..0000000 --- a/src/blog/sane-explanation-of-compilers/img/tree-simple.svg +++ /dev/null @@ -1,81 +0,0 @@ - - #background: #fdf6e3 - [+]->[*] - [*]->[1] - [*]->[2] - [+]->[3] - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - * - - - - - - - - - - - 1 - - - - - - - - - - - 2 - - - - - - - - - - - 3 - - - - - - - - - \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/index.du b/src/blog/sane-explanation-of-compilers/index.du index 40c6cff..9c05e81 100644 --- a/src/blog/sane-explanation-of-compilers/index.du +++ b/src/blog/sane-explanation-of-compilers/index.du @@ -1,7 +1,12 @@ # Sane explanation of compilers -This blogpost is split into multiple parts. +'define expected-reading 48 min +'define created 30 March 2023 +'define edited 24 April 2023 +[$pagenav] + +This is a series on compilers, with the goal of explaining all topics as simply as possible. More information is available in the first part. .bulleted -- [url ./part-1.html Part 1]: Introduction and The frontend (aka How do we understand code) +- [url ./part-1 Part 1]: Introduction and The frontend (aka How do we understand code) diff --git a/src/blog/sane-explanation-of-compilers/part-1.du b/src/blog/sane-explanation-of-compilers/part-1.du deleted file mode 100644 index 33d8a1a..0000000 --- a/src/blog/sane-explanation-of-compilers/part-1.du +++ /dev/null @@ -1,1364 +0,0 @@ -# Sane explanation of compilers: Part 1 - -'define expected-reading 48 min -'define created 30 March 2023 -'define edited 24 April 2023 -[$pagenav] - -:contents -:numbered -1. [url #preface Preface] -2. [url #i.-how-do-we-understand-code I. How do we understand code?] - :numbered - 1. [url #math Math] - .numbered - 1. [url #regular-expressions Regular expressions] - 2. [url #grammars Grammars] - - 2. [url #lexers-and-parsers Lexers and parsers] - :numbered - 1. [url #implementational-difficulties-with-grammars Implementational difficulties with grammars] - 2. [url #practical-usage Practical usage] - 3. [url #parsing-algorithms Parsing algorithms] - .numbered - 1. [url #ll LL] - 2. [url #lr LR] - 3. [url #lrs-family LR's family] - 4. [url #more-info More info] - - 4. [url #abstract-syntax-tree Abstract syntax tree] - - - 3. [url #implementing-a-front-end Implementing a front-end] - .numbered - 1. [url #lex lex] - 2. [url #yacc yacc] - 3. [url #doing-it-properly Doing it properly] - - -3. [url #appendix Appendinx] - - - -Recently I signed up for a small university course about compilers. -As I found out, most materials on the topic are either so surface level you barely learn anything, or so specific you need to already know the subject. -My goal is to try and fill that void, explaining everything in a detailed way that is understandable for beginners. - -## Preface - -Processors work with numbers, every action (instruction) a processor can do is defined by numbers. -However, writing code only with numbers is tedious, error-prone and simple stuff sometimes take ages to implement. -That is why, over time, programmers have tried to abstract sets of instructions into words and text constructions that are easier to work with. - -'footnote how-did-we-get-here Though it's obvious, I would like to point out that modern compilers are most often written in languages that are themselves compiled. You might ask yourself "where did the first compiler come from?" The answer is simple: it was written with "numbers", with direct instructions to the processor, from the time when computers were simple enough that *[all]* programming was done with CPU instructions. - -Thanks to abstraction, programmers have been able to create more and more complicated projects with less and less work, that is more portable and compatible with different machines. -The trade-off is that we need to translate our nice programmer code to numbers a CPU would understand. -This job is delegated to a special type of program, called a compiler[$how-did-we-get-here], which is separated into three main layers/components: - -.numbered -1. Front-end: it understands the program, what the code is trying to do -2. Middle: does optimizations and simplifications, though this part is often inside the back-end -3. Back-end: generates the actual machine instructions - -In this post, we'll be looking only at the first one. - -## I. How do we understand code? - -Syntax in current programming languages can be quite sophisticated and ambiguous. For example (taken from [url https://en.wikipedia.org/wiki/Most_vexing_parse wikipedia]) this C++ snippet - -``` -void f(double d) { - int i(int(d)); -} - -could be interpreted either as a: - -.numbered -1. declaration of variable `[i]`, which is initialized with the value of `[int(d)]` (which just casts `[d]` to an integer), or -2. function declaration, since superfluous parenthesis around function parameter declarations are allowed (meaning it is treated as `[int i(int d)]`) - -Note that I'm talking about syntax, the valid selection and layouts of characters, and not semantics, the underlying meaning. -Compiled programming languages associate specific templates of code (text) to their "corresponding" machine instructions, there isn't anything special you need to understand in a piece of code. -In natural languages, you could construct a sentence which follows all grammatical rules, but is absolute nonsense, like [url https://en.wikipedia.org/wiki/Colorless_green_ideas_sleep_furiously "Colorless green ideas sleep furiously"]. - -The problem of semantics is extremely difficult, however in our case we only care about syntax. -And this is where math comes into play. - -### Math - -A word of caution, I'll use a decent amount of fancy names, so I have a way to more easily refer to different concepts. -Don't be scared and skip over them if they're confusing you. - -There is a special field in mathematics that deals with syntactical analysis: [url https://en.wikipedia.org/wiki/Formal_language formal language theory]. -Although even the basics merit a whole university course, I'll still try to roughly explain what's important, for those that haven't studied it.\n -This field works primarily with strings via the help of two (relevant for our purposes) instruments: regular expressions and grammars. - -#### Regular expressions - -You might've heard or even already used regexes in a text editor. -They're small text expressions which specify a text pattern. -When a word can be "generated" from the pattern, we say the regex matches the word. - -At their very base, such a pattern is created with the following rules: - -:unordered -- [url https://en.wikipedia.org/wiki/Concatenation concatenation] - you can put characters next to each other to form strings, so a normal string like `[accent]` is also a regex.\n - With braces we can group stuff. - For example, in `[a(cc)ent]` the two "c"s are grouped together. - This is used to form more sophisticated structures later on, but currently it doesn't alter anything, like the braces in `[(1+2)+3]` and `[1+(2+3)]` don't alter the result sum. - -- [url https://en.wikipedia.org/wiki/Alternation_%28formal_language_theory%29 alternation] - allow different (alternative) strings in the same spot.\n - We do so with vertical bars (which signify an "or" relationship), the group can be "replaced" with either value. - For example `[acce(n | p)t]`, matches both "acce=[n]=t" or "acce=[p]=t".\n - There is no limit on how many alternatives you can have and what their lengths are: - - ```regex - acc(ordingly | ent | urate) - - matches - - .bulleted - - "acc=[ordingly]=" - - "acc=[ent]=" and - - "acc=[urate]=". - - There is no limit on how many groups (with alternation) you can have: - - ```regex - ac(quirem | c | hievem)en(t | ts) - - matches - - .bulleted - - "ac=[quirem]=en=[t]=" - - "ac=[quirem]=en=[ts]=" - - "ac=[c]=en=[t]= - - "ac=[c]=en=[ts]= - - "ac=[hievem]=en=[t]=" and - - "ac=[hievem]=en=[ts]=". - -- [url https://en.wikipedia.org/wiki/Kleene_star Kleene star] - repeat the preceding letter or group, 0 or more times.\n - An asterisk is used to mark this operation (hence the name, "Kleene =[star]="). - - ```regex - accents* - - will match - - .bulleted - - "accent" - - "accent=[s]= - - "accent=[ss]= - - "accent=[sss]= - - and so on - - Of course, the asterisk could also be in the middle of the regex, like - - ```regex - ac*ent - - which matches - - .bulleted - - "acent" - - "ac=[c]=ent" - - "ac=[cc]=ent" - - ... - - With a group we can repeat entire strings, - - ```regex - acc(ent)*s - - matches - - .bulleted - - "accs" - - "acc=[ent]=s" - - "acc=[entent]=s - - "acc=[ententent]=s" - - ... - - An alternation allows any alternative to be chosen at any point, so - - ```regex - acce(n | p)*t - - matches - - .bulleted - - "accet" - - "acce=[n]=t - - "acce=[p]=t - - "acce=[nn]=t - - "acce=[np]=t" - - "acce=[pn]=t" - - "acce=[pp]=t" - - "acce=[nnn]=t" - - ... - - -In the modern day, regular expressions permit [url https://en.wikipedia.org/wiki/Regular_expression#Syntax additional constructions] like matching the start or end of a string, matching character groups or character classes. -There are also sometimes features which significantly increase the matching capabilities of normal regexes, like [url https://en.wikipedia.org/wiki/Regular_expression#Patterns_for_non-regular_languages lookahead and lookbehind assertions]. - -#### Grammars - -However, even then regular expressions are fairly weak, they can't match recursive structures. -Let's say you want to be able to have math expressions in your language, you can have braces inside braces as much as you like, but in a certain manner: - -.bulleted -- `[1+2]` is a valid expression -- `[(9-8)+2]` also is valid -- `[((3*3)-(2^3))+2]` is too -- however `[()3*3)-(2^3()+2]` isn't valid - -Such a structure is recursive, because it can contain "itself", `[1+2]` can be a standalone math expression, but it can also be contained in another expression, like `[(1+2)*3]`, which itself could be inside another bigger expressions and so on. -For this task we have a whole separate tool: grammars. - -'footnote terminals-and-nonterminals Formally, we call the rule "names" nonterminals, the idea being that all normal characters, that could appear in our string, are called terminals, since they would terminate replacement and string generation. For the purposes of this post, I won't use the formal names, with the hope that using less terminology will increase understandability. - -The syntax is different from regexes: a grammar is divided into (production) rules, each rule is named and specifies an expression. -Expressions are a concatenation of strings and/or names[$terminals-and-nonterminals]. -Names won't appear in our matched strings, the idea is that a name will be replaced with it's expression, kind of like a placeholders. -Additionally, there can be multiple rules with the same name, meaning a rule can be replaced by different stuff at any point of the string generation. - -'footnote grammar-syntax This is pretty much the mathematical notation, often in CS the Backus-Naur form is used, but I find that a bit harder to read. You might also make the argument that I should use a proper unicode arrow symbol, like `[→]`, but `[->]` is way more convinient. - -Always one name is chosen to be the starting point, and generation continues from there. -In math literature, generally we write rules in the form `[Name -> Expression]` [$grammar-syntax], and often names are a single capital letter. -Additionally, often the starting name is called `[S]`. -Finally, in future examples, I'll use the arrow `[⇒]` to show derivations, how the generated string looks after one or two replacements of the name with one of it's expressions. - -For an example, with the grammar - -```grammar -S -> aSb -S -> cc - -we can generate the string `[aaaaccbbbb]` (remember, our starting point is `[S]`): - -```grammar+ -S ⇒ aSb ⇒ aaSbb ⇒ aaaSbbb ⇒ aaaaSbbbb ⇒ aaaaccbbbb - -or `[accb]`: - -```grammar+ -S ⇒ aSb ⇒ accb - -where a, b and c are just English characters. -To simplify our lives, often rules with the same name are merged into one rule, separated by vertical bars (`[S -> aSb | cc]`). -This is practically alternation but in a different context. - -As for our math expressions, we could create something like this: - -```grammar -N -> 0 | 1 | 2 | ... all possible numbers -E -> N | (E) | E + E | E - E | E * E | E / E | E ^ E - -which will successfully be able to match (generate) `[((3 * 3) - (2 ^ 3)) + 2]`: - -```grammar+ -E ⇒ E + E ⇒ (E) + N ⇒ (E - E) + 2 ⇒ ((E) - (E)) + 2 ⇒ ((E * E) - (E ^ E)) + 2 ⇒ ((N * N) - (N ^ N)) + 2 ⇒ ((3 * 3) - (2 ^ 3)) + 2 - -but there are no expressions (rules) that allow `[()3+3()]` or `[)3(-6]` to be "matched". - -### Lexers and parsers - -You might've noticed that for `[N]` I use three dots and the note "all possible numbers" to signify that N can be any positive whole number. -In practice we need formally define that grammar: - -```grammar -N -> 0 | 1 | 2 | ... | 9 | 0N | 1N | 2N | ... | 9N - -'footnote regular-grammars Such grammars are called [url https://en.wikipedia.org/wiki/Regular_grammar regular grammars], more formally, every rule has to contain at most one name and that name must be at the start or end of the string characters. - -With this you might've realised that grammars can do anything a regular expressions can: concatenation and alternation are "built-in", and the Kleene star can be done with another rule in the form `[R -> strR | str]` [$regular-grammars]. -The reason we use both is that grammars are significantly slower and more complicated in code than regexes. - -#### Implementational difficulties with grammars - -Implementing (very roughly) the regular expression `[acce(n | p)*t]` is quite easy: - -:code_cpp -bool matchesp(string str) { - string regex = "acce_t"; - - for (int si = 0, ri = 0; si < str.size() && ri < regex.size(); si++, ri++) { - while (regex[ri] == '_') - if (str[si] == 'n' || str[si] == 'p') - si++; - else - ri++; - } - - if (str[si] != regex[ri]) - return false; - } - return true; -} - - -In brief, for concatenated characters we do a simple comparison per character. -When we reach "\_", which I use to specify we're in the `[(n | p)*]` part of the regex, we do comparisons with the characters n and p as much as we can, after which we continue as normal. - -Of course, this version only supports one regular expressions, however a "proper" implementation would look conceptually similar: a couple of different loops and equality checks, with maybe an additional array or two. -The huge benefit is that that loops and arrays are stupid fast, a (simple) C++ for loop could be translated into [url https://godbolt.org/#g:!%28%28g:!%28%28g:!%28%28h:codeEditor,i:%28filename:'1',fontScale:14,fontUsePx:'0',j:1,lang:c%2B%2B,selection:%28endColumn:16,endLineNumber:6,positionColumn:16,positionLineNumber:6,selectionStartColumn:16,selectionStartLineNumber:6,startColumn:16,startLineNumber:6%29,source:'//+Type+your+code+here,+or+load+an+example.%0Aint+f%28int+num%29+%7B%0A++++for+%28int+i+%3D+1%3B+i+%3C+100%3B+i%2B%2B%29+%7B%0A++++++++num+*%3D+100%3B%0A++++%7D%0A++++return+num%3B%0A%7D'%29,l:'5',n:'0',o:'C%2B%2B+source+%231',t:'0'%29%29,k:50,l:'4',n:'0',o:'',s:0,t:'0'%29,%28g:!%28%28h:compiler,i:%28compiler:g122,deviceViewOpen:'1',filters:%28b:'0',binary:'1',binaryObject:'1',commentOnly:'0',demangle:'0',directives:'0',execute:'1',intel:'0',libraryCode:'0',trim:'1'%29,flagsViewOpen:'1',fontScale:14,fontUsePx:'0',j:1,lang:c%2B%2B,libs:!%28%29,options:'',selection:%28endColumn:1,endLineNumber:1,positionColumn:1,positionLineNumber:1,selectionStartColumn:1,selectionStartLineNumber:1,startColumn:1,startLineNumber:1%29,source:1%29,l:'5',n:'0',o:'+x86-64+gcc+12.2+%28Editor+%231%29',t:'0'%29%29,k:50,l:'4',n:'0',o:'',s:0,t:'0'%29%29,l:'2',n:'0',o:'',t:'0'%29%29,version:4 a handful of processor instructions]. - -On the other hand, usually grammars are more complicated and resource-intensive. -Certain ones are simple enough to implement, like `[S -> aSb | cc]`: - -```cpp -bool matchesp(string str) { - if (str == "cc") - return true; - else if (str[0] == 'a' && str[str.size() - 1] == 'b') - return matchesp(str.substr(1, str.size() - 2)); - else - return false; -} - -'footnote context-free-grammars Be careful about cases that are too complicated, the grammar I'm showing is "context-free", but that topic is way beyond the scope of this post. - -However think about more complicated cases[$context-free-grammars], like `[S -> aSbSa | cc]`. -Implementing this one is quite a bit more difficult, since we don't know where `[b]` is located on the first iteration, it could either be the 4th character `[accb...a]` or the 9th `[aaccbccab...a]`. -Searching for it is also complicated, it could be the first, the last or any other `[b]` in the middle of the string. - -Another difficulty with grammars is ambiguity, let's look at - -```grammar -E -> E + E | E * E | n - -how would it generate `[n + n * n]`? -There are two completely valid ways to do so: - -```grammar+ -E ⇒ E + E ⇒ n + E * E ⇒ n + n * n - -```grammar+ -E ⇒ E * E ⇒ E + E * n ⇒ n + n * n - -How exactly we deal with these problems comes later, for now the takeaway is that we have to try and use regular expressions wherever we can, and leave grammars as a bit of a last resort. - -#### Practical usage - -'footnote static-typing Yeah yeah, in a statically typed language you might not be able to do operations on doubles and integers at the same time, but you get the point. - -Let's think a bit more about grammars: they're really good at handling structure, especially recursive structure, so it's best if we use them only for that. -Our math expression grammar (from [url #grammars here]) shouldn't really care if we're trying to sum integers or floats or chars[$static-typing], it's important how everything is laid out and that our operations are done with numbers. - -This is why we're going to separate and generalize the code into [url https://en.wikipedia.org/wiki/Lexical_analysis#Token tokens], basically into building blocks. -As a rough example, we want `[1 + (2 * 3)]` to be transformed into `[NUMBER MATHOP LBRACE NUMBER MATHOP NUMBER RBRACE]`, `[int var = 13;]` into `[TYPE NAME ASSIGNMENT NUMBER]` and so on, where "TYPE", "NUMBER", "NAME", ... are tokens. -Now we're reducing the amount of "objects" our grammars need to work on: from all characters to many (but significantly fewer) tokens. -Also we're simplifying our work, since we're trying to figure out the overall structure, it doesn't matter if we're multiplying or summing numbers, it's important that we're doing a mathematical operation on them. -Of course, each token will actually be a pair, containing it's name (NUMBER, TYPE, ...) and it's value, but our grammars will work with the names only and output the actual values at the very end. -Usually we identify at least six token types: - -.bulleted -- identifiers: variable and function names for example -- keywords: `[if]`, `[while]`, `[for]`, `[return]`, ... -- separators: braces, `[;]` -- operators: `[+]`, `[-]`, `[<=]` -- literals: numbers, strings, characters -- comments - -As you might've noticed, here we can use regular expressions, grouping consecutive characters is what regexes are good for. -Additionally, we're going to be reading the file, a decently slow operation, only once and for a comparatively very little time. - -In the compiler front-end, the part that implements our regular expressions is called the "lexer", and the part that implements our grammars - the "parser". -In the last section we already talked about how easy implementing regexes (lexers) is and how difficult implementing grammars (parsers) is, let's look at the latter. - -#### Parsing algorithms - -Proper algorithms that "implement" grammars are kind of complicated, and most can handle only specific types of grammars. -Generally we categorize them into: - -.bulleted -- [url https://en.wikipedia.org/wiki/Top-down_parsing top-down] algorithms, which work in the way we've been talking about grammars: start with a name and replace it with it's expression, and continue replacements until we have no more names -- [url https://en.wikipedia.org/wiki/Bottom-up_parsing bottom-up] algorithms, where we do the opposite: for each substring we find from which rule it could've come from, replace it with the corresponding name, and continue upwards like that, until we reach the starting name - -Furthermore, if our algorithms allow for ambiguity (meaning we can replace a name with any one of multiple expressions), we divide them into: - -.bulleted -- left-most derivation algorithms, where we choose the left-most valid expression -- right-most derivation algorithms, where we chose the right-most valid expression - -The advantage and disadvantages of every type kind of depend on the exact algorithm and grammar. -Looking at some of the more famous ones: - -##### LL - -[url https://en.wikipedia.org/wiki/LL_parser LL]: top-down left-most derivations algorithm that can parse only specific types of grammars. -In these grammars, every expression is uniquely identifiable by just looking at the characters (tokens) in the very beginning. -The amount of token we need to look for is very important, so we usually define LL(k) as an LL grammar that needs to look at k amount of tokens. -An "LL(k) grammar" is, by extension, a grammar which can be parsed with an LL(k) parser. - -So, the way we'll check if an expression matches our grammar, is by starting with the start name and reading the next k character from the input. -We check the first k character of all expressions, and if anyone of them is a name, we try with substitution for every possibility with that name, essentially peeking into the name's possibilities. - -:unordered -- As an example, the grammar: - - ```grammar - F -> a - S -> F | (S + F) - - is LL(1). - Let's read the input `[(a + a)]` (the "v" points to the symbol we're looking/peeking at): - - :code - I v - Input: (a + a) - - I v - Production: S - - - we start with `[S]`, we peek and get the `[(]`, in our production we're looking at a name, S. - Let's check the first character of every expression of S. - In the first expression, the first "character" is `[F]`, a name, so we look at every expression for `[F]`. - There is only one, which is "a", different than "(", no dice. - Now we check the second expression for S, it's first character is "(", success! - We replace: - - :code - I v - Input: (a + a) - - I v - Production: (S + F) - - - Now the first symbol in our production is also `[(]`, so we increment the symbols we're looking/peeking at - - :code - I v - Input: (a + a) - - I v - Production: (S + F) - - - we peek and get the `[a]`, in our production the second "character" is the name `[S]`, `[S]` has to become `[a]`. - We do that same check from before, first expression for `[S]` is `[F]`, whose first (and only) expression is `[a]`, success! - - :code - I v - Input: (a + a) - - I v - Production: (a + F) - - - The middle characters " + " are matched and at the second `[F]` we check it's expressions, it has only `[a]`, success, replace, end of input, we're done. - - Another LL(1) example can be found in [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Top-down/LL1.html Karl R. Abrahamson notes]. - -- The grammar - - ```grammar - B -> b - C -> c - S -> aaB | aaC - - is LL(3). - For the input `[aab]`, if we look at the first two "a"s, we still don't know which rule to use: `[aaB]` or `[aaC]`. - However if we look at the third symbol, "b", then we can determine that we need `[aaB]`. - -- Usually parsers that implement this procedure are called [url https://en.wikipedia.org/wiki/Recursive_descent_parser recursive descent parsers] - - -##### LR - -[url https://en.wikipedia.org/wiki/LR_parser LR]: bottom-up right-most derivations algorithm. -Conceptually, LR works in the exact opposite way to LL: for every character in the input, we find a rule that substitutes it (or that substitutes the current collection of names and character) and do so until we've reached the starting name. - -Figuring out an approach isn't as intuitive and easy as with LL, but it still isn't too hard. -It all boils down to using a whole bunch of conditional statements and tracing where we "came" from. - -:unordered -- Let's look at the first LL example: - - ```grammar - F -> a - S -> F | (S + F) - - We need to track the current state of the string, because imagine we've just read an "a", it could either be from the left or right handside of "a + a", and that changes what character to expect next. - We're going to track the state with different numbers, defined like this: - - .table - |=State (number)=||=Current expression ends with=| - |:0 :||:Nothing; this is the start:| - |:1 :||:The name `[S]`; the final number should be this:| - |:2 :||:The name `[F]`:| - |:3 :||:The character "(":| - |:4 :||:The character "a":| - |:5 :||:"(S":| - |:6 :||:"(S + ":| - |:7 :||:"(S + F":| - |:8 :||:"(S + F)"; this is the one before the final:| - - You can think of every state number as the valid, but "unfinished" state of the current string. - For now don't think about the way this table is defined, it'll make sense later. - But you could make a state for literally every valid string composition: "(", "(a", "(F", "(S", .... - - Keep in mind that the way I'll show the algorithm step by step isn't really correct, but will suffice for the current moment. - Reading a character from the input means appending it to the right of String: - - ``` - String : ( - State : 0 - - We've read "(", so our string "ends" on a "(". - - ``` - String : (a - State : 3 - - We've read "a", our string "ends" on an "a". - - ``` - String : (a - State : 4 - - Here comes the clever bit, for certain states, we'll stop reading forward and do something else. - If we're on state 4, our string ends on an "a", so we're going to go backwards and replace it with an F, meaning the current state should be 2. - - ``` - String : (F - State : 2 - - Here is another such rule, if our state is 2, our string ends with an "F" and we replace it with an S, now the state is 5. - That's why we have a different state for "(S + F", if we didn't, our state would become 2 and we would make "(S + S", which isn't valid. - So, think of 2 as "this is the first F". - - ``` - String : (S - State : 5 - - We have a different state for "(S", because if we didn't, we would've gone to state 1, and could've ended the algorithm then and there, without reading the rest of the input. - Otherwise, we continue as normal, we read the `[+]` and the spaces around it, going to state 6. - - ``` - String : (S + a - State : 6 - - Now, our string ends with an "a", it's tempting to change our state to 4, but we shouldn't, this would remove our progress so far! - Better yet, this state will also be special, if we have a 6 and we end with "a", rather than going to 4, we'll substitute "a" with F now and change state to 7. - - ``` - String : (S + F - State : 7 - - And now it's trivial, we read ")", so we go to state 8. - But if we're at state 8, don't continue reading, replace with state 1. - Now that we're at state 1, if we read any character (including end of string), we'll say the string up to now matches and we're done! - - It's also helpful to put what we do in different situations in another table: - - .table - |=State=||=If ends on=||=Change state to=||=Also do=| - |:0 :||:"(" :||:3 :||: :| - |:1 :||: :||: :||:Nothing, we matched:| - |:2 :||: :||:5 :||:Replace "F" with "S":| - |:3 :||:"a" :||:4 :||::| - |:4 :||: :||:2 :||:Replace "a" with "F":| - |:5 :||:" + " :||:6 :||: :| - |:6 :||:"a" :||:7 :||:Replace "a" with "F":| - |:7 :||:")" :||:8 :||: :| - |:8 :||: :||:1 :||:Replace "(S + F)" with "S":| - - where, if there is a value in "If ends on", we read the next character, while otherwise we don't and we just execute "Change state to" and "Also do". - -- You're probably wondering why the table works so weird, like why on state 3 we don't directly replace "a" with "S" and change to state 5 for example. - The reason is that the way I showed the algorithm isn't exactly how it normally works. - Rather than tracing the current state in a variable, we trace it with a [url https://en.wikipedia.org/wiki/Stack_%28abstract_data_type%29 stack]. - - When you read a character and have to go to another state, the state is added to the stack (this operation is called "shift"). - When we have to change characters from String (without reading anything), with a string (expression) of length j, we remove the last j added states and do a normal shift from the last state and character in String (this operation is called "reduce"). - -- Let's look at the same example, done with shift and reduce operations (without me showing the proper table), where pushing and popping items from the stack is done from the right side of the array: - - ``` - String : ( - State : [0] - - We've read "(", add 3 on top of the stack aka shift(3). - Then we read the "a", shift(4). - - ``` - String : (a - State : [0, 3, 4] - - With 4 we'll do a reduction, "a" gets replaced by F, 4 is removed: - - ``` - String : (F - State : [0, 3] - - If last state is 3 and last character is "F", we're doing shift(2). - - ``` - String : (F - State : [0, 3, 2] - - Now, we'll be doing a reduction, where F is replaced by S: - - ``` - String : (S - State : [0, 3] - - If last state is 3 and last character is "S", we're dong shift(5). - - And so on, we continue. - As you can see, we go back and forth, from 3 to 4, back to 3, then to 2, back to 3, then to 5. - This way we avoid doing different things on different states and we don't need to store the whole string, only the last few characters. - Additionally, in the real world nobody makes this table by hand, it is generated by a program, and this method makes table generation easier. - -- As for the complete table, you can view it on the [url https://smlweb.cpsc.ucalgary.ca/lr0.php?grammar=S+-%3E+F+%7C+%28+S+%2B+F+%29.%0AF+-%3E+a.&substs= Context Free Grammar Checker],\n - or with a more pretty styling on [url https://mdaines.github.io/grammophone/?s=UyAtPiBGIHwgIigiIFMgIisiIEYgIikiLgpGIC0+IGEu#/ Grammophone], opening "Parsing table" on the row "LR(0)". - - To read the table, remember that: - - .bulleted - - on every row you have each state - - on every column you have each possible value in the string and "$" which signifies the end of the string - - with `[sr]` or `[shift]` (or just with a number) you go to another state - - with `[r]` or `[reduce]` you make the substitution in the current string and go back one state - - `[acc]` or `[accept]` means you've reached the end and the string is valid - - -##### LR's family - -LR is more of a family of algorithms, because there are multiple different ways to generate the table, each with it's own benefits and drawbacks. -This subject is a really big can of beans, which I'm not knowledgeable enough to talk about. -What you need to know is that there are 3 often used types: [url https://en.wikipedia.org/wiki/Canonical_LR_parser CLR], [url https://en.wikipedia.org/wiki/Simple_LR_parser SLR] and [url https://en.wikipedia.org/wiki/LALR_parser LALR]. - -.bulleted -- CLR is an LR parser with 1 character lookahead (LR(1)), following [url TODO Knuth]'s original theory. - It requires a lot of memory for table generation, so Frank DeRemer created two simplifications: -- the SLR and LALR, which use significantly less memory but work with a bit less grammars - -LALR is the most commonly used LR parser, however most compilers are custom-tailored pieces of software that very loosely follow the methods of LL and LR. - -##### More info - -If you want to learn more, I would highly suggest: - -.bulleted -- [url https://blog.reverberate.org/2013/07/ll-and-lr-parsing-demystified.html LL and LR Parsing Demystified] and [url https://blog.reverberate.org/2013/09/ll-and-lr-in-context-why-parsing-tools.html LL and LR in Context: Why Parsing Tools Are Hard] by [url https://blog.reverberate.org/about/ Josh Haberman] -- [url http://jeffreykegler.github.io/Ocean-of-Awareness-blog/individual/2014/11/ll.html Parsing: Top-down versus bottom-up] by [url https://jeffreykegler.github.io/personal/ Jeffrey Kegler] -- [url https://tratt.net/laurie/blog/2020/which_parsing_approach.html Which Parsing Approach?] by [url https://tratt.net/laurie/ Laurence Tratt] -- [url https://tomassetti.me/guide-parsing-algorithms-terminology/ A Guide to Parsing: Algorithms and Terminology] by [url https://inre.me/ Gabriele Tomassetti] -- [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Parsing/index.html Parsing], [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Top-down/index.html Top-Down Parsers] and [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Bottom-up/index.html Bottom-Up Parsers] from [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Bottom-up/index.html Karl R. Abrahamson]'s notes -- [url https://serokell.io/blog/how-to-implement-lr1-parser How to Impement an LR(1) Parser] by Kirill Andreev - -#### Abstract syntax tree - -We already decided that the lexer will output tokens (and characters) to the parser, but we didn't talk about what the parser is supposed to output. -It definitely has to be some sort of structural representation of the overall code, for example, for `[3 + 1 * 2]` we have to be able to easily find out what the left and right handsides of the `[+]` and `[*]` are. - -'footnote nomnoml The graphics are made with [url https://nomnoml.com/ nomnoml] by [url https://github.com/skanaar Daniel Kallin] - -Let's draw out[$nomnoml] this relationship, where the left arrow connects to the left handside and the right accordingly: - -[image ./img/tree-simple.svg] - -Hey, that looks an awful lot like a [url https://en.wikipedia.org/wiki/Tree_%28data_structure%29 tree]! -Trees are nodes, some sort of elements, where there is a connection from one element to some other elements, with the only rule being that cycles =[aren't]= allowed: - -[image ./img/tree-cycle.svg] - -Great, but can we represent everything else with a tree? -Why, of course, for example we can represent: - -```cpp -if (a > b) - c = 5; -else - d += 10; - -as - -[image ./img/tree-if.svg] - -You might ask how we generate the tree, and the answer is simple: on every step we add nodes to the tree, depending on the situation. -Let's look at a familiar example: - -```grammar -N -> 1 | 2 | 3 -E -> N | (E) | E + E | E * E - -Let's look at the input string `[2 * (1 + 3)]`, while abstracting ourselves from any particular algorithm: - -:unordered -- we read the `[2]`, that is a number, we add it as a node - - .p [image ./img/tree-building-1.svg] - -- then we read the `[*]`, we now know our expression is in the form `[E* E]`. - If you think about it, up to this point, we've basically been building a tree for the left handside. - - =[If]= the topmost element in that tree is a brace, - - .p [image ./img/tree-building-ex1.svg] - - we connect it to the left handside of `[*]`: - - .p [image ./img/tree-building-ex2.svg] - - =[Otherwise]= (when it's `[+]` or `[*]` or a number), because of precedence, we'll attach the latest number to the left handside. - However we have to be careful, if something connects to the number, we now need it to connect to the `[*]`.\n - For example, let's say we had `[1 + 2 * 3]`, and we've just read the `[*]`. - The tree up to this point would look like, - - .p [image ./img/tree-building-ex3.svg] - - where the latest number we've found was `[2]`, but the `[+]` connects to it. - So, we need to make the `[2]` connect to the left handside of `[*]`, and make `[+]` connect to `[*]` instead of `[2]` - - .p [image ./img/tree-building-ex4.svg] - - In our original case, nothing connects to the `[2]` in `[2 * (1 + 3)]`, so we can just connect it to the left handside of `[*]` - - .p [image ./img/tree-building-2.svg] - - where the three dots signify there isn't anything there (but we expect there to be). - Somewhere we'll store that the next element we find has to be connected to the right handside. - -- now we read `[(]`, we cannot have it by itself, so we directly decide we're working with `[(E)]` (in our full algorithm we would need a check for existance of matching brace). - We'll create a `[()]` node and because of the last step, we'll connect it to the right handside. - - .p [image ./img/tree-building-3.svg] - - Now, moving forward, we'll be creating a subtree, essentially making a second tree, and after that is done, we'll connect it's topmost element (root) to the `[()]` node. - -- and so on. - - -The overall algorithm is to add and change nodes every time you match something with a grammar rule, and the adding and changing is done, depending on the language, the current structure and so on. - -When discussing the middle and back-end layers, we'll talk about how to work with a generated AST, but for now your clue is the [url https://en.wikipedia.org/wiki/Visitor_pattern visitor pattern]. - -### Implementing a front-end - -If you want, you could implement your own lexer and parser manually, but as a start, it's worthwhile to try and use already created stuff. -[url https://en.wikipedia.org/wiki/Compiler-compiler Compiler-compilers] are tools that create parsers (and more) from their normal formal definition. - -The two popular ones from the old days are lex and yacc. -lex is a lexer generator and yacc is a LALR parser generator, both of which generate a corresponding C implementation of the given regular expressions and grammars. - -#### lex - -Lex is the original program, from 1975, in the modern day it's successor/implementation is the open-source [url https://github.com/westes/flex flex]. - -A lex source file is divided into three parts: - -:code_lex -%{ -// C code that will be put in the program verbatim -%} - -/* Definitions */ - -%% - - /* Rules */ - -%% - -// C code - - -where `[%{%}]` is used for includes and declarations, the rules are your regular expressions and the C code is the general logic of the lexer. -Usually you wouldn't write anything in the C code section, the defaults are fine, but you'll probably need to do one or two includes in the `[%{%}]` section. - -Rules are in the form `[regex { C_code }]`. -To simplify your life, you can write only the regex or a part of it inside the "definitions" part of the file, and give it a name to use later. -Referencing those names in later regexes is done by surrounding them in curly braces. -Usually the `[C_code]` in the regex would be a return (more commonly called "emit") of a token, which will be available in a header file, but more on that later. - -Let's look at a very simple example program: every whole number is printed as "NUMBER", while everything else is printed as-is: - -:code_lex -%{ -%} - -%% - -[0-9]+ { printf("NUMBER"); } -[A-z]+ { printf("%s", yytext); } - -%% - - -The syntax `[[0-9]]` is the same as `[(0 | 1 | ... | 9)]`, it works for any letters and is available in all modern regexes. -Note that inside every regex's "body" you have an assortment of variables already set, one of them being `[yytext]` which is just the whole match. -Additionally lex matches the longest possible match, so if we had a rule `[[0-z]]`, only it will be matched. - -If this file is located in `[lexer.l]`, we can transform it into a C file with: - -```bash -flex lexer.l - -Usually the result file is always named `[lex.yy.c]`. -Afterwards, we can compile the C code into a working compiler: - -```bash -gcc lex.yy.c -lfl - -Without the `[-lfl]` flag, we would get a linking error, because of the `[main]` and `[yywrap]` function, which are required but aren't defined in our generated code. -There are three ways to fix this problem: - -:numbered -1. The one I've opted for, use the `[-lfl]` flag to compile our C code with a (flex) library which provides both functions. - Usually the lexer isn't meant to be ran by itself, so for the current time being, `[-lfl]` is the best option, the other two are fine when we get to `[yacc]`. - -2. Put `[%option noyywrap]` in the definitions section, which means the lexer will only read one file. - -3. Define it ourselves. - It should return 0 if our lexer should continue with the next given file, after finishing with the current one, or 1 if it should stop execution. - The function could be written in either "C code" section, but I personally prefer the last one. - - ```lex - int yywrap() { return 0; } - - -Of course, you can use your lex implementation or C compiler of choice, and there this might not be a problem. -Now, if you run `[./a.out]` you would get a prompt, every line you write will be lexically analyzed and in our case, everything will be printed out. -Entering - -``` -The number 10 changed to 8, 38, and 19 sequentially, before becoming 0. - -will output - -``` -The number NUMBER changed to NUMBER, NUMBER, and NUMBER sequentially, before becoming NUMBER. - -Now let's try something more sophisticated, let's surround numbers and strings in HTML tags, the stuff that power webpages. -We would want numbers to be in the format: - -``` -number - -and strings: - -``` -string - -We'll define numbers as both whole digits and floating point numbers (with sign) and strings can be surrounded in either single or double quotes. - -:code_lex -%{ -%} - -/* Only multiline comments are supported, not comments that start with // */ -/* This is a definition, it's in the format "NAME regex" */ -DIGIT [0-9] - -%% - /* Comments in here have to be indented, otherwise you'll get an error */ - /* ? means 0 or 1 of the preceding group */ - /* + means 1 or more of the preceding group */ - /* the backward slashes are used because . and + have special meaning in regexes */ -[\+-]?{DIGIT}+(\.{DIGIT}+)? { printf("%s", yytext); } - - /* [^] is a group of characters, like [], however it specifies "all characters except the ones given" */ - /* this way if we have "This is a string with " three quotes" it's not going to match the whole thing */ -\"[^"]*\"|'[^']*' { printf("%s", yytext); } - -%% - - -With the input - -``` -John said 10 times "2 + 2 = 5" but '2 + 2 = 4'! - -we'll get - -``` -John said 10 times "2 + 2 = 5" but '2 + 2 = 4'! - -You've probably already noticed that the number inside the strings aren't matched, because the string rule results in a longer match. -In certain programming languages, that don't have [url https://en.wikipedia.org/wiki/String_interpolation string interpolation] this is fine, but for others our regexes have to be a lot more complicated. - -#### yacc - -Similarly to lex, yacc originates from 1975, and it's modern replacement is [url https://www.gnu.org/software/bison/ GNU Bison]. - -A yacc file looks very similarly to a lex one: - -:code_yacc -%{ -// C code that will be put in the program verbatim -%} - -/* Declarations */ - -%% - - /* Rules */ - -%% - -// C code - - -however rather than definitions you have declarations, and the rules are in a completely different format (because they're grammars). - -Token names are defined in `[declarations]`, in the format - -```yacc -%token NAME1 NAME2 ... - -You can also define precedence with `[%left]` and `[%right]`. -There are also other options you can set, like the start name with `[%start]`. - -Grammars are in the form - -```yacc -name : TOKEN1 TOKEN2 ... - ; - -where alternation is defined like - -```yacc -name : TOKEN1 - | TOKEN2 - | TOKEN3 - ; - -and it always must end with a `[;]`. -On each alternation, you can (should) specify a body, which will contain C code to be executed, if this rule is reached - -```yacc -name : TOKEN1 { C code here } - | TOKEN2 - | TOKEN3 { C code here } - ; - -Another difference is that, since the parser needs a list of tokens, you cannot run it by itself, you'll also need to setup a lexer. - -'footnote shift-reduce-conflicts You might be tempted to try and make one for the grammar inside the [url #abstract-syntax-tree AST] section, but that one is ever so slightly problematic, because it's ambiguous (I showed why at the end of [url #implementational-difficulties-with-grammars Implementational difficulties with grammars]). Bison [url https://www.gnu.org/software/bison/manual/html_node/Shift_002fReduce.html can handle ambiguity by always choosing to shift], but I didn't want to complicate things. - -Let's implement a parser for the grammar[$shift-reduce-conflicts]: - -```grammar -N -> 1 | 2 | 3 -S -> N | (S + S) | (S * S) - -First, our tokens, we want to replace all characters with a token. -For 1, 2 and 3, let's use the token name "NUMBER", for "(" and ")", LBRACE and RBRACE accordingly, and since there is no precedence with this grammar, we don't actually care if we have `[(S + S)]` or `[(S * S)]`, so we can assign both `[+]` and `[*]` to the token MATHOP. -Meaning, our grammar would behave more like this: - -```grammar -N -> 1 | 2 | 3 -M -> + | * -S -> N | (S M S) - -First, we need to make our `[lexer.l]`, it is going to be very simple: - -:code_lex -%{ -#include "parser.tab.h" -%} - -%% - - /* Remember, [] is a group, [abcd] behaves like "(a | b | c | d)" and "\\" escapes characters */ -[123] { return NUMBER: } -[\+\\*] { return MATHOP; } -\( { return LBRACE; } -\) { return RBRACE: } - -%% - - -Now, tokens (names) are defined inside grammars, however our lexer needs to emit (return) them. -That's why, when running bison on our `[parser.y]`, we'll also create a C header file, which contains those names already defined, so they're accessible inside `[lexer.l]`. -This file is in the format `[FILENAME.tab.h]`, where `[FILENAME.y]` is our source yacc file. - -Second, we need to make our `[parser.y]`: - -:code_yacc -%{ -#include "stdio.h" -int yylex(); -int yyerror(char *s); -%} - -%token NUMBER MATHOP LBRACE RBRACE - -%% - -expr : NUMBER - | LBRACE expr MATHOP expr RBRACE - ; - -%% - -int yyerror(char *s) { - printf("Error: %s\\n", s); - return 0; -} - -int main() { - yyparse(); -} - - -A quirk with GNU Bison is that you need to declare the function `[yylex]` and `[yyerror]` beforehand, manually define `[yyerror]` and run `[yyparse]` in the main function. -These are precisely the first 5 and the last 8 lines of code, but for now you don't need to think about them. - -Let's get to compiling. -First, the lexer: - -```bash -flex lexer.l - -then our parser: - -```bash -bison -d parser.y - -where the `[-d]` option means we'll also output the `[parser.tab.h]` header file. -Finally, we compile both of them together into one executable: - -```bash -gcc lex.yy.c parse.tab.c -lfl - -Now when you run `[./a.out]`, if you enter a valid string, you'll see a blank line, but if it isn't valid, you'll get "Error: syntax error". - -#### Doing it properly - -The way we created that lexer and parser worked, but it wasn't very useful and has multiple issues we have to address: - -:ordered -1. When entering a string that isn't matched at all, it gets printed back like nothing happened. - For example, if we enter the letter `[a]` or any string which doesn't have `[(]`, `[)]`, `[1]`, `[2]`, `[3]`, `[+]` or `[-]` characters, it will just get printed back. - Ideally, we want to raise an error, so we're going to add one rule which catches any (non-special) character and throws an error with the built-in `[yy_fatal_error]`: - - ```lex - . { yy_fatal_error("Bad character"); } - - This rule must be the last rule, since we don't want it to be matched, if there is another rule that matches. - Remember that lex matches the longest possible string and if there are multiple rules which match the same amount of characters, only the first one is matched. - -2. For every token name, returned by the lexer, we also need to have it's actual value. - We could make a `[struct]`, which holds both a token name and it's value, but there is another [url https://www.gnu.org/software/bison/manual/html_node/Token-Values.html "built-in" mechanism] for that in yacc/Bison. - We would set the global variable `[yylval]` to the token value, and it will be changed for every token. - - By default it's type is `[int]` (in Bison), but usually you would want it to be a [url https://en.wikipedia.org/wiki/Union_type#C/C++ union]. - Let's make it a union of int and char, though the first part is kind of useless, since we're only working with single digit numbers. - \n - ```yacc - %union { - int num; - char chr; - } - - Now, inside our lexer, rules would look like: - - ```yacc - RULE1 { yylval.num = 129; return TOKEN; } - RULE2 { yylval.chr = 'a'; return TOKEN; } - - Inside C blocks for grammar rules we can reference the token values with `[\$i]`, where `[i]` is the i'th "token" in the expression. - For example, in this rule, `[\$1]` would be `[(]`, `[\$3]` would be value of token `[B]` and `[\$5]` would be the value of the rule name: - - ```yacc - rule : ( A B ) rule - ; - - You can set rule "return" values by setting the variable `[\$$]`: - - ```yacc - rule : LBRACE rule RBRACE { $$ = 10; } - - The way you set it's type is discussed later. - However, `[\$i]` isn't of type union, we cannot do something like `[\$i.num]`, it is the precise value that was set for the token. - Therefore, we also need to define which value from the union every token will set, which is done by adding `[]` before every defined token (and `[PROPERTY]` is a property name inside the union): - - ```yacc - %token NUMBER MATHOP LBRACE RBRACE - - 'footnote better-bison-errors You can add `[%define parse.error verbose]` inside definitions of your yacc file, and it will tell you what character was expected and what character was read. - -3. If we enter one valid math expression, we'll get no error, but when we enter another, we'll get a "syntax error"[$better-bison-errors]. - That is because we try to match only one expression, after it is matched our parser is waiting for an end of file character. - Let's make it so we would try to match an expression on every line: - - ```yacc - start : expr - | expr start - ; - - A rule like `[expr : expr expr]` will produce shift/reduce conflicts. - Another option is duplicating our `[expr]` rules and adding `[expr]` in the beginning or end, but that is cumbersome if we want to make changes. - -4. We're getting input from the user, but our source text (code) will be in a file. - First, our `[main]` function needs to take program parameters: - - ```yacc - int main(int argc, char* argv[]) - - Then we need to take the first argument, and open a file. - Additionally, we have to check if the file was successfully opened, and print an error and stop execution if so: - - ```yacc - FILE* input = fopen(argv[1], "r"); - if (input == NULL) { - yyerror("cannot read input file!"); - return 1; - } - - Now for the clever bit, actual parsing in generated code is done on the file `[yyin]`, which by default is the standard user input. - To read a file, all we need to do is change the value of that variable. - Also, that variable is defined externally, so first we need to add to (the top) C section: - - ```yacc - extern FILE* yyin; - - and then put in our `[main]`: - - ```yacc - yyin = input; - - On \*NIX systems (Linux, MacOS, OpenBSD, ...) you can still use user input by passing the file `[/dev/stdin]`. - If you want your front-end to support multiple files, you would just need to wrap the whole body of main in a `[for]` loop, iterating over the whole `[argv]` array (remember that `[argc]` is the length of the array). - -5. We need to construct an AST. - Since the exact tree structure depends on your needs, I'll make the simplest possible tree in pure C that works for this grammar. - I won't explain this in much detail, since it is just normal C code. - - :code_c - #include - #include - - typedef struct node { - char val; - struct node *left; - struct node *right; - } node; - - node* newNode(char val, node *left, node *right) { - node* out = malloc(sizeof(node)); - out->val = val; - out->left = left; - out->right = right; - return out; - } - - void printTree(node *n, int level) { - if (n == NULL) return; - // %*s will add level amounts of spaces (but with the "" string) - printf("%*s|-%c\n", level, "", n->val); - printTree(n->left, level + 1); - printTree(n->right, level + 1); - } - - void freeNode(node *n) { - if (n == NULL) return; - freeNode(n->left); - freeNode(n->right); - free(n); - } - - - However, all of this is quite a lot, and it would take a lot of space in `[parser.y]`, so let's separate it into `[tree.c]` and add a header file `[tree.h]` which contains the definitions: - - :code_c - #include - #include - - typedef struct node { - char val; - struct node *left; - struct node *right; - } node; - - node* newNode(char val, node *left, node *right); - void printTree(node *n, int level); - void freeNode(node *n); - - - Of course, we should add some ifndef-def precompiler code, but that isn't relevant for now. - - The way we're going to use trees in our grammar is as follows: for the `[NUMBER]` expression of `[expr]`, we'll create a node with the (char) value of the number and "return" it. - For the expression of `[expr]` we'll "return" a node with the value of the math operator and left and right will be the returned values. - Inside start, after we've parsed an `[expr]`, we'll print out the "returned" tree and free it's memory. - - I'm putting returned in quotes, because (as mentioned), we'll just be setting the value of `[\$$]`. - `[\$$]`'s type is set with the `[%type]` option, which works exactly like `[%token]`, but with grammar rule names. - However, this means that `[node]` must be available in `[%union]`, which is unfortunate but isn't a problem, we'll just add it. - - Now, finally, it's time to include our `[tree.h]` in `[parser.y]`, but we'll also need to include it in `[lexer.l]`. - The reason is that the `[%union]` appears in `[parser.tab.h]`, which means our node structure is referenced in `[parser.tab.h]`, but `[tree.h]` isn't included there. - - -All in all, we'll have 4 source files for our simple front-end: - -:unordered -- `[lexer.l]` - - :code_lex - %{ - #include "tree.h" - #include "parser.tab.h" - %} - - %option noyywrap - - %% - - [123] { yylval.num = atoi(yytext); return NUMBER; } - [\+\*] { yylval.chr = yytext[0] ; return MATHOP; } - \( { yylval.chr = '(' ; return LBRACE; } - \) { yylval.chr = ')' ; return RBRACE; } - \ { } /* Don't do anything on spaces */ - - . { yy_fatal_error("Bad character"); } - - %% - - -- `[parser.y]` - - :code_yacc - %{ - #include - #include "tree.h" - - extern FILE* yyin; - - int yylex(); - int yyerror(const char *s); - %} - - %union { - int num; - char chr; - node *n; - } - %token NUMBER MATHOP LBRACE RBRACE - %type expr - - %% - - start : expr { printTree($1, 0); freeNode($1); } - | expr start { printTree($1, 0); freeNode($1); } - ; - - expr : NUMBER { $$ = newNode($1 + '0', NULL, NULL); } - | LBRACE expr MATHOP expr RBRACE { $$ = newNode($3, $2, $4); } - ; - - %% - - int yyerror(const char *s) { - printf("Error: %s\n", s); - return 0; - } - - int main(int argc, char* argv[]) { - FILE* input = fopen(argv[1], "r"); - if (input == NULL) { - yyerror("reading input file!"); - return 1; - } - - yyin = input; - yyparse(); - } - - -- `[tree.h]` - - :code_c - #ifndef TREE - #define TREE - - #include - #include - - typedef struct node { - char val; - struct node *left; - struct node *right; - } node; - - node* newNode(char val, node *left, node *right); - void printTree(node *n, int level); - void freeNode(node *n); - - #endif - - -- `[tree.c]` - - :code_c - #include "tree.h" - - node* newNode(char val, node *left, node *right) { - node* out = malloc(sizeof(node)); - out->val = val; - out->left = left; - out->right = right; - return out; - } - - void printTree(node *n, int level) { - if (n == NULL) return; - printf("%*s|-%c\n", level, "", n->val); - printTree(n->left, level + 1); - printTree(n->right, level + 1); - } - - void freeNode(node *n) { - if (n == NULL) return; - freeNode(n->left); - freeNode(n->right); - free(n); - } - - - -We compile everything: - -```bash -flex lexer.l && bison parser.y && gcc lex.yy.c parse.tab.c tree.c - -Now, let's create the file `[test.txt]`: - -``` -((1+1) * ((3+3) * 2)) - -and run our front-end on it: - -```bash -./a.out test.txt - -Our output will be: - -``` -|-* - |-+ - |-1 - |-1 - |-* - |-+ - |-3 - |-3 - |-2 - -Perfect, we have a good looking tree! - -## Appendix - -Originally I planned to put everything in one blog post, but writing the front-end (this part) alone took me 2 and a half weeks of (inconsistent, I was suddenly hit with multiple exams and homeworks in uni) writing. -Additionally, while doing research, I found out the course actually skipped over a lot of important topics, which I'll first need to research. -Future parts will be linked right below the title at the very top, but it will probably be quite a while until I write them. - -Many thanks to [url https://github.com/dtagarev Dimitar Tagarev] for his help with reviewing this post. diff --git a/src/blog/sane-explanation-of-compilers/part-1/example_lex/lexer.l b/src/blog/sane-explanation-of-compilers/part-1/example_lex/lexer.l new file mode 100644 index 0000000..f110da4 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/example_lex/lexer.l @@ -0,0 +1,9 @@ +%{ +%} + +%% + +[0-9]+ { printf("NUMBER"); } +[A-z]+ { printf("%s", yytext); } + +%% diff --git a/src/blog/sane-explanation-of-compilers/part-1/example_lex/numbers_html.l b/src/blog/sane-explanation-of-compilers/part-1/example_lex/numbers_html.l new file mode 100644 index 0000000..47f40b7 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/example_lex/numbers_html.l @@ -0,0 +1,19 @@ +%{ +%} + +/* Only multiline comments are supported, not comments that start with // */ +/* This is a definition, it's in the format "NAME regex" */ +DIGIT [0-9] + +%% + /* Comments in here have to be indented, otherwise you'll get an error */ + /* ? means 0 or 1 of the preceding group */ + /* + means 1 or more of the preceding group */ + /* the backward slashes are used because . and + have special meaning in regexes */ +[\+-]?{DIGIT}+(\.{DIGIT}+)? { printf("%s", yytext); } + + /* [^] is a group of characters, like [], however it specifies "all characters except the ones given" */ + /* this way if we have "This is a string with " three quotes" it's not going to match the whole thing */ +\"[^"]*\"|'[^']*' { printf("%s", yytext); } + +%% diff --git a/src/blog/sane-explanation-of-compilers/part-1/example_yacc/lexer.l b/src/blog/sane-explanation-of-compilers/part-1/example_yacc/lexer.l new file mode 100644 index 0000000..d029fc0 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/example_yacc/lexer.l @@ -0,0 +1,13 @@ +%{ +#include "parser.tab.h" +%} + +%% + + /* Remember, [] is a group, [abcd] behaves like "(a | b | c | d)" and "\\" escapes characters */ +[123] { return NUMBER: } +[\+\\*] { return MATHOP; } +\( { return LBRACE; } +\) { return RBRACE: } + +%% diff --git a/src/blog/sane-explanation-of-compilers/part-1/example_yacc/parser.y b/src/blog/sane-explanation-of-compilers/part-1/example_yacc/parser.y new file mode 100644 index 0000000..69636d9 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/example_yacc/parser.y @@ -0,0 +1,24 @@ +%{ +#include "stdio.h" +int yylex(); +int yyerror(char *s); +%} + +%token NUMBER MATHOP LBRACE RBRACE + +%% + +expr : NUMBER + | LBRACE expr MATHOP expr RBRACE + ; + +%% + +int yyerror(char *s) { + printf("Error: %s\\n", s); + return 0; +} + +int main() { + yyparse(); +} diff --git a/src/blog/sane-explanation-of-compilers/part-1/grammar_trace_1.txt b/src/blog/sane-explanation-of-compilers/part-1/grammar_trace_1.txt new file mode 100644 index 0000000..747ef4a --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/grammar_trace_1.txt @@ -0,0 +1,26 @@ +F -> a +S -> F | (S + F) +--- +I v +Input: (a + a) + +I v +Production: S +--- +I v +Input: (a + a) + +I v +Production: (S + F) +--- +I v +Input: (a + a) + +I v +Production: (S + F) +--- +I v +Input: (a + a) + +I v +Production: (a + F) diff --git a/src/blog/sane-explanation-of-compilers/part-1/grammar_trace_2.txt b/src/blog/sane-explanation-of-compilers/part-1/grammar_trace_2.txt new file mode 100644 index 0000000..40d957f --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/grammar_trace_2.txt @@ -0,0 +1,38 @@ +F -> a +S -> F | (S + F) +--- +String : ( +State : 0 +--- +String : (a +State : 3 +--- +String : (a +State : 4 +--- +String : (F +State : 2 +--- +String : (S +State : 5 +--- +String : (S + a +State : 6 +--- +String : (S + F +State : 7 +--- Shift-reduce --- +String : ( +State : [0] +--- +String : (a +State : [0, 3, 4] +--- +String : (F +State : [0, 3] +--- +String : (F +State : [0, 3, 2] +--- +String : (S +State : [0, 3] diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-1.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-1.svg new file mode 100644 index 0000000..174f003 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-1.svg @@ -0,0 +1,26 @@ + + #background: #fdf6e3 + [2] + + + + + + + + + + + + + + 2 + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-2.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-2.svg new file mode 100644 index 0000000..48e2636 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-2.svg @@ -0,0 +1,53 @@ + + #background: #fdf6e3 + [*]->[2] + [*]->[<abstract>...] + + + + + + + + + + + + + + + + + + * + + + + + + + + + + + 2 + + + + + + + + + + + ... + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-3.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-3.svg new file mode 100644 index 0000000..ea526df --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-3.svg @@ -0,0 +1,53 @@ + + #background: #fdf6e3 + [*]->[2] + [*]->[()] + + + + + + + + + + + + + + + + + + * + + + + + + + + + + + 2 + + + + + + + + + + + () + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex1.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex1.svg new file mode 100644 index 0000000..d5addd2 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex1.svg @@ -0,0 +1,39 @@ + + #background: #fdf6e3 + [()]->[<abstract>...] + + + + + + + + + + + + + + + + () + + + + + + + + + + + ... + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex2.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex2.svg new file mode 100644 index 0000000..fcf84a1 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex2.svg @@ -0,0 +1,67 @@ + + #background: #fdf6e3 + [*]->[()] + [*]->[<abstract>....] + [()]->[<abstract>...] + + + + + + + + + + + + + + + + + + + + * + + + + + + + + + + + () + + + + + + + + + + + .... + + + + + + + + + + + ... + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex3.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex3.svg new file mode 100644 index 0000000..d15267e --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex3.svg @@ -0,0 +1,53 @@ + + #background: #fdf6e3 + [+]->[1] + [+]->[2] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + 2 + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex4.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex4.svg new file mode 100644 index 0000000..7642a42 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-building-ex4.svg @@ -0,0 +1,81 @@ + + #background: #fdf6e3 + [+]->[1] + [+]->[*] + [*]->[2] + [*]->[<abstract>...] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + * + + + + + + + + + + + 2 + + + + + + + + + + + ... + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-cycle.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-cycle.svg new file mode 100644 index 0000000..9a7371c --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-cycle.svg @@ -0,0 +1,240 @@ + + + + + #background: #fdf6e3 + [A]->[B] + [B]->[C] + [C]->[A] + + + + + + + + + + + + + + + + + + + + A + + + + + + + + + + B + + + + + + + + + + C + + + + + + + + + + + diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-if.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-if.svg new file mode 100644 index 0000000..06ddcf6 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-if.svg @@ -0,0 +1,179 @@ + + #background: #fdf6e3 + [if]->[>] + [>]->[a] + [>]->[b] + [if]->[body when true] + [body when true]->[=] + [=]->[c] + [=]->[5] + [if]->[body when false] + [body when false]->[+=] + [+=]->[d] + [+=]->[10] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + if + + + + + + + + + + + > + + + + + + + + + + + a + + + + + + + + + + + b + + + + + + + + + + + body when true + + + + + + + + + + + = + + + + + + + + + + + c + + + + + + + + + + + 5 + + + + + + + + + + + body when false + + + + + + + + + + + += + + + + + + + + + + + d + + + + + + + + + + + 10 + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/img/tree-simple.svg b/src/blog/sane-explanation-of-compilers/part-1/img/tree-simple.svg new file mode 100644 index 0000000..9f978a7 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/img/tree-simple.svg @@ -0,0 +1,81 @@ + + #background: #fdf6e3 + [+]->[*] + [*]->[1] + [*]->[2] + [+]->[3] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + * + + + + + + + + + + + 1 + + + + + + + + + + + 2 + + + + + + + + + + + 3 + + + + + + + + + \ No newline at end of file diff --git a/src/blog/sane-explanation-of-compilers/part-1/index.du b/src/blog/sane-explanation-of-compilers/part-1/index.du new file mode 100644 index 0000000..74ad490 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/index.du @@ -0,0 +1,1071 @@ +# Sane explanation of compilers: Part 1 + +'define expected-reading 48 min +'define created 30 March 2023 +'define edited 24 April 2023 +[$pagenav] + +:contents +:numbered +1. [url #preface Preface] +2. [url #i.-how-do-we-understand-code I. How do we understand code?] + :numbered + 1. [url #math Math] + .numbered + 1. [url #regular-expressions Regular expressions] + 2. [url #grammars Grammars] + + 2. [url #lexers-and-parsers Lexers and parsers] + :numbered + 1. [url #implementational-difficulties-with-grammars Implementational difficulties with grammars] + 2. [url #practical-usage Practical usage] + 3. [url #parsing-algorithms Parsing algorithms] + .numbered + 1. [url #ll LL] + 2. [url #lr LR] + 3. [url #lrs-family LR's family] + 4. [url #more-info More info] + + 4. [url #abstract-syntax-tree Abstract syntax tree] + + + 3. [url #implementing-a-front-end Implementing a front-end] + .numbered + 1. [url #lex lex] + 2. [url #yacc yacc] + 3. [url #doing-it-properly Doing it properly] + + +3. [url #appendix Appendinx] + + + +Recently I signed up for a small university course about compilers. +As I found out, most materials on the topic are either so surface level you barely learn anything, or so specific you need to already know the subject. +My goal is to try and fill that void, explaining everything in a detailed way that is understandable for beginners. + +## Preface + +Processors work with numbers, every action (instruction) a processor can do is defined by numbers. +However, writing code only with numbers is tedious, error-prone and simple stuff sometimes take ages to implement. +That is why, over time, programmers have tried to abstract sets of instructions into words and text constructions that are easier to work with. + +'footnote how-did-we-get-here Though it's obvious, I would like to point out that modern compilers are most often written in languages that are themselves compiled. You might ask yourself "where did the first compiler come from?" The answer is simple: it was written with "numbers", with direct instructions to the processor, from the time when computers were simple enough that *[all]* programming was done with CPU instructions. + +Thanks to abstraction, programmers have been able to create more and more complicated projects with less and less work, that is more portable and compatible with different machines. +The trade-off is that we need to translate our nice programmer code to numbers a CPU would understand. +This job is delegated to a special type of program, called a compiler[$how-did-we-get-here], which is separated into three main layers/components: + +.numbered +1. Front-end: it understands the program, what the code is trying to do +2. Middle: does optimizations and simplifications, though this part is often inside the back-end +3. Back-end: generates the actual machine instructions + +In this post, we'll be looking only at the first one. + +## I. How do we understand code? + +Syntax in current programming languages can be quite sophisticated and ambiguous. For example (taken from [url https://en.wikipedia.org/wiki/Most_vexing_parse wikipedia]) this C++ snippet + +``` +void f(double d) { + int i(int(d)); +} + +could be interpreted either as a: + +.numbered +1. declaration of variable `[i]`, which is initialized with the value of `[int(d)]` (which just casts `[d]` to an integer), or +2. function declaration, since superfluous parenthesis around function parameter declarations are allowed (meaning it is treated as `[int i(int d)]`) + +Note that I'm talking about syntax, the valid selection and layouts of characters, and not semantics, the underlying meaning. +Compiled programming languages associate specific templates of code (text) to their "corresponding" machine instructions, there isn't anything special you need to understand in a piece of code. +In natural languages, you could construct a sentence which follows all grammatical rules, but is absolute nonsense, like [url https://en.wikipedia.org/wiki/Colorless_green_ideas_sleep_furiously "Colorless green ideas sleep furiously"]. + +The problem of semantics is extremely difficult, however in our case we only care about syntax. +And this is where math comes into play. + +### Math + +A word of caution, I'll use a decent amount of fancy names, so I have a way to more easily refer to different concepts. +Don't be scared and skip over them if they're confusing you. + +There is a special field in mathematics that deals with syntactical analysis: [url https://en.wikipedia.org/wiki/Formal_language formal language theory]. +Although even the basics merit a whole university course, I'll still try to roughly explain what's important, for those that haven't studied it.\n +This field works primarily with strings via the help of two (relevant for our purposes) instruments: regular expressions and grammars. + +#### Regular expressions + +You might've heard or even already used regexes in a text editor. +They're small text expressions which specify a text pattern. +When a word can be "generated" from the pattern, we say the regex matches the word. + +At their very base, such a pattern is created with the following rules: + +:unordered +- [url https://en.wikipedia.org/wiki/Concatenation concatenation] - you can put characters next to each other to form strings, so a normal string like `[accent]` is also a regex.\n + With braces we can group stuff. + For example, in `[a(cc)ent]` the two "c"s are grouped together. + This is used to form more sophisticated structures later on, but currently it doesn't alter anything, like the braces in `[(1+2)+3]` and `[1+(2+3)]` don't alter the result sum. + +- [url https://en.wikipedia.org/wiki/Alternation_%28formal_language_theory%29 alternation] - allow different (alternative) strings in the same spot.\n + We do so with vertical bars (which signify an "or" relationship), the group can be "replaced" with either value. + For example `[acce(n | p)t]`, matches both "acce=[n]=t" or "acce=[p]=t".\n + There is no limit on how many alternatives you can have and what their lengths are: + + ```regex +acc(ordingly | ent | urate) + + matches + + .bulleted + - "acc=[ordingly]=" + - "acc=[ent]=" and + - "acc=[urate]=". + + There is no limit on how many groups (with alternation) you can have: + + ```regex +ac(quirem | c | hievem)en(t | ts) + + matches + + .bulleted + - "ac=[quirem]=en=[t]=" + - "ac=[quirem]=en=[ts]=" + - "ac=[c]=en=[t]= + - "ac=[c]=en=[ts]= + - "ac=[hievem]=en=[t]=" and + - "ac=[hievem]=en=[ts]=". + +- [url https://en.wikipedia.org/wiki/Kleene_star Kleene star] - repeat the preceding letter or group, 0 or more times.\n + An asterisk is used to mark this operation (hence the name, "Kleene =[star]="). + + ```regex +accents* + + will match + + .bulleted + - "accent" + - "accent=[s]= + - "accent=[ss]= + - "accent=[sss]= + - and so on + + Of course, the asterisk could also be in the middle of the regex, like + + ```regex +ac*ent + + which matches + + .bulleted + - "acent" + - "ac=[c]=ent" + - "ac=[cc]=ent" + - ... + + With a group we can repeat entire strings, + + ```regex +acc(ent)*s + + matches + + .bulleted + - "accs" + - "acc=[ent]=s" + - "acc=[entent]=s + - "acc=[ententent]=s" + - ... + + An alternation allows any alternative to be chosen at any point, so + + ```regex +acce(n | p)*t + + matches + + .bulleted + - "accet" + - "acce=[n]=t + - "acce=[p]=t + - "acce=[nn]=t + - "acce=[np]=t" + - "acce=[pn]=t" + - "acce=[pp]=t" + - "acce=[nnn]=t" + - ... + + +In the modern day, regular expressions permit [url https://en.wikipedia.org/wiki/Regular_expression#Syntax additional constructions] like matching the start or end of a string, matching character groups or character classes. +There are also sometimes features which significantly increase the matching capabilities of normal regexes, like [url https://en.wikipedia.org/wiki/Regular_expression#Patterns_for_non-regular_languages lookahead and lookbehind assertions]. + +#### Grammars + +However, even then regular expressions are fairly weak, they can't match recursive structures. +Let's say you want to be able to have math expressions in your language, you can have braces inside braces as much as you like, but in a certain manner: + +.bulleted +- `[1+2]` is a valid expression +- `[(9-8)+2]` also is valid +- `[((3*3)-(2^3))+2]` is too +- however `[()3*3)-(2^3()+2]` isn't valid + +Such a structure is recursive, because it can contain "itself", `[1+2]` can be a standalone math expression, but it can also be contained in another expression, like `[(1+2)*3]`, which itself could be inside another bigger expressions and so on. +For this task we have a whole separate tool: grammars. + +'footnote terminals-and-nonterminals Formally, we call the rule "names" nonterminals, the idea being that all normal characters, that could appear in our string, are called terminals, since they would terminate replacement and string generation. For the purposes of this post, I won't use the formal names, with the hope that using less terminology will increase understandability. + +The syntax is different from regexes: a grammar is divided into (production) rules, each rule is named and specifies an expression. +Expressions are a concatenation of strings and/or names[$terminals-and-nonterminals]. +Names won't appear in our matched strings, the idea is that a name will be replaced with it's expression, kind of like a placeholders. +Additionally, there can be multiple rules with the same name, meaning a rule can be replaced by different stuff at any point of the string generation. + +'footnote grammar-syntax This is pretty much the mathematical notation, often in CS the Backus-Naur form is used, but I find that a bit harder to read. You might also make the argument that I should use a proper unicode arrow symbol, like `[→]`, but `[->]` is way more convinient. + +Always one name is chosen to be the starting point, and generation continues from there. +In math literature, generally we write rules in the form `[Name -> Expression]` [$grammar-syntax], and often names are a single capital letter. +Additionally, often the starting name is called `[S]`. +Finally, in future examples, I'll use the arrow `[⇒]` to show derivations, how the generated string looks after one or two replacements of the name with one of it's expressions. + +For an example, with the grammar + +```grammar +S -> aSb +S -> cc + +we can generate the string `[aaaaccbbbb]` (remember, our starting point is `[S]`): + +```grammar+ +S ⇒ aSb ⇒ aaSbb ⇒ aaaSbbb ⇒ aaaaSbbbb ⇒ aaaaccbbbb + +or `[accb]`: + +```grammar+ +S ⇒ aSb ⇒ accb + +where a, b and c are just English characters. +To simplify our lives, often rules with the same name are merged into one rule, separated by vertical bars (`[S -> aSb | cc]`). +This is practically alternation but in a different context. + +As for our math expressions, we could create something like this: + +```grammar +N -> 0 | 1 | 2 | ... all possible numbers +E -> N | (E) | E + E | E - E | E * E | E / E | E ^ E + +which will successfully be able to match (generate) `[((3 * 3) - (2 ^ 3)) + 2]`: + +```grammar+ +E ⇒ E + E ⇒ (E) + N ⇒ (E - E) + 2 ⇒ ((E) - (E)) + 2 ⇒ ((E * E) - (E ^ E)) + 2 ⇒ ((N * N) - (N ^ N)) + 2 ⇒ ((3 * 3) - (2 ^ 3)) + 2 + +but there are no expressions (rules) that allow `[()3+3()]` or `[)3(-6]` to be "matched". + +### Lexers and parsers + +You might've noticed that for `[N]` I use three dots and the note "all possible numbers" to signify that N can be any positive whole number. +In practice we need formally define that grammar: + +```grammar +N -> 0 | 1 | 2 | ... | 9 | 0N | 1N | 2N | ... | 9N + +'footnote regular-grammars Such grammars are called [url https://en.wikipedia.org/wiki/Regular_grammar regular grammars], more formally, every rule has to contain at most one name and that name must be at the start or end of the string characters. + +With this you might've realised that grammars can do anything a regular expressions can: concatenation and alternation are "built-in", and the Kleene star can be done with another rule in the form `[R -> strR | str]` [$regular-grammars]. +The reason we use both is that grammars are significantly slower and more complicated in code than regexes. + +#### Implementational difficulties with grammars + +Implementing (very roughly) the regular expression `[acce(n | p)*t]` is quite easy: + +```c++ +'include ./regex_parser_simple.cpp + +In brief, for concatenated characters we do a simple comparison per character. +When we reach "\_", which I use to specify we're in the `[(n | p)*]` part of the regex, we do comparisons with the characters n and p as much as we can, after which we continue as normal. + +Of course, this version only supports one regular expressions, however a "proper" implementation would look conceptually similar: a couple of different loops and equality checks, with maybe an additional array or two. +The huge benefit is that that loops and arrays are stupid fast, a (simple) C++ for loop could be translated into [url https://godbolt.org/#g:!%28%28g:!%28%28g:!%28%28h:codeEditor,i:%28filename:'1',fontScale:14,fontUsePx:'0',j:1,lang:c%2B%2B,selection:%28endColumn:16,endLineNumber:6,positionColumn:16,positionLineNumber:6,selectionStartColumn:16,selectionStartLineNumber:6,startColumn:16,startLineNumber:6%29,source:'//+Type+your+code+here,+or+load+an+example.%0Aint+f%28int+num%29+%7B%0A++++for+%28int+i+%3D+1%3B+i+%3C+100%3B+i%2B%2B%29+%7B%0A++++++++num+*%3D+100%3B%0A++++%7D%0A++++return+num%3B%0A%7D'%29,l:'5',n:'0',o:'C%2B%2B+source+%231',t:'0'%29%29,k:50,l:'4',n:'0',o:'',s:0,t:'0'%29,%28g:!%28%28h:compiler,i:%28compiler:g122,deviceViewOpen:'1',filters:%28b:'0',binary:'1',binaryObject:'1',commentOnly:'0',demangle:'0',directives:'0',execute:'1',intel:'0',libraryCode:'0',trim:'1'%29,flagsViewOpen:'1',fontScale:14,fontUsePx:'0',j:1,lang:c%2B%2B,libs:!%28%29,options:'',selection:%28endColumn:1,endLineNumber:1,positionColumn:1,positionLineNumber:1,selectionStartColumn:1,selectionStartLineNumber:1,startColumn:1,startLineNumber:1%29,source:1%29,l:'5',n:'0',o:'+x86-64+gcc+12.2+%28Editor+%231%29',t:'0'%29%29,k:50,l:'4',n:'0',o:'',s:0,t:'0'%29%29,l:'2',n:'0',o:'',t:'0'%29%29,version:4 a handful of processor instructions]. + +On the other hand, usually grammars are more complicated and resource-intensive. +Certain ones are simple enough to implement, like `[S -> aSb | cc]`: + +```cpp +bool matchesp(string str) { + if (str == "cc") + return true; + else if (str[0] == 'a' && str[str.size() - 1] == 'b') + return matchesp(str.substr(1, str.size() - 2)); + else + return false; +} + +'footnote context-free-grammars Be careful about cases that are too complicated, the grammar I'm showing is "context-free", but that topic is way beyond the scope of this post. + +However think about more complicated cases[$context-free-grammars], like `[S -> aSbSa | cc]`. +Implementing this one is quite a bit more difficult, since we don't know where `[b]` is located on the first iteration, it could either be the 4th character `[accb...a]` or the 9th `[aaccbccab...a]`. +Searching for it is also complicated, it could be the first, the last or any other `[b]` in the middle of the string. + +Another difficulty with grammars is ambiguity, let's look at + +```grammar +E -> E + E | E * E | n + +how would it generate `[n + n * n]`? +There are two completely valid ways to do so: + +```grammar+ +E ⇒ E + E ⇒ n + E * E ⇒ n + n * n + +```grammar+ +E ⇒ E * E ⇒ E + E * n ⇒ n + n * n + +How exactly we deal with these problems comes later, for now the takeaway is that we have to try and use regular expressions wherever we can, and leave grammars as a bit of a last resort. + +#### Practical usage + +'footnote static-typing Yeah yeah, in a statically typed language you might not be able to do operations on doubles and integers at the same time, but you get the point. + +Let's think a bit more about grammars: they're really good at handling structure, especially recursive structure, so it's best if we use them only for that. +Our math expression grammar (from [url #grammars here]) shouldn't really care if we're trying to sum integers or floats or chars[$static-typing], it's important how everything is laid out and that our operations are done with numbers. + +This is why we're going to separate and generalize the code into [url https://en.wikipedia.org/wiki/Lexical_analysis#Token tokens], basically into building blocks. +As a rough example, we want `[1 + (2 * 3)]` to be transformed into `[NUMBER MATHOP LBRACE NUMBER MATHOP NUMBER RBRACE]`, `[int var = 13;]` into `[TYPE NAME ASSIGNMENT NUMBER]` and so on, where "TYPE", "NUMBER", "NAME", ... are tokens. +Now we're reducing the amount of "objects" our grammars need to work on: from all characters to many (but significantly fewer) tokens. +Also we're simplifying our work, since we're trying to figure out the overall structure, it doesn't matter if we're multiplying or summing numbers, it's important that we're doing a mathematical operation on them. +Of course, each token will actually be a pair, containing it's name (NUMBER, TYPE, ...) and it's value, but our grammars will work with the names only and output the actual values at the very end. +Usually we identify at least six token types: + +.bulleted +- identifiers: variable and function names for example +- keywords: `[if]`, `[while]`, `[for]`, `[return]`, ... +- separators: braces, `[;]` +- operators: `[+]`, `[-]`, `[<=]` +- literals: numbers, strings, characters +- comments + +As you might've noticed, here we can use regular expressions, grouping consecutive characters is what regexes are good for. +Additionally, we're going to be reading the file, a decently slow operation, only once and for a comparatively very little time. + +In the compiler front-end, the part that implements our regular expressions is called the "lexer", and the part that implements our grammars - the "parser". +In the last section we already talked about how easy implementing regexes (lexers) is and how difficult implementing grammars (parsers) is, let's look at the latter. + +#### Parsing algorithms + +Proper algorithms that "implement" grammars are kind of complicated, and most can handle only specific types of grammars. +Generally we categorize them into: + +.bulleted +- [url https://en.wikipedia.org/wiki/Top-down_parsing top-down] algorithms, which work in the way we've been talking about grammars: start with a name and replace it with it's expression, and continue replacements until we have no more names +- [url https://en.wikipedia.org/wiki/Bottom-up_parsing bottom-up] algorithms, where we do the opposite: for each substring we find from which rule it could've come from, replace it with the corresponding name, and continue upwards like that, until we reach the starting name + +Furthermore, if our algorithms allow for ambiguity (meaning we can replace a name with any one of multiple expressions), we divide them into: + +.bulleted +- left-most derivation algorithms, where we choose the left-most valid expression +- right-most derivation algorithms, where we chose the right-most valid expression + +The advantage and disadvantages of every type kind of depend on the exact algorithm and grammar. +Looking at some of the more famous ones: + +##### LL + +[url https://en.wikipedia.org/wiki/LL_parser LL]: top-down left-most derivations algorithm that can parse only specific types of grammars. +In these grammars, every expression is uniquely identifiable by just looking at the characters (tokens) in the very beginning. +The amount of token we need to look for is very important, so we usually define LL(k) as an LL grammar that needs to look at k amount of tokens. +An "LL(k) grammar" is, by extension, a grammar which can be parsed with an LL(k) parser. + +So, the way we'll check if an expression matches our grammar, is by starting with the start name and reading the next k character from the input. +We check the first k character of all expressions, and if anyone of them is a name, we try with substitution for every possibility with that name, essentially peeking into the name's possibilities. + +:unordered +- As an example, the grammar: + + ```grammar + 'snippet 1 2 ./grammar_trace_1.txt + + is LL(1). + Let's read the input `[(a + a)]` (the "v" points to the symbol we're looking/peeking at): + + ```grammar + 'snippet 4 8 ./grammar_trace_1.txt + + we start with `[S]`, we peek and get the `[(]`, in our production we're looking at a name, S. + Let's check the first character of every expression of S. + In the first expression, the first "character" is `[F]`, a name, so we look at every expression for `[F]`. + There is only one, which is "a", different than "(", no dice. + Now we check the second expression for S, it's first character is "(", success! + We replace: + + ```grammar + 'snippet 10 14 ./grammar_trace_1.txt + + Now the first symbol in our production is also `[(]`, so we increment the symbols we're looking/peeking at + + ```grammar + 'snippet 16 20 ./grammar_trace_1.txt + + we peek and get the `[a]`, in our production the second "character" is the name `[S]`, `[S]` has to become `[a]`. + We do that same check from before, first expression for `[S]` is `[F]`, whose first (and only) expression is `[a]`, success! + + ```grammar + 'snippet 22 26 ./grammar_trace_1.txt + + The middle characters " + " are matched and at the second `[F]` we check it's expressions, it has only `[a]`, success, replace, end of input, we're done. + + Another LL(1) example can be found in [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Top-down/LL1.html Karl R. Abrahamson notes]. + +- The grammar + + ```grammar +B -> b +C -> c +S -> aaB | aaC + + is LL(3). + For the input `[aab]`, if we look at the first two "a"s, we still don't know which rule to use: `[aaB]` or `[aaC]`. + However if we look at the third symbol, "b", then we can determine that we need `[aaB]`. + +- Usually parsers that implement this procedure are called [url https://en.wikipedia.org/wiki/Recursive_descent_parser recursive descent parsers] + + +##### LR + +[url https://en.wikipedia.org/wiki/LR_parser LR]: bottom-up right-most derivations algorithm. +Conceptually, LR works in the exact opposite way to LL: for every character in the input, we find a rule that substitutes it (or that substitutes the current collection of names and character) and do so until we've reached the starting name. + +Figuring out an approach isn't as intuitive and easy as with LL, but it still isn't too hard. +It all boils down to using a whole bunch of conditional statements and tracing where we "came" from. + +:unordered +- Let's look at the first LL example: + + ```grammar + 'snippet 1 2 ./grammar_trace_2.txt + + We need to track the current state of the string, because imagine we've just read an "a", it could either be from the left or right handside of "a + a", and that changes what character to expect next. + We're going to track the state with different numbers, defined like this: + + .table + |=State (number)=||=Current expression ends with=| + |:0 :||:Nothing; this is the start:| + |:1 :||:The name `[S]`; the final number should be this:| + |:2 :||:The name `[F]`:| + |:3 :||:The character "(":| + |:4 :||:The character "a":| + |:5 :||:"(S":| + |:6 :||:"(S + ":| + |:7 :||:"(S + F":| + |:8 :||:"(S + F)"; this is the one before the final:| + + You can think of every state number as the valid, but "unfinished" state of the current string. + For now don't think about the way this table is defined, it'll make sense later. + But you could make a state for literally every valid string composition: "(", "(a", "(F", "(S", .... + + Keep in mind that the way I'll show the algorithm step by step isn't really correct, but will suffice for the current moment. + Reading a character from the input means appending it to the right of String: + + ``` + 'snippet 4 5 ./grammar_trace_2.txt + + We've read "(", so our string "ends" on a "(". + + ``` + 'snippet 7 8 ./grammar_trace_2.txt + + We've read "a", our string "ends" on an "a". + + ``` + 'snippet 10 11 ./grammar_trace_2.txt + + Here comes the clever bit, for certain states, we'll stop reading forward and do something else. + If we're on state 4, our string ends on an "a", so we're going to go backwards and replace it with an F, meaning the current state should be 2. + + ``` + 'snippet 13 14 ./grammar_trace_2.txt + + Here is another such rule, if our state is 2, our string ends with an "F" and we replace it with an S, now the state is 5. + That's why we have a different state for "(S + F", if we didn't, our state would become 2 and we would make "(S + S", which isn't valid. + So, think of 2 as "this is the first F". + + ``` + 'snippet 16 17 ./grammar_trace_2.txt + + We have a different state for "(S", because if we didn't, we would've gone to state 1, and could've ended the algorithm then and there, without reading the rest of the input. + Otherwise, we continue as normal, we read the `[+]` and the spaces around it, going to state 6. + + ``` + 'snippet 19 20 ./grammar_trace_2.txt + + Now, our string ends with an "a", it's tempting to change our state to 4, but we shouldn't, this would remove our progress so far! + Better yet, this state will also be special, if we have a 6 and we end with "a", rather than going to 4, we'll substitute "a" with F now and change state to 7. + + ``` + 'snippet 22 23 ./grammar_trace_2.txt + + And now it's trivial, we read ")", so we go to state 8. + But if we're at state 8, don't continue reading, replace with state 1. + Now that we're at state 1, if we read any character (including end of string), we'll say the string up to now matches and we're done! + + It's also helpful to put what we do in different situations in another table: + + .table + |=State=||=If ends on=||=Change state to=||=Also do=| + |:0 :||:"(" :||:3 :||: :| + |:1 :||: :||: :||:Nothing, we matched:| + |:2 :||: :||:5 :||:Replace "F" with "S":| + |:3 :||:"a" :||:4 :||::| + |:4 :||: :||:2 :||:Replace "a" with "F":| + |:5 :||:" + " :||:6 :||: :| + |:6 :||:"a" :||:7 :||:Replace "a" with "F":| + |:7 :||:")" :||:8 :||: :| + |:8 :||: :||:1 :||:Replace "(S + F)" with "S":| + + where, if there is a value in "If ends on", we read the next character, while otherwise we don't and we just execute "Change state to" and "Also do". + +- You're probably wondering why the table works so weird, like why on state 3 we don't directly replace "a" with "S" and change to state 5 for example. + The reason is that the way I showed the algorithm isn't exactly how it normally works. + Rather than tracing the current state in a variable, we trace it with a [url https://en.wikipedia.org/wiki/Stack_%28abstract_data_type%29 stack]. + + When you read a character and have to go to another state, the state is added to the stack (this operation is called "shift"). + When we have to change characters from String (without reading anything), with a string (expression) of length j, we remove the last j added states and do a normal shift from the last state and character in String (this operation is called "reduce"). + +- Let's look at the same example, done with shift and reduce operations (without me showing the proper table), where pushing and popping items from the stack is done from the right side of the array: + + ``` + 'snippet 25 26 ./grammar_trace_2.txt + + We've read "(", add 3 on top of the stack aka shift(3). + Then we read the "a", shift(4). + + ``` + 'snippet 28 29 ./grammar_trace_2.txt + + With 4 we'll do a reduction, "a" gets replaced by F, 4 is removed: + + ``` + 'snippet 31 32 ./grammar_trace_2.txt + + If last state is 3 and last character is "F", we're doing shift(2). + + ``` + 'snippet 34 35 ./grammar_trace_2.txt + + Now, we'll be doing a reduction, where F is replaced by S: + + ``` + 'snippet 37 38 ./grammar_trace_2.txt + + If last state is 3 and last character is "S", we're dong shift(5). + + And so on, we continue. + As you can see, we go back and forth, from 3 to 4, back to 3, then to 2, back to 3, then to 5. + This way we avoid doing different things on different states and we don't need to store the whole string, only the last few characters. + Additionally, in the real world nobody makes this table by hand, it is generated by a program, and this method makes table generation easier. + +- As for the complete table, you can view it on the [url https://smlweb.cpsc.ucalgary.ca/lr0.php?grammar=S+-%3E+F+%7C+%28+S+%2B+F+%29.%0AF+-%3E+a.&substs= Context Free Grammar Checker],\n + or with a more pretty styling on [url https://mdaines.github.io/grammophone/?s=UyAtPiBGIHwgIigiIFMgIisiIEYgIikiLgpGIC0+IGEu#/ Grammophone], opening "Parsing table" on the row "LR(0)". + + To read the table, remember that: + + .bulleted + - on every row you have each state + - on every column you have each possible value in the string and "$" which signifies the end of the string + - with `[sr]` or `[shift]` (or just with a number) you go to another state + - with `[r]` or `[reduce]` you make the substitution in the current string and go back one state + - `[acc]` or `[accept]` means you've reached the end and the string is valid + + +##### LR's family + +LR is more of a family of algorithms, because there are multiple different ways to generate the table, each with it's own benefits and drawbacks. +This subject is a really big can of beans, which I'm not knowledgeable enough to talk about. +What you need to know is that there are 3 often used types: [url https://en.wikipedia.org/wiki/Canonical_LR_parser CLR], [url https://en.wikipedia.org/wiki/Simple_LR_parser SLR] and [url https://en.wikipedia.org/wiki/LALR_parser LALR]. + +.bulleted +- CLR is an LR parser with 1 character lookahead (LR(1)), following [url TODO Knuth]'s original theory. + It requires a lot of memory for table generation, so Frank DeRemer created two simplifications: +- the SLR and LALR, which use significantly less memory but work with a bit less grammars + +LALR is the most commonly used LR parser, however most compilers are custom-tailored pieces of software that very loosely follow the methods of LL and LR. + +##### More info + +If you want to learn more, I would highly suggest: + +.bulleted +- [url https://blog.reverberate.org/2013/07/ll-and-lr-parsing-demystified.html LL and LR Parsing Demystified] and [url https://blog.reverberate.org/2013/09/ll-and-lr-in-context-why-parsing-tools.html LL and LR in Context: Why Parsing Tools Are Hard] by [url https://blog.reverberate.org/about/ Josh Haberman] +- [url http://jeffreykegler.github.io/Ocean-of-Awareness-blog/individual/2014/11/ll.html Parsing: Top-down versus bottom-up] by [url https://jeffreykegler.github.io/personal/ Jeffrey Kegler] +- [url https://tratt.net/laurie/blog/2020/which_parsing_approach.html Which Parsing Approach?] by [url https://tratt.net/laurie/ Laurence Tratt] +- [url https://tomassetti.me/guide-parsing-algorithms-terminology/ A Guide to Parsing: Algorithms and Terminology] by [url https://inre.me/ Gabriele Tomassetti] +- [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Parsing/index.html Parsing], [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Top-down/index.html Top-Down Parsers] and [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Bottom-up/index.html Bottom-Up Parsers] from [url http://www.cs.ecu.edu/karl/5220/spr16/Notes/Bottom-up/index.html Karl R. Abrahamson]'s notes +- [url https://serokell.io/blog/how-to-implement-lr1-parser How to Impement an LR(1) Parser] by Kirill Andreev + +#### Abstract syntax tree + +We already decided that the lexer will output tokens (and characters) to the parser, but we didn't talk about what the parser is supposed to output. +It definitely has to be some sort of structural representation of the overall code, for example, for `[3 + 1 * 2]` we have to be able to easily find out what the left and right handsides of the `[+]` and `[*]` are. + +'footnote nomnoml The graphics are made with [url https://nomnoml.com/ nomnoml] by [url https://github.com/skanaar Daniel Kallin] + +Let's draw out[$nomnoml] this relationship, where the left arrow connects to the left handside and the right accordingly: + +[image ./img/tree-simple.svg] + +Hey, that looks an awful lot like a [url https://en.wikipedia.org/wiki/Tree_%28data_structure%29 tree]! +Trees are nodes, some sort of elements, where there is a connection from one element to some other elements, with the only rule being that cycles =[aren't]= allowed: + +[image ./img/tree-cycle.svg] + +Great, but can we represent everything else with a tree? +Why, of course, for example we can represent: + +```cpp +if (a > b) + c = 5; +else + d += 10; + +as + +[image ./img/tree-if.svg] + +You might ask how we generate the tree, and the answer is simple: on every step we add nodes to the tree, depending on the situation. +Let's look at a familiar example: + +```grammar +N -> 1 | 2 | 3 +E -> N | (E) | E + E | E * E + +Let's look at the input string `[2 * (1 + 3)]`, while abstracting ourselves from any particular algorithm: + +:unordered +- we read the `[2]`, that is a number, we add it as a node + + .p [image ./img/tree-building-1.svg] + +- then we read the `[*]`, we now know our expression is in the form `[E* E]`. + If you think about it, up to this point, we've basically been building a tree for the left handside. + + =[If]= the topmost element in that tree is a brace, + + .p [image ./img/tree-building-ex1.svg] + + we connect it to the left handside of `[*]`: + + .p [image ./img/tree-building-ex2.svg] + + =[Otherwise]= (when it's `[+]` or `[*]` or a number), because of precedence, we'll attach the latest number to the left handside. + However we have to be careful, if something connects to the number, we now need it to connect to the `[*]`.\n + For example, let's say we had `[1 + 2 * 3]`, and we've just read the `[*]`. + The tree up to this point would look like, + + .p [image ./img/tree-building-ex3.svg] + + where the latest number we've found was `[2]`, but the `[+]` connects to it. + So, we need to make the `[2]` connect to the left handside of `[*]`, and make `[+]` connect to `[*]` instead of `[2]` + + .p [image ./img/tree-building-ex4.svg] + + In our original case, nothing connects to the `[2]` in `[2 * (1 + 3)]`, so we can just connect it to the left handside of `[*]` + + .p [image ./img/tree-building-2.svg] + + where the three dots signify there isn't anything there (but we expect there to be). + Somewhere we'll store that the next element we find has to be connected to the right handside. + +- now we read `[(]`, we cannot have it by itself, so we directly decide we're working with `[(E)]` (in our full algorithm we would need a check for existance of matching brace). + We'll create a `[()]` node and because of the last step, we'll connect it to the right handside. + + .p [image ./img/tree-building-3.svg] + + Now, moving forward, we'll be creating a subtree, essentially making a second tree, and after that is done, we'll connect it's topmost element (root) to the `[()]` node. + +- and so on. + + +The overall algorithm is to add and change nodes every time you match something with a grammar rule, and the adding and changing is done, depending on the language, the current structure and so on. + +When discussing the middle and back-end layers, we'll talk about how to work with a generated AST, but for now your clue is the [url https://en.wikipedia.org/wiki/Visitor_pattern visitor pattern]. + +### Implementing a front-end + +If you want, you could implement your own lexer and parser manually, but as a start, it's worthwhile to try and use already created stuff. +[url https://en.wikipedia.org/wiki/Compiler-compiler Compiler-compilers] are tools that create parsers (and more) from their normal formal definition. + +The two popular ones from the old days are lex and yacc. +lex is a lexer generator and yacc is a LALR parser generator, both of which generate a corresponding C implementation of the given regular expressions and grammars. + +#### lex + +Lex is the original program, from 1975, in the modern day it's successor/implementation is the open-source [url https://github.com/westes/flex flex]. + +A lex source file is divided into three parts: + +```lex +'include ./template.l + +where `[%{%}]` is used for includes and declarations, the rules are your regular expressions and the C code is the general logic of the lexer. +Usually you wouldn't write anything in the C code section, the defaults are fine, but you'll probably need to do one or two includes in the `[%{%}]` section. + +Rules are in the form `[regex { C_code }]`. +To simplify your life, you can write only the regex or a part of it inside the "definitions" part of the file, and give it a name to use later. +Referencing those names in later regexes is done by surrounding them in curly braces. +Usually the `[C_code]` in the regex would be a return (more commonly called "emit") of a token, which will be available in a header file, but more on that later. + +Let's look at a very simple example program: every whole number is printed as "NUMBER", while everything else is printed as-is: + +```lex +'include ./lexer.l + +The syntax `[[0-9]]` is the same as `[(0 | 1 | ... | 9)]`, it works for any letters and is available in all modern regexes. +Note that inside every regex's "body" you have an assortment of variables already set, one of them being `[yytext]` which is just the whole match. +Additionally lex matches the longest possible match, so if we had a rule `[[0-z]]`, only it will be matched. + +If this file is located in `[lexer.l]`, we can transform it into a C file with: + +```bash +flex lexer.l + +Usually the result file is always named `[lex.yy.c]`. +Afterwards, we can compile the C code into a working compiler: + +```bash +gcc lex.yy.c -lfl + +Without the `[-lfl]` flag, we would get a linking error, because of the `[main]` and `[yywrap]` function, which are required but aren't defined in our generated code. +There are three ways to fix this problem: + +:numbered +1. The one I've opted for, use the `[-lfl]` flag to compile our C code with a (flex) library which provides both functions. + Usually the lexer isn't meant to be ran by itself, so for the current time being, `[-lfl]` is the best option, the other two are fine when we get to `[yacc]`. + +2. Put `[%option noyywrap]` in the definitions section, which means the lexer will only read one file. + +3. Define it ourselves. + It should return 0 if our lexer should continue with the next given file, after finishing with the current one, or 1 if it should stop execution. + The function could be written in either "C code" section, but I personally prefer the last one. + + ```lex +int yywrap() { return 0; } + + +Of course, you can use your lex implementation or C compiler of choice, and there this might not be a problem. +Now, if you run `[./a.out]` you would get a prompt, every line you write will be lexically analyzed and in our case, everything will be printed out. +Entering + +``` +The number 10 changed to 8, 38, and 19 sequentially, before becoming 0. + +will output + +``` +The number NUMBER changed to NUMBER, NUMBER, and NUMBER sequentially, before becoming NUMBER. + +Now let's try something more sophisticated, let's surround numbers and strings in HTML tags, the stuff that power webpages. +We would want numbers to be in the format: + +``` +number + +and strings: + +``` +string + +We'll define numbers as both whole digits and floating point numbers (with sign) and strings can be surrounded in either single or double quotes. + +```lex +'include ./example_lex/numbers_html.l + +With the input + +``` +John said 10 times "2 + 2 = 5" but '2 + 2 = 4'! + +we'll get + +``` +John said 10 times "2 + 2 = 5" but '2 + 2 = 4'! + +You've probably already noticed that the number inside the strings aren't matched, because the string rule results in a longer match. +In certain programming languages, that don't have [url https://en.wikipedia.org/wiki/String_interpolation string interpolation] this is fine, but for others our regexes have to be a lot more complicated. + +#### yacc + +Similarly to lex, yacc originates from 1975, and it's modern replacement is [url https://www.gnu.org/software/bison/ GNU Bison]. + +A yacc file looks very similarly to a lex one: + +```yacc +'include ./template.y + +however rather than definitions you have declarations, and the rules are in a completely different format (because they're grammars). + +Token names are defined in `[declarations]`, in the format + +```yacc +%token NAME1 NAME2 ... + +You can also define precedence with `[%left]` and `[%right]`. +There are also other options you can set, like the start name with `[%start]`. + +Grammars are in the form + +```yacc +name : TOKEN1 TOKEN2 ... + ; + +where alternation is defined like + +```yacc +name : TOKEN1 + | TOKEN2 + | TOKEN3 + ; + +and it always must end with a `[;]`. +On each alternation, you can (should) specify a body, which will contain C code to be executed, if this rule is reached + +```yacc +name : TOKEN1 { C code here } + | TOKEN2 + | TOKEN3 { C code here } + ; + +Another difference is that, since the parser needs a list of tokens, you cannot run it by itself, you'll also need to setup a lexer. + +'footnote shift-reduce-conflicts You might be tempted to try and make one for the grammar inside the [url #abstract-syntax-tree AST] section, but that one is ever so slightly problematic, because it's ambiguous (I showed why at the end of [url #implementational-difficulties-with-grammars Implementational difficulties with grammars]). Bison [url https://www.gnu.org/software/bison/manual/html_node/Shift_002fReduce.html can handle ambiguity by always choosing to shift], but I didn't want to complicate things. + +Let's implement a parser for the grammar[$shift-reduce-conflicts]: + +```grammar +N -> 1 | 2 | 3 +S -> N | (S + S) | (S * S) + +First, our tokens, we want to replace all characters with a token. +For 1, 2 and 3, let's use the token name "NUMBER", for "(" and ")", LBRACE and RBRACE accordingly, and since there is no precedence with this grammar, we don't actually care if we have `[(S + S)]` or `[(S * S)]`, so we can assign both `[+]` and `[*]` to the token MATHOP. +Meaning, our grammar would behave more like this: + +```grammar +N -> 1 | 2 | 3 +M -> + | * +S -> N | (S M S) + +First, we need to make our `[lexer.l]`, it is going to be very simple: + +```lex +'include ./example_yacc/lexer.l + +Now, tokens (names) are defined inside grammars, however our lexer needs to emit (return) them. +That's why, when running bison on our `[parser.y]`, we'll also create a C header file, which contains those names already defined, so they're accessible inside `[lexer.l]`. +This file is in the format `[FILENAME.tab.h]`, where `[FILENAME.y]` is our source yacc file. + +Second, we need to make our `[parser.y]`: + +```yacc +'include ./example_yacc/parser.y + +A quirk with GNU Bison is that you need to declare the function `[yylex]` and `[yyerror]` beforehand, manually define `[yyerror]` and run `[yyparse]` in the main function. +These are precisely the first 5 and the last 8 lines of code, but for now you don't need to think about them. + +Let's get to compiling. +First, the lexer: + +```bash +flex lexer.l + +then our parser: + +```bash +bison -d parser.y + +where the `[-d]` option means we'll also output the `[parser.tab.h]` header file. +Finally, we compile both of them together into one executable: + +```bash +gcc lex.yy.c parse.tab.c -lfl + +Now when you run `[./a.out]`, if you enter a valid string, you'll see a blank line, but if it isn't valid, you'll get "Error: syntax error". + +#### Doing it properly + +The way we created that lexer and parser worked, but it wasn't very useful and has multiple issues we have to address: + +:ordered +1. When entering a string that isn't matched at all, it gets printed back like nothing happened. + For example, if we enter the letter `[a]` or any string which doesn't have `[(]`, `[)]`, `[1]`, `[2]`, `[3]`, `[+]` or `[-]` characters, it will just get printed back. + Ideally, we want to raise an error, so we're going to add one rule which catches any (non-special) character and throws an error with the built-in `[yy_fatal_error]`: + + ```lex +. { yy_fatal_error("Bad character"); } + + This rule must be the last rule, since we don't want it to be matched, if there is another rule that matches. + Remember that lex matches the longest possible string and if there are multiple rules which match the same amount of characters, only the first one is matched. + +2. For every token name, returned by the lexer, we also need to have it's actual value. + We could make a `[struct]`, which holds both a token name and it's value, but there is another [url https://www.gnu.org/software/bison/manual/html_node/Token-Values.html "built-in" mechanism] for that in yacc/Bison. + We would set the global variable `[yylval]` to the token value, and it will be changed for every token. + + By default it's type is `[int]` (in Bison), but usually you would want it to be a [url https://en.wikipedia.org/wiki/Union_type#C/C++ union]. + Let's make it a union of int and char, though the first part is kind of useless, since we're only working with single digit numbers. + \n + ```yacc +%union { + int num; + char chr; +} + + Now, inside our lexer, rules would look like: + + ```yacc +RULE1 { yylval.num = 129; return TOKEN; } +RULE2 { yylval.chr = 'a'; return TOKEN; } + + Inside C blocks for grammar rules we can reference the token values with `[\$i]`, where `[i]` is the i'th "token" in the expression. + For example, in this rule, `[\$1]` would be `[(]`, `[\$3]` would be value of token `[B]` and `[\$5]` would be the value of the rule name: + + ```yacc +rule : ( A B ) rule + ; + + You can set rule "return" values by setting the variable `[\$$]`: + + ```yacc +rule : LBRACE rule RBRACE { $$ = 10; } + + The way you set it's type is discussed later. + However, `[\$i]` isn't of type union, we cannot do something like `[\$i.num]`, it is the precise value that was set for the token. + Therefore, we also need to define which value from the union every token will set, which is done by adding `[]` before every defined token (and `[PROPERTY]` is a property name inside the union): + + ```yacc +%token NUMBER MATHOP LBRACE RBRACE + + 'footnote better-bison-errors You can add `[%define parse.error verbose]` inside definitions of your yacc file, and it will tell you what character was expected and what character was read. + +3. If we enter one valid math expression, we'll get no error, but when we enter another, we'll get a "syntax error"[$better-bison-errors]. + That is because we try to match only one expression, after it is matched our parser is waiting for an end of file character. + Let's make it so we would try to match an expression on every line: + + ```yacc +start : expr + | expr start + ; + + A rule like `[expr : expr expr]` will produce shift/reduce conflicts. + Another option is duplicating our `[expr]` rules and adding `[expr]` in the beginning or end, but that is cumbersome if we want to make changes. + +4. We're getting input from the user, but our source text (code) will be in a file. + First, our `[main]` function needs to take program parameters: + + ```yacc +int main(int argc, char* argv[]) + + Then we need to take the first argument, and open a file. + Additionally, we have to check if the file was successfully opened, and print an error and stop execution if so: + + ```yacc +FILE* input = fopen(argv[1], "r"); +if (input == NULL) { + yyerror("cannot read input file!"); + return 1; +} + + Now for the clever bit, actual parsing in generated code is done on the file `[yyin]`, which by default is the standard user input. + To read a file, all we need to do is change the value of that variable. + Also, that variable is defined externally, so first we need to add to (the top) C section: + + ```yacc +extern FILE* yyin; + + and then put in our `[main]`: + + ```yacc +yyin = input; + + On \*NIX systems (Linux, MacOS, OpenBSD, ...) you can still use user input by passing the file `[/dev/stdin]`. + If you want your front-end to support multiple files, you would just need to wrap the whole body of main in a `[for]` loop, iterating over the whole `[argv]` array (remember that `[argc]` is the length of the array). + +5. We need to construct an AST. + Since the exact tree structure depends on your needs, I'll make the simplest possible tree in pure C that works for this grammar. + I won't explain this in much detail, since it is just normal C code. + + ```c +#include \ +#include \ + 'snippet 2 $ ./proper/tree.c + + However, all of this is quite a lot, and it would take a lot of space in `[parser.y]`, so let's separate it into `[tree.c]` and add a header file `[tree.h]` which contains the definitions: + + ```c + 'snippet 4 15 ./proper/tree.h + + Of course, we should add some ifndef-def precompiler code, but that isn't relevant for now. + + The way we're going to use trees in our grammar is as follows: for the `[NUMBER]` expression of `[expr]`, we'll create a node with the (char) value of the number and "return" it. + For the expression of `[expr]` we'll "return" a node with the value of the math operator and left and right will be the returned values. + Inside start, after we've parsed an `[expr]`, we'll print out the "returned" tree and free it's memory. + + I'm putting returned in quotes, because (as mentioned), we'll just be setting the value of `[\$$]`. + `[\$$]`'s type is set with the `[%type]` option, which works exactly like `[%token]`, but with grammar rule names. + However, this means that `[node]` must be available in `[%union]`, which is unfortunate but isn't a problem, we'll just add it. + + Now, finally, it's time to include our `[tree.h]` in `[parser.y]`, but we'll also need to include it in `[lexer.l]`. + The reason is that the `[%union]` appears in `[parser.tab.h]`, which means our node structure is referenced in `[parser.tab.h]`, but `[tree.h]` isn't included there. + + +All in all, we'll have 4 source files for our simple front-end: + +:unordered +- `[lexer.l]` + + ```lex + 'include ./proper/lexer.l + +- `[parser.y]` + + ```yacc + 'include ./proper/parser.y + +- `[tree.h]` + + ```c + 'include ./proper/tree.h + +- `[tree.c]` + + ```c + 'include ./proper/tree.c + + +We compile everything: + +```bash +flex lexer.l && bison -d parser.y && gcc lex.yy.c parser.tab.c tree.c + +Now, let's create the file `[test.txt]`: + +``` +'include proper/test.txt + +and run our front-end on it: + +```bash +./a.out test.txt + +Our output will be: + +``` +'include proper/test_output.txt + +Perfect, we have a good looking tree! + +## Appendix + +Originally I planned to put everything in one blog post, but writing the front-end (this part) alone took me 2 and a half weeks of (inconsistent, I was suddenly hit with multiple exams and homeworks in uni) writing. +Additionally, while doing research, I found out the course actually skipped over a lot of important topics, which I'll first need to research. +Future parts will be linked right below the title at the very top, but it will probably be quite a while until I write them. + +Many thanks to [url https://github.com/dtagarev Dimitar Tagarev] for his help with reviewing this post. diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/lex.yy.c b/src/blog/sane-explanation-of-compilers/part-1/proper/lex.yy.c new file mode 100644 index 0000000..79e7294 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/lex.yy.c @@ -0,0 +1,1771 @@ + +#line 2 "lex.yy.c" + +#define YY_INT_ALIGNED short int + +/* A lexical scanner generated by flex */ + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 6 +#define YY_FLEX_SUBMINOR_VERSION 4 +#if YY_FLEX_SUBMINOR_VERSION > 0 +#define FLEX_BETA +#endif + +/* First, we deal with platform-specific or compiler-specific issues. */ + +/* begin standard C headers. */ +#include +#include +#include +#include + +/* end standard C headers. */ + +/* flex integer type definitions */ + +#ifndef FLEXINT_H +#define FLEXINT_H + +/* C99 systems have . Non-C99 systems may or may not. */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + +/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, + * if you want the limit (max/min) macros for int types. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include +typedef int8_t flex_int8_t; +typedef uint8_t flex_uint8_t; +typedef int16_t flex_int16_t; +typedef uint16_t flex_uint16_t; +typedef int32_t flex_int32_t; +typedef uint32_t flex_uint32_t; +#else +typedef signed char flex_int8_t; +typedef short int flex_int16_t; +typedef int flex_int32_t; +typedef unsigned char flex_uint8_t; +typedef unsigned short int flex_uint16_t; +typedef unsigned int flex_uint32_t; + +/* Limits of integral types. */ +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#ifndef SIZE_MAX +#define SIZE_MAX (~(size_t)0) +#endif + +#endif /* ! C99 */ + +#endif /* ! FLEXINT_H */ + +/* begin standard C++ headers. */ + +/* TODO: this is always defined, so inline it */ +#define yyconst const + +#if defined(__GNUC__) && __GNUC__ >= 3 +#define yynoreturn __attribute__((__noreturn__)) +#else +#define yynoreturn +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an + * integer in range [0..255] for use as an array index. + */ +#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN (yy_start) = 1 + 2 * +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START (((yy_start) - 1) / 2) +#define YYSTATE YY_START +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#ifndef YY_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k. + * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. + * Ditto for the __ia64__ case accordingly. + */ +#define YY_BUF_SIZE 32768 +#else +#define YY_BUF_SIZE 16384 +#endif /* __ia64__ */ +#endif + +/* The state buf must be large enough to hold one state per character in the main buffer. + */ +#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif + +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + +extern int yyleng; + +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + + #define YY_LESS_LINENO(n) + #define YY_LINENO_REWIND_TO(ptr) + +/* Return all but the first "n" matched characters back to the input stream. */ +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + *yy_cp = (yy_hold_char); \ + YY_RESTORE_YY_MORE_OFFSET \ + (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) +#define unput(c) yyunput( c, (yytext_ptr) ) + +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + int yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + int yy_bs_lineno; /**< The line count. */ + int yy_bs_column; /**< The column count. */ + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; + +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + + }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ + ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ + : NULL) +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. + */ +#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; +static int yy_n_chars; /* number of characters read into yy_ch_buf */ +int yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = NULL; +static int yy_init = 0; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart ( FILE *input_file ); +void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); +void yy_delete_buffer ( YY_BUFFER_STATE b ); +void yy_flush_buffer ( YY_BUFFER_STATE b ); +void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state ( void ); + +static void yyensure_buffer_stack ( void ); +static void yy_load_buffer_state ( void ); +static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file ); +#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER ) + +YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); +YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len ); + +void *yyalloc ( yy_size_t ); +void *yyrealloc ( void *, yy_size_t ); +void yyfree ( void * ); + +#define yy_new_buffer yy_create_buffer +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ + } +#define yy_set_bol(at_bol) \ + { \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ + } +#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) + +/* Begin user sect3 */ + +#define yywrap() (/*CONSTCOND*/1) +#define YY_SKIP_YYWRAP +typedef flex_uint8_t YY_CHAR; + +FILE *yyin = NULL, *yyout = NULL; + +typedef int yy_state_type; + +extern int yylineno; +int yylineno = 1; + +extern char *yytext; +#ifdef yytext_ptr +#undef yytext_ptr +#endif +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state ( void ); +static yy_state_type yy_try_NUL_trans ( yy_state_type current_state ); +static int yy_get_next_buffer ( void ); +static void yynoreturn yy_fatal_error ( const char* msg ); + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + (yytext_ptr) = yy_bp; \ + yyleng = (int) (yy_cp - yy_bp); \ + (yy_hold_char) = *yy_cp; \ + *yy_cp = '\0'; \ + (yy_c_buf_p) = yy_cp; +#define YY_NUM_RULES 7 +#define YY_END_OF_BUFFER 8 +/* This struct is not used in this scanner, + but its presence is necessary. */ +struct yy_trans_info + { + flex_int32_t yy_verify; + flex_int32_t yy_nxt; + }; +static const flex_int16_t yy_accept[12] = + { 0, + 0, 0, 8, 6, 7, 5, 3, 4, 2, 1, + 0 + } ; + +static const YY_CHAR yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 1, 1, 1, 1, 1, 1, 1, 4, + 5, 6, 6, 1, 1, 1, 1, 1, 7, 7, + 7, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + } ; + +static const YY_CHAR yy_meta[8] = + { 0, + 1, 1, 1, 1, 1, 1, 1 + } ; + +static const flex_int16_t yy_base[12] = + { 0, + 0, 0, 8, 9, 9, 9, 9, 9, 9, 9, + 9 + } ; + +static const flex_int16_t yy_def[12] = + { 0, + 11, 1, 11, 11, 11, 11, 11, 11, 11, 11, + 0 + } ; + +static const flex_int16_t yy_nxt[17] = + { 0, + 4, 5, 6, 7, 8, 9, 10, 11, 3, 11, + 11, 11, 11, 11, 11, 11 + } ; + +static const flex_int16_t yy_chk[17] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 3, 11, 11, + 11, 11, 11, 11, 11, 11 + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +extern int yy_flex_debug; +int yy_flex_debug = 0; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "lexer.l" +#line 2 "lexer.l" +#include "tree.h" +#include "parser.tab.h" +#line 451 "lex.yy.c" +#line 452 "lex.yy.c" + +#define INITIAL 0 + +#ifndef YY_NO_UNISTD_H +/* Special case for "unistd.h", since it is non-ANSI. We include it way + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +#include +#endif + +#ifndef YY_EXTRA_TYPE +#define YY_EXTRA_TYPE void * +#endif + +static int yy_init_globals ( void ); + +/* Accessor methods to globals. + These are made visible to non-reentrant scanners for convenience. */ + +int yylex_destroy ( void ); + +int yyget_debug ( void ); + +void yyset_debug ( int debug_flag ); + +YY_EXTRA_TYPE yyget_extra ( void ); + +void yyset_extra ( YY_EXTRA_TYPE user_defined ); + +FILE *yyget_in ( void ); + +void yyset_in ( FILE * _in_str ); + +FILE *yyget_out ( void ); + +void yyset_out ( FILE * _out_str ); + + int yyget_leng ( void ); + +char *yyget_text ( void ); + +int yyget_lineno ( void ); + +void yyset_lineno ( int _line_number ); + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap ( void ); +#else +extern int yywrap ( void ); +#endif +#endif + +#ifndef YY_NO_UNPUT + + static void yyunput ( int c, char *buf_ptr ); + +#endif + +#ifndef yytext_ptr +static void yy_flex_strncpy ( char *, const char *, int ); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen ( const char * ); +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput ( void ); +#else +static int input ( void ); +#endif + +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k */ +#define YY_READ_BUF_SIZE 16384 +#else +#define YY_READ_BUF_SIZE 8192 +#endif /* __ia64__ */ +#endif + +/* Copy whatever the last rule matched to the standard output. */ +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ + { \ + int c = '*'; \ + int n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else \ + { \ + errno=0; \ + while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + { \ + if( errno != EINTR) \ + { \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + break; \ + } \ + errno=0; \ + clearerr(yyin); \ + } \ + }\ +\ + +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* end tables serialization structures and prototypes */ + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL_IS_OURS 1 + +extern int yylex (void); + +#define YY_DECL int yylex (void) +#endif /* !YY_DECL */ + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK /*LINTED*/break; +#endif + +#define YY_RULE_SETUP \ + YY_USER_ACTION + +/** The main scanner function which does all the work. + */ +YY_DECL +{ + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + + if ( !(yy_init) ) + { + (yy_init) = 1; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! (yy_start) ) + (yy_start) = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_load_buffer_state( ); + } + + { +#line 8 "lexer.l" + + +#line 672 "lex.yy.c" + + while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ + { + yy_cp = (yy_c_buf_p); + + /* Support of yytext. */ + *yy_cp = (yy_hold_char); + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = (yy_start); +yy_match: + do + { + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 12 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + ++yy_cp; + } + while ( yy_base[yy_current_state] != 9 ); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + if ( yy_act == 0 ) + { /* have to back up */ + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + yy_act = yy_accept[yy_current_state]; + } + + YY_DO_BEFORE_ACTION; + +do_action: /* This label is used only to access EOF actions. */ + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = (yy_hold_char); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + +case 1: +YY_RULE_SETUP +#line 10 "lexer.l" +{ yylval.num = atoi(yytext); return NUMBER; } + YY_BREAK +case 2: +YY_RULE_SETUP +#line 11 "lexer.l" +{ yylval.chr = yytext[0] ; return MATHOP; } + YY_BREAK +case 3: +YY_RULE_SETUP +#line 12 "lexer.l" +{ yylval.chr = '(' ; return LBRACE; } + YY_BREAK +case 4: +YY_RULE_SETUP +#line 13 "lexer.l" +{ yylval.chr = ')' ; return RBRACE; } + YY_BREAK +case 5: +YY_RULE_SETUP +#line 14 "lexer.l" +{ } /* Don't do anything on spaces */ + YY_BREAK +case 6: +YY_RULE_SETUP +#line 16 "lexer.l" +{ yy_fatal_error("Bad character"); } + YY_BREAK +case 7: +YY_RULE_SETUP +#line 18 "lexer.l" +ECHO; + YY_BREAK +#line 764 "lex.yy.c" +case YY_STATE_EOF(INITIAL): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = (yy_hold_char); + YY_RESTORE_YY_MORE_OFFSET + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between YY_CURRENT_BUFFER and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++(yy_c_buf_p); + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = (yy_c_buf_p); + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_END_OF_FILE: + { + (yy_did_buffer_switch_on_eof) = 0; + + if ( yywrap( ) ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = + (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + (yy_c_buf_p) = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of user's declarations */ +} /* end of yylex */ + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ +static int yy_get_next_buffer (void) +{ + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = (yytext_ptr); + int number_to_move, i; + int ret_val; + + if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1); + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; + + else + { + int num_to_read = + YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; + + int yy_c_buf_p_offset = + (int) ((yy_c_buf_p) - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + int new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yyrealloc( (void *) b->yy_ch_buf, + (yy_size_t) (b->yy_buf_size + 2) ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = NULL; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - + number_to_move - 1; + + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), + (yy_n_chars), num_to_read ); + + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + if ( (yy_n_chars) == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + /* Extend the array by 50%, plus the number we really need. */ + int new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( + (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size ); + if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); + /* "- 2" to take care of EOB's */ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); + } + + (yy_n_chars) += number_to_move; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + + (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; + + return ret_val; +} + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + + static yy_state_type yy_get_previous_state (void) +{ + yy_state_type yy_current_state; + char *yy_cp; + + yy_current_state = (yy_start); + + for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) + { + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 12 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + } + + return yy_current_state; +} + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) +{ + int yy_is_jam; + char *yy_cp = (yy_c_buf_p); + + YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 12 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_is_jam = (yy_current_state == 11); + + return yy_is_jam ? 0 : yy_current_state; +} + +#ifndef YY_NO_UNPUT + + static void yyunput (int c, char * yy_bp ) +{ + char *yy_cp; + + yy_cp = (yy_c_buf_p); + + /* undo effects of setting up yytext */ + *yy_cp = (yy_hold_char); + + if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 ) + { /* need to shift things up to make room */ + /* +2 for EOB chars. */ + int number_to_move = (yy_n_chars) + 2; + char *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2]; + char *source = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]; + + while ( source > YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + *--dest = *--source; + + yy_cp += (int) (dest - source); + yy_bp += (int) (dest - source); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = + (yy_n_chars) = (int) YY_CURRENT_BUFFER_LVALUE->yy_buf_size; + + if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 ) + YY_FATAL_ERROR( "flex scanner push-back overflow" ); + } + + *--yy_cp = (char) c; + + (yytext_ptr) = yy_bp; + (yy_hold_char) = *yy_cp; + (yy_c_buf_p) = yy_cp; +} + +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus + static int yyinput (void) +#else + static int input (void) +#endif + +{ + int c; + + *(yy_c_buf_p) = (yy_hold_char); + + if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + /* This was really a NUL. */ + *(yy_c_buf_p) = '\0'; + + else + { /* need more input */ + int offset = (int) ((yy_c_buf_p) - (yytext_ptr)); + ++(yy_c_buf_p); + + switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /*FALLTHROUGH*/ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap( ) ) + return 0; + + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = (yytext_ptr) + offset; + break; + } + } + } + + c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ + *(yy_c_buf_p) = '\0'; /* preserve yytext */ + (yy_hold_char) = *++(yy_c_buf_p); + + return c; +} +#endif /* ifndef YY_NO_INPUT */ + +/** Immediately switch to a different input stream. + * @param input_file A readable stream. + * + * @note This function does not reset the start condition to @c INITIAL . + */ + void yyrestart (FILE * input_file ) +{ + + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_init_buffer( YY_CURRENT_BUFFER, input_file ); + yy_load_buffer_state( ); +} + +/** Switch to a different input buffer. + * @param new_buffer The new input buffer. + * + */ + void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) +{ + + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (); + if ( YY_CURRENT_BUFFER == new_buffer ) + return; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + YY_CURRENT_BUFFER_LVALUE = new_buffer; + yy_load_buffer_state( ); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + (yy_did_buffer_switch_on_eof) = 1; +} + +static void yy_load_buffer_state (void) +{ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + (yy_hold_char) = *(yy_c_buf_p); +} + +/** Allocate and initialize an input buffer state. + * @param file A readable stream. + * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. + * + * @return the allocated buffer state. + */ + YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) +{ + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; +} + +/** Destroy the buffer. + * @param b a buffer created with yy_create_buffer() + * + */ + void yy_delete_buffer (YY_BUFFER_STATE b ) +{ + + if ( ! b ) + return; + + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yyfree( (void *) b->yy_ch_buf ); + + yyfree( (void *) b ); +} + +/* Initializes or reinitializes a buffer. + * This function is sometimes called more than once on the same buffer, + * such as during a yyrestart() or at EOF. + */ + static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + +{ + int oerrno = errno; + + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + + /* If b is the current buffer, then yy_init_buffer was _probably_ + * called from yyrestart() or through yy_get_next_buffer. + * In that case, we don't want to reset the lineno or column. + */ + if (b != YY_CURRENT_BUFFER){ + b->yy_bs_lineno = 1; + b->yy_bs_column = 0; + } + + b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; + + errno = oerrno; +} + +/** Discard all buffered characters. On the next scan, YY_INPUT will be called. + * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. + * + */ + void yy_flush_buffer (YY_BUFFER_STATE b ) +{ + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == YY_CURRENT_BUFFER ) + yy_load_buffer_state( ); +} + +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + * @param new_buffer The new state. + * + */ +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + (yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_LVALUE = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; +} + +/** Removes and deletes the top of the stack, if present. + * The next element becomes the new top. + * + */ +void yypop_buffer_state (void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + if ((yy_buffer_stack_top) > 0) + --(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; + } +} + +/* Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +static void yyensure_buffer_stack (void) +{ + yy_size_t num_to_alloc; + + if (!(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + (yy_buffer_stack_max) = num_to_alloc; + (yy_buffer_stack_top) = 0; + return; + } + + if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + yy_size_t grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = (yy_buffer_stack_max) + grow_size; + (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + ((yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + /* zero only the new slots.*/ + memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + (yy_buffer_stack_max) = num_to_alloc; + } +} + +/** Setup the input buffer state to scan directly from a user-specified character buffer. + * @param base the character buffer + * @param size the size in bytes of the character buffer + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +{ + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return NULL; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = NULL; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; +} + +/** Setup the input buffer state to scan a string. The next call to yylex() will + * scan from a @e copy of @a str. + * @param yystr a NUL-terminated string to scan + * + * @return the newly allocated buffer state object. + * @note If you want to scan bytes that may contain NUL values, then use + * yy_scan_bytes() instead. + */ +YY_BUFFER_STATE yy_scan_string (const char * yystr ) +{ + + return yy_scan_bytes( yystr, (int) strlen(yystr) ); +} + +/** Setup the input buffer state to scan the given bytes. The next call to yylex() will + * scan from a @e copy of @a bytes. + * @param yybytes the byte buffer to scan + * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) +{ + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + int i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = (yy_size_t) (_yybytes_len + 2); + buf = (char *) yyalloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < _yybytes_len; ++i ) + buf[i] = yybytes[i]; + + buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; +} + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +static void yynoreturn yy_fatal_error (const char* msg ) +{ + fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); +} + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + yytext[yyleng] = (yy_hold_char); \ + (yy_c_buf_p) = yytext + yyless_macro_arg; \ + (yy_hold_char) = *(yy_c_buf_p); \ + *(yy_c_buf_p) = '\0'; \ + yyleng = yyless_macro_arg; \ + } \ + while ( 0 ) + +/* Accessor methods (get/set functions) to struct members. */ + +/** Get the current line number. + * + */ +int yyget_lineno (void) +{ + + return yylineno; +} + +/** Get the input stream. + * + */ +FILE *yyget_in (void) +{ + return yyin; +} + +/** Get the output stream. + * + */ +FILE *yyget_out (void) +{ + return yyout; +} + +/** Get the length of the current token. + * + */ +int yyget_leng (void) +{ + return yyleng; +} + +/** Get the current token. + * + */ + +char *yyget_text (void) +{ + return yytext; +} + +/** Set the current line number. + * @param _line_number line number + * + */ +void yyset_lineno (int _line_number ) +{ + + yylineno = _line_number; +} + +/** Set the input stream. This does not discard the current + * input buffer. + * @param _in_str A readable stream. + * + * @see yy_switch_to_buffer + */ +void yyset_in (FILE * _in_str ) +{ + yyin = _in_str ; +} + +void yyset_out (FILE * _out_str ) +{ + yyout = _out_str ; +} + +int yyget_debug (void) +{ + return yy_flex_debug; +} + +void yyset_debug (int _bdebug ) +{ + yy_flex_debug = _bdebug ; +} + +static int yy_init_globals (void) +{ + /* Initialization is the same as for the non-reentrant scanner. + * This function is called from yylex_destroy(), so don't allocate here. + */ + + (yy_buffer_stack) = NULL; + (yy_buffer_stack_top) = 0; + (yy_buffer_stack_max) = 0; + (yy_c_buf_p) = NULL; + (yy_init) = 0; + (yy_start) = 0; + +/* Defined in main.c */ +#ifdef YY_STDINIT + yyin = stdin; + yyout = stdout; +#else + yyin = NULL; + yyout = NULL; +#endif + + /* For future reference: Set errno on error, since we are called by + * yylex_init() + */ + return 0; +} + +/* yylex_destroy is for both reentrant and non-reentrant scanners. */ +int yylex_destroy (void) +{ + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer( YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + yypop_buffer_state(); + } + + /* Destroy the stack itself. */ + yyfree((yy_buffer_stack) ); + (yy_buffer_stack) = NULL; + + /* Reset the globals. This is important in a non-reentrant scanner so the next time + * yylex() is called, initialization will occur. */ + yy_init_globals( ); + + return 0; +} + +/* + * Internal utility routines. + */ + +#ifndef yytext_ptr +static void yy_flex_strncpy (char* s1, const char * s2, int n ) +{ + + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; +} +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (const char * s ) +{ + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; +} +#endif + +void *yyalloc (yy_size_t size ) +{ + return malloc(size); +} + +void *yyrealloc (void * ptr, yy_size_t size ) +{ + + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return realloc(ptr, size); +} + +void yyfree (void * ptr ) +{ + free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ +} + +#define YYTABLES_NAME "yytables" + +#line 18 "lexer.l" + + diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/lexer.l b/src/blog/sane-explanation-of-compilers/part-1/proper/lexer.l new file mode 100644 index 0000000..9babc14 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/lexer.l @@ -0,0 +1,18 @@ +%{ +#include "tree.h" +#include "parser.tab.h" +%} + +%option noyywrap + +%% + +[123] { yylval.num = atoi(yytext); return NUMBER; } +[\+\*] { yylval.chr = yytext[0] ; return MATHOP; } +\( { yylval.chr = '(' ; return LBRACE; } +\) { yylval.chr = ')' ; return RBRACE; } +\ { } /* Don't do anything on spaces */ + +. { yy_fatal_error("Bad character"); } + +%% diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.c b/src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.c new file mode 100644 index 0000000..90b604c --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.c @@ -0,0 +1,1298 @@ +/* A Bison parser, made by GNU Bison 3.8.2. */ + +/* Bison implementation for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +/* Identify Bison output, and Bison version. */ +#define YYBISON 30802 + +/* Bison version string. */ +#define YYBISON_VERSION "3.8.2" + +/* Skeleton name. */ +#define YYSKELETON_NAME "yacc.c" + +/* Pure parsers. */ +#define YYPURE 0 + +/* Push parsers. */ +#define YYPUSH 0 + +/* Pull parsers. */ +#define YYPULL 1 + + + + +/* First part of user prologue. */ +#line 1 "parser.y" + +#include +#include "tree.h" + +extern FILE* yyin; + +int yylex(); +int yyerror(const char *s); + +#line 81 "parser.tab.c" + +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +#include "parser.tab.h" +/* Symbol kind. */ +enum yysymbol_kind_t +{ + YYSYMBOL_YYEMPTY = -2, + YYSYMBOL_YYEOF = 0, /* "end of file" */ + YYSYMBOL_YYerror = 1, /* error */ + YYSYMBOL_YYUNDEF = 2, /* "invalid token" */ + YYSYMBOL_NUMBER = 3, /* NUMBER */ + YYSYMBOL_MATHOP = 4, /* MATHOP */ + YYSYMBOL_LBRACE = 5, /* LBRACE */ + YYSYMBOL_RBRACE = 6, /* RBRACE */ + YYSYMBOL_YYACCEPT = 7, /* $accept */ + YYSYMBOL_start = 8, /* start */ + YYSYMBOL_expr = 9 /* expr */ +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; + + + + +#ifdef short +# undef short +#endif + +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + and (if available) are included + so that the code can choose integer types of a good width. */ + +#ifndef __PTRDIFF_MAX__ +# include /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif + +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ + +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; +#endif + +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; +#else +typedef short yytype_int16; +#endif + +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + . */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 +#endif + +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; +#else +typedef short yytype_uint8; +#endif + +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; +#else +typedef int yytype_uint16; +#endif + +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned +# endif +#endif + +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef yytype_int8 yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_(Msgid) dgettext ("bison-runtime", Msgid) +# endif +# endif +# ifndef YY_ +# define YY_(Msgid) Msgid +# endif +#endif + + +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if !defined yyoverflow + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS +# include /* INFRINGES ON USER NAME SPACE */ + /* Use EXIT_SUCCESS as a witness for stdlib.h. */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's 'empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined EXIT_SUCCESS \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include /* INFRINGES ON USER NAME SPACE */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined EXIT_SUCCESS +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined EXIT_SUCCESS +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* !defined yyoverflow */ + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yy_state_t yyss_alloc; + YYSTYPE yyvs_alloc; +}; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE)) \ + + YYSTACK_GAP_MAXIMUM) + +# define YYCOPY_NEEDED 1 + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYPTRDIFF_T yynewbytes; \ + YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ + } \ + while (0) + +#endif + +#if defined YYCOPY_NEEDED && YYCOPY_NEEDED +/* Copy COUNT objects from SRC to DST. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(Dst, Src, Count) \ + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) +# else +# define YYCOPY(Dst, Src, Count) \ + do \ + { \ + YYPTRDIFF_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (Dst)[yyi] = (Src)[yyi]; \ + } \ + while (0) +# endif +# endif +#endif /* !YYCOPY_NEEDED */ + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL 6 +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST 8 + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS 7 +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS 3 +/* YYNRULES -- Number of rules. */ +#define YYNRULES 5 +/* YYNSTATES -- Number of states. */ +#define YYNSTATES 11 + +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK 261 + + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) + +/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM + as returned by yylex. */ +static const yytype_int8 yytranslate[] = +{ + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, + 5, 6 +}; + +#if YYDEBUG +/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ +static const yytype_int8 yyrline[] = +{ + 0, 21, 21, 22, 25, 26 +}; +#endif + +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if YYDEBUG || 0 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ + "\"end of file\"", "error", "\"invalid token\"", "NUMBER", "MATHOP", + "LBRACE", "RBRACE", "$accept", "start", "expr", YY_NULLPTR +}; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} +#endif + +#define YYPACT_NINF (-4) + +#define yypact_value_is_default(Yyn) \ + ((Yyn) == YYPACT_NINF) + +#define YYTABLE_NINF (-1) + +#define yytable_value_is_error(Yyn) \ + 0 + +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +static const yytype_int8 yypact[] = +{ + -1, -4, -1, 1, -1, 3, -4, -4, -1, -3, + -4 +}; + +/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + Performed when YYTABLE does not specify something else to do. Zero + means the default is an error. */ +static const yytype_int8 yydefact[] = +{ + 0, 4, 0, 0, 2, 0, 1, 3, 0, 0, + 5 +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const yytype_int8 yypgoto[] = +{ + -4, 4, -2 +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const yytype_int8 yydefgoto[] = +{ + 0, 3, 4 +}; + +/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule whose + number is the opposite. If YYTABLE_NINF, syntax error. */ +static const yytype_int8 yytable[] = +{ + 5, 6, 1, 10, 2, 0, 9, 8, 7 +}; + +static const yytype_int8 yycheck[] = +{ + 2, 0, 3, 6, 5, -1, 8, 4, 4 +}; + +/* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + state STATE-NUM. */ +static const yytype_int8 yystos[] = +{ + 0, 3, 5, 8, 9, 9, 0, 8, 4, 9, + 6 +}; + +/* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ +static const yytype_int8 yyr1[] = +{ + 0, 7, 8, 8, 9, 9 +}; + +/* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ +static const yytype_int8 yyr2[] = +{ + 0, 2, 1, 2, 1, 5 +}; + + +enum { YYENOMEM = -2 }; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYNOMEM goto yyexhaustedlab + + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF + + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (0) + + + + +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Kind, Value); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (0) + + +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ + +static void +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep) +{ + FILE *yyoutput = yyo; + YY_USE (yyoutput); + if (!yyvaluep) + return; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + YY_USE (yykind); + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ + +static void +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep) +{ + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); + + yy_symbol_value_print (yyo, yykind, yyvaluep); + YYFPRINTF (yyo, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +static void +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop) +{ + YYFPRINTF (stderr, "Stack now"); + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF (stderr, " %d", yybot); + } + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)); \ +} while (0) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +static void +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, + int yyrule) +{ + int yylno = yyrline[yyrule]; + int yynrhs = yyr2[yyrule]; + int yyi; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)]); + YYFPRINTF (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyssp, yyvsp, Rule); \ +} while (0) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +int yydebug; +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) +# define YY_STACK_PRINT(Bottom, Top) +# define YY_REDUCE_PRINT(Rule) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + + + + + +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +static void +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep) +{ + YY_USE (yyvaluep); + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp); + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + YY_USE (yykind); + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/* Lookahead token kind. */ +int yychar; + +/* The semantic value of the lookahead symbol. */ +YYSTYPE yylval; +/* Number of syntax errors so far. */ +int yynerrs; + + + + +/*----------. +| yyparse. | +`----------*/ + +int +yyparse (void) +{ + yy_state_fast_t yystate = 0; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus = 0; + + /* Refer to the stacks through separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; + + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; + + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; + + int yyn; + /* The return value of yyparse. */ + int yyresult; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + + + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yychar = YYEMPTY; /* Cause a token to be read. */ + + goto yysetstate; + + +/*------------------------------------------------------------. +| yynewstate -- push a new state, which is found in yystate. | +`------------------------------------------------------------*/ +yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp); + + if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + YYNOMEM; +#else + { + /* Get the current used size of the three stacks, in elements. */ + YYPTRDIFF_T yysize = yyssp - yyss + 1; + +# if defined yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + yy_state_t *yyss1 = yyss; + YYSTYPE *yyvs1 = yyvs; + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), + &yystacksize); + yyss = yyss1; + yyvs = yyvs1; + } +# else /* defined YYSTACK_RELOCATE */ + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + YYNOMEM; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yy_state_t *yyss1 = yyss; + union yyalloc *yyptr = + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); + if (! yyptr) + YYNOMEM; + YYSTACK_RELOCATE (yyss_alloc, yyss); + YYSTACK_RELOCATE (yyvs_alloc, yyvs); +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ + + + if (yystate == YYFINAL) + YYACCEPT; + + goto yybackup; + + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yydefault; + + /* Not known => get a lookahead token if don't already have one. */ + + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex (); + } + + if (yychar <= YYEOF) + { + yychar = YYEOF; + yytoken = YYSYMBOL_YYEOF; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == YYerror) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = YYUNDEF; + yytoken = YYSYMBOL_YYerror; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); + yystate = yyn; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + /* Discard the shifted token. */ + yychar = YYEMPTY; + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + '$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; + + + YY_REDUCE_PRINT (yyn); + switch (yyn) + { + case 2: /* start: expr */ +#line 21 "parser.y" + { printTree((yyvsp[0].n), 0); freeNode((yyvsp[0].n)); } +#line 1067 "parser.tab.c" + break; + + case 3: /* start: expr start */ +#line 22 "parser.y" + { printTree((yyvsp[-1].n), 0); freeNode((yyvsp[-1].n)); } +#line 1073 "parser.tab.c" + break; + + case 4: /* expr: NUMBER */ +#line 25 "parser.y" + { (yyval.n) = newNode((yyvsp[0].num) + '0', NULL, NULL); } +#line 1079 "parser.tab.c" + break; + + case 5: /* expr: LBRACE expr MATHOP expr RBRACE */ +#line 26 "parser.y" + { (yyval.n) = newNode((yyvsp[-2].chr), (yyvsp[-3].n), (yyvsp[-1].n)); } +#line 1085 "parser.tab.c" + break; + + +#line 1089 "parser.tab.c" + + default: break; + } + /* User semantic actions sometimes alter yychar, and that requires + that yytoken be updated with the new translation. We take the + approach of translating immediately before every use of yytoken. + One alternative is translating here after every semantic action, + but that translation would be missed if the semantic action invokes + YYABORT, YYACCEPT, or YYERROR immediately after altering yychar or + if it invokes YYBACKUP. In the case of YYABORT or YYACCEPT, an + incorrect destructor might then be invoked immediately. In the + case of YYERROR or YYBACKUP, subsequent parser actions might lead + to an incorrect destructor call or verbose syntax error message + before the lookahead is translated. */ + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc); + + YYPOPSTACK (yylen); + yylen = 0; + + *++yyvsp = yyval; + + /* Now 'shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yynewstate; + + +/*--------------------------------------. +| yyerrlab -- here on detecting error. | +`--------------------------------------*/ +yyerrlab: + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; + yyerror (YY_("syntax error")); + } + + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + if (yychar <= YYEOF) + { + /* Return failure if at end of input. */ + if (yychar == YYEOF) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; + ++yynerrs; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + YYPOPSTACK (yylen); + yylen = 0; + YY_STACK_PRINT (yyss, yyssp); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + /* Pop stack until we find a state that shifts the error token. */ + for (;;) + { + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + + yydestruct ("Error: popping", + YY_ACCESSING_SYMBOL (yystate), yyvsp); + YYPOPSTACK (1); + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp); + } + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp); + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturnlab; + + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturnlab; + + +/*-----------------------------------------------------------. +| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. | +`-----------------------------------------------------------*/ +yyexhaustedlab: + yyerror (YY_("memory exhausted")); + yyresult = 2; + goto yyreturnlab; + + +/*----------------------------------------------------------. +| yyreturnlab -- parsing is finished, clean up and return. | +`----------------------------------------------------------*/ +yyreturnlab: + if (yychar != YYEMPTY) + { + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = YYTRANSLATE (yychar); + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval); + } + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif + + return yyresult; +} + +#line 29 "parser.y" + + +int yyerror(const char *s) { + printf("Error: %s\n", s); + return 0; +} + +int main(int argc, char* argv[]) { + FILE* input = fopen(argv[1], "r"); + if (input == NULL) { + yyerror("reading input file!"); + return 1; + } + + yyin = input; + yyparse(); +} diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.h b/src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.h new file mode 100644 index 0000000..52740b7 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/parser.tab.h @@ -0,0 +1,90 @@ +/* A Bison parser, made by GNU Bison 3.8.2. */ + +/* Bison interface for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +#ifndef YY_YY_PARSER_TAB_H_INCLUDED +# define YY_YY_PARSER_TAB_H_INCLUDED +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG +extern int yydebug; +#endif + +/* Token kinds. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + enum yytokentype + { + YYEMPTY = -2, + YYEOF = 0, /* "end of file" */ + YYerror = 256, /* error */ + YYUNDEF = 257, /* "invalid token" */ + NUMBER = 258, /* NUMBER */ + MATHOP = 259, /* MATHOP */ + LBRACE = 260, /* LBRACE */ + RBRACE = 261 /* RBRACE */ + }; + typedef enum yytokentype yytoken_kind_t; +#endif + +/* Value type. */ +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +union YYSTYPE +{ +#line 11 "parser.y" + + int num; + char chr; + node *n; + +#line 76 "parser.tab.h" + +}; +typedef union YYSTYPE YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif + + +extern YYSTYPE yylval; + + +int yyparse (void); + + +#endif /* !YY_YY_PARSER_TAB_H_INCLUDED */ diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/parser.y b/src/blog/sane-explanation-of-compilers/part-1/proper/parser.y new file mode 100644 index 0000000..9946346 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/parser.y @@ -0,0 +1,45 @@ +%{ +#include +#include "tree.h" + +extern FILE* yyin; + +int yylex(); +int yyerror(const char *s); +%} + +%union { + int num; + char chr; + node *n; +} +%token NUMBER MATHOP LBRACE RBRACE +%type expr + +%% + +start : expr { printTree($1, 0); freeNode($1); } + | expr start { printTree($1, 0); freeNode($1); } + ; + +expr : NUMBER { $$ = newNode($1 + '0', NULL, NULL); } + | LBRACE expr MATHOP expr RBRACE { $$ = newNode($3, $2, $4); } + ; + +%% + +int yyerror(const char *s) { + printf("Error: %s\n", s); + return 0; +} + +int main(int argc, char* argv[]) { + FILE* input = fopen(argv[1], "r"); + if (input == NULL) { + yyerror("reading input file!"); + return 1; + } + + yyin = input; + yyparse(); +} diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/test.txt b/src/blog/sane-explanation-of-compilers/part-1/proper/test.txt new file mode 100644 index 0000000..51121f3 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/test.txt @@ -0,0 +1 @@ +((1+1) * ((3+3) * 2)) diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/test_output.txt b/src/blog/sane-explanation-of-compilers/part-1/proper/test_output.txt new file mode 100644 index 0000000..db41b84 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/test_output.txt @@ -0,0 +1,9 @@ +|-* + |-+ + |-1 + |-1 + |-* + |-+ + |-3 + |-3 + |-2 diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/tree.c b/src/blog/sane-explanation-of-compilers/part-1/proper/tree.c new file mode 100644 index 0000000..6119d5c --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/tree.c @@ -0,0 +1,24 @@ +#include "tree.h" + +node* newNode(char val, node *left, node *right) { + node* out = malloc(sizeof(node)); + out->val = val; + out->left = left; + out->right = right; + return out; +} + +void printTree(node *n, int level) { + if (n == NULL) return; + // %*s will add level amounts of spaces (but with the "" string) + printf("%*s|-%c\n", level, "", n->val); + printTree(n->left, level + 1); + printTree(n->right, level + 1); +} + +void freeNode(node *n) { + if (n == NULL) return; + freeNode(n->left); + freeNode(n->right); + free(n); +} diff --git a/src/blog/sane-explanation-of-compilers/part-1/proper/tree.h b/src/blog/sane-explanation-of-compilers/part-1/proper/tree.h new file mode 100644 index 0000000..fdddab3 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/proper/tree.h @@ -0,0 +1,17 @@ +#ifndef TREE +#define TREE + +#include +#include + +typedef struct node { + char val; + struct node *left; + struct node *right; +} node; + +node* newNode(char val, node *left, node *right); +void printTree(node *n, int level); +void freeNode(node *n); + +#endif diff --git a/src/blog/sane-explanation-of-compilers/part-1/regex_parser_simple.cpp b/src/blog/sane-explanation-of-compilers/part-1/regex_parser_simple.cpp new file mode 100644 index 0000000..c0a5a4e --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/regex_parser_simple.cpp @@ -0,0 +1,16 @@ +bool matchesp(string str) { + string regex = "acce_t"; + + for (int si = 0, ri = 0; si < str.size() && ri < regex.size(); si++, ri++) { + while (regex[ri] == '_') + if (str[si] == 'n' || str[si] == 'p') + si++; + else + ri++; + } + + if (str[si] != regex[ri]) + return false; + } + return true; +} diff --git a/src/blog/sane-explanation-of-compilers/part-1/template.l b/src/blog/sane-explanation-of-compilers/part-1/template.l new file mode 100644 index 0000000..9dd2557 --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/template.l @@ -0,0 +1,13 @@ +%{ +// C code that will be put in the program verbatim +%} + +/* Definitions */ + +%% + + /* Rules */ + +%% + +// C code diff --git a/src/blog/sane-explanation-of-compilers/part-1/template.y b/src/blog/sane-explanation-of-compilers/part-1/template.y new file mode 100644 index 0000000..f7712de --- /dev/null +++ b/src/blog/sane-explanation-of-compilers/part-1/template.y @@ -0,0 +1,13 @@ +%{ +// C code that will be put in the program verbatim +%} + +/* Declarations */ + +%% + + /* Rules */ + +%% + +// C code -- cgit v1.3.1-23-gd82d