#include "lexcer.h" #include "lexconst.h" #include #include // using LexCer::TokenClass; // using LexCer::Token; // using LexCer::keywords; // using LexCer::preprocessor; // using LexCer::tokenToClass; // using LexCer::spanBegin; // using LexCer::spanEnd; //const char* Tokenizer::spanBegin = "push_back(c); // we may need that last bit ungetc((unsigned) c, in); auto directive = lexeme->substr(1, lexeme->size() - 1); if (!preprocessor.count(directive)) return false; if (directive.compare("define") == 0) { //readDefine(); } return true; } bool Tokenizer::readNextString(std::string *lexeme, char oldc) { char c, prevc = oldc; while (((c = fgetc(in)) != '"' || prevc == '\\') && c != '\n' && c != EOF) { lexeme->push_back(c); prevc = c; } if (c == '\n' || c == EOF) { ungetc((unsigned) c, in); return false; } lexeme->push_back(c); return true; } bool Tokenizer::readNextIdentifier(std::string *lexeme) { char c; bool valid = true; while (isalnum(c = fgetc(in)) || c == '_') { lexeme->push_back(c); } ungetc((unsigned) c, in); return valid; } bool Tokenizer::readNextNumber(char beg, double *number, std::string *numberRepr) { bool valid = true; num_t base = DEC; char c; if (beg == '0') { c = fgetc(in); numberRepr->push_back(c); if (c == 'b') base = BIN; else if (c == 'x' || c == 'X') base = HEX; else if (isdigit(c)) base = OCT; else ungetc((unsigned) c, in); } while (isValidDigit(c = fgetc(in), base)) { numberRepr->push_back(c); } if (c == '.') { numberRepr->push_back(c); while (isValidDigit(c = fgetc(in), base)) numberRepr->push_back(c); } if (tolower(c) == 'e') { numberRepr->push_back(c); if (base != DEC) return false; if (!isdigit(c = fgetc(in)) && c != '-' && c != '+') { ungetc((unsigned) c, in); return false; } numberRepr->push_back(c); while (isValidDigit(c = fgetc(in), base)) numberRepr->push_back(c); } char lower = tolower(c); if (lower == 'u') { numberRepr->push_back(c); lower = tolower(c = fgetc(in)); } if (lower == 'f' || lower == 'l') numberRepr->push_back(c); else ungetc((unsigned) c, in); *number = this->atod(*numberRepr); // not working ... return true; } bool Tokenizer::isValidDigit(char c, num_t numType) const { bool valid; switch (numType) { case DEC: return isdigit(c); case BIN: return (c == '0' || c == '1'); case OCT: return (c >= '0' || c <= '7'); case HEX: return (isdigit(c) || tolower(c) >= 'a' || tolower(c) <= 'f'); default: return false; } } bool Tokenizer::readLineComment(std::string *lexeme) { char c; while ((c = fgetc(in)) != '\n' && c != EOF) lexeme->push_back(c); ungetc((unsigned) c, in); return true; // for consistency } bool Tokenizer::readMultilineComment(std::string *lexeme) { char c, prevc = (char) -1; while (((c = fgetc(in)) != '/' && prevc != '*') && c != EOF) lexeme->push_back(c); if (c == EOF) { return false; } lexeme->push_back(c); // closing slash return true; } bool Tokenizer::readInclude(std::string *lexeme) { char c; bool valid = true; while ((c = fgetc(in)) != '>') { lexeme->push_back(c); if (!isalpha(c) && c != '_' && c != '.') valid = false; } lexeme->push_back(c); return valid; } bool Tokenizer::readUntilBlank(std::string *lexeme) { char c; while (!isblank(c = fgetc(in)) && c != EOF) ; ungetc((unsigned) c, in); return true; } bool Tokenizer::writeToOutput() { if (token.type == TokenClass::Eof) { printf("End of file!\n"); fputc('\n', out); return false; } if (token.type == TokenClass::Skip) { if (fputs(token.lexeme.c_str(), out) == EOF) { printf("Error while writing to output"); return false; } return true; } printf("Token of type %s at line %d: ", tokenToClass.at(token.type).c_str(), token.line); if (token.type == TokenClass::Number) printf("%f\n", token.number); else printf("%s\n", token.lexeme.c_str()); auto lexeme = escapeHTML(token.lexeme); std::string output; output.append(spanBegin).append(tokenToClass.at(token.type)).append("\">").append(lexeme).append(spanEnd); if (fputs(output.c_str(), out) == EOF) { printf("Error while writing to output"); return false; } return true; } std::string Tokenizer::escapeHTML(const std::string &lexeme) { std::string result; for (auto c : lexeme) { switch(c) { case '<': result.append("<"); break; case '>': result.append(">"); break; case '&': result.append("&"); break; default: result.push_back(c); break; } } return result; } TokenClass Tokenizer::nextToken(std::string *lexeme, double *number) { if (token.type == TokenClass::Eof) { return TokenClass::Invalid; } char c = fgetc(in); if (c == EOF) { return TokenClass::Eof; } lexeme->push_back(c); // operator || symbol || string || preprocessor directive // probably should check for errors like '5 * % 4;', but... switch(c) { case '\n': ++line; case '\t': case ' ': case ';': case ',': case '{': case '}': return TokenClass::Skip; case '(': return TokenClass::LeftParen; case ')': return TokenClass::RightParen; case '[': case ']': case '?': case ':': return TokenClass::Operator; case '*': c = fgetc(in); if (c == '/') { return TokenClass::Invalid; } ungetc((unsigned) c, in); c = '*'; case '^': case '%': case '=': case '!': case '~': c = fgetc(in); if (c != '=') ungetc((unsigned) c, in); else lexeme->push_back(c); return TokenClass::Operator; case '<': case '>': c = fgetc(in); if (c == lexeme->at(lexeme->size() - 1)) { lexeme->push_back(c); c = fgetc(in); if (c == '=') lexeme->push_back(c); else ungetc((unsigned) c, in); } else if (lexeme->at(lexeme->size() - 1) == '<' && isalpha(c)) { lexeme->push_back(c); if (!readInclude(lexeme)) return TokenClass::Invalid; return TokenClass::String; } else ungetc((unsigned) c, in); return TokenClass::Operator; case '&': case '|': case '+': c = fgetc(in); if (c == lexeme->at(lexeme->size() - 1) || c == '=') lexeme->push_back(c); else ungetc((unsigned) c, in); return TokenClass::Operator; case '-': c = fgetc(in); if (c == '-' || c == '=' || c == '>') lexeme->push_back(c); else ungetc((unsigned) c, in); return TokenClass::Operator; case '#': if (readNextDirective(lexeme)) return TokenClass::PreprocessorDirective; return TokenClass::Invalid; case '\'': lexeme->push_back(c = fgetc(in)); if (c == '\\') { lexeme->push_back(c = fgetc(in)); } lexeme->push_back(c = fgetc(in)); // the closing mark if (c != '\'') return TokenClass::Invalid; return TokenClass::String; case '"': if (readNextString(lexeme, c)) return TokenClass::String; return TokenClass::Invalid; case '/': c = fgetc(in); lexeme->push_back(c); switch (c) { case '=': return TokenClass::Operator; case '/': if (!readLineComment(lexeme)) return TokenClass::Invalid; return TokenClass::Comment; case '*': if (!readMultilineComment(lexeme)) return TokenClass::Invalid; return TokenClass::Comment; default: if (!isblank(c) || !isdigit(c)) { ungetc((unsigned) c, in); lexeme->pop_back(); return TokenClass::Invalid; } return TokenClass::Operator; } case '\\': c = fgetc(in); ungetc((unsigned) c, in); if (c != '\n') return TokenClass::Invalid; // stray '\' return TokenClass::Operator; } // identifier || keyword || typename if (isalpha(c) || c == '_') { if (readNextIdentifier(lexeme)) { if (keywords.count(*lexeme) != 0) return TokenClass::Keyword; else if (typenames.count(*lexeme) != 0) return TokenClass::Typename; return TokenClass::Identifier; } else { return TokenClass::Invalid; } } if (!isdigit(c)) { return TokenClass::Invalid; } // number if (readNextNumber(c, number, lexeme)) return TokenClass::Number; return TokenClass::Invalid; } std::string getOutputFileName(const std::string &name) { auto pos = name.find_last_of('.'); if (pos == std::string::npos) { return name + ".html"; } return name.substr(0, pos) + ".html"; } void lexCit(FILE *in, FILE *out) { Tokenizer hl {in, out}; std::string lexeme; double number; TokenClass type; do { type = hl.nextToken(&lexeme, &number); hl.makeToken(type, hl.lineNumber(), lexeme, number); if (!hl.writeToOutput()) break; lexeme = ""; number = 0.0; } while(hl.currentToken().type != TokenClass::Eof || hl.currentToken().type != TokenClass::Invalid); } } /* Namespace Lexcer */