Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #include <variant>
- #include <optional>
- #include <istream>
- #include <sstream>
- #include <stdexcept>
- #include <iostream>
- #include <typeinfo>
- #include <random>
- #include "../../Library/Application Support/JetBrains/CLion2021.2/scratches/tokenizer.h"
- #include <memory>
- class Object : public std::enable_shared_from_this<Object> {
- public:
- virtual ~Object() = default;
- };
- class Number : public Object {
- int64_t value_;
- public:
- Number(int value) : value_(value) {
- }
- int GetValue() const {
- return value_;
- }
- };
- class Symbol : public Object {
- std::string s_;
- public:
- Symbol(const std::string& s) : s_(s) {
- }
- const std::string& GetName() const {
- return s_;
- }
- };
- class Cell : public Object {
- std::shared_ptr<Object> head_ = nullptr;
- std::shared_ptr<Object> tail_ = nullptr;
- public:
- Cell(auto h, auto t) {
- head_ = h;
- tail_ = t;
- }
- std::shared_ptr<Object> GetFirst() const {
- return head_;
- }
- std::shared_ptr<Object> GetSecond() const {
- return tail_;
- }
- };
- template <class T>
- std::shared_ptr<T> As(const std::shared_ptr<Object>& obj) {
- return std::dynamic_pointer_cast<T>(obj);
- }
- template <class T>
- bool Is(const std::shared_ptr<Object>& obj) {
- if (obj == nullptr) {
- std::cerr << "чекни мать" << std::endl;
- return false;
- }
- return (typeid(T) == typeid(*obj));
- }
- std::shared_ptr<Object> Read(Tokenizer* tokenizer);
- std::shared_ptr<Cell> ReadList(Tokenizer* tokenizer) {
- if (tokenizer->IsEnd()) {
- throw SyntaxError("nothing more to parse");
- }
- auto t = tokenizer->GetToken();
- // empty list
- if (t.index() == 1 && (std::get<BracketToken>(t) == BracketToken::CLOSE)) {
- tokenizer->Next();
- return nullptr;
- }
- //not empty
- auto first_elem = Read(tokenizer);
- auto t1 = tokenizer->GetToken(); //check a dot
- //pair
- if (t1.index() == 4) {
- tokenizer->Next(); //next after a dot
- auto second_elem = Read(tokenizer);
- auto br = tokenizer->GetToken();
- if (br.index() != 1 || (std::get<BracketToken>(br) != BracketToken::CLOSE)) {
- throw SyntaxError("invalid pair syntax");
- } else {
- tokenizer->Next(); //next after a bracket
- return std::shared_ptr<Cell>(new Cell(first_elem, second_elem));
- }
- }
- //list
- while (!tokenizer->IsEnd()) {
- }
- }
- std::shared_ptr<Object> Read(Tokenizer* tokenizer) {
- if (tokenizer->IsEnd()) {
- throw SyntaxError("nothing to parse");
- }
- auto t = tokenizer->GetToken();
- if (t.index() == 0) {
- tokenizer->Next();
- return std::shared_ptr<Number>(new Number(std::get<ConstantToken>(t).value_));
- } else if (t.index() == 1) {
- switch(std::get<BracketToken>(t)) {
- case BracketToken::CLOSE : throw SyntaxError("unexpected )");
- case BracketToken::OPEN : {
- tokenizer->Next();
- return ReadList(tokenizer);
- }
- }
- } else if (t.index() == 2) {
- tokenizer->Next();
- return std::shared_ptr<Symbol>(new Symbol(std::get<SymbolToken>(t).name_));
- } else if (t.index() == 3) {
- throw SyntaxError("unexpected '");
- } else if (t.index() == 4) {
- throw SyntaxError("unexpected .");
- }
- }
- void ShouldThrow() {
- std::string input = "1@";
- std::stringstream ss{input};
- Tokenizer tokenizer{&ss};
- tokenizer.Next();
- }
- auto ReadFull(const std::string& str) {
- std::stringstream ss{str};
- Tokenizer tokenizer{&ss};
- auto obj = Read(&tokenizer);
- assert(tokenizer.IsEnd());
- return obj;
- }
- std::string RandomSymbol(std::default_random_engine* rng) {
- std::uniform_int_distribution<char> symbol('a', 'z');
- std::string s;
- for (int i = 0; i < 5; ++i) {
- s.push_back(symbol(*rng));
- }
- return s;
- }
- int main() {
- {
- std::stringstream ss{"445+)'."};
- Tokenizer tokenizer{&ss};
- assert(!tokenizer.IsEnd());
- // If next line fails to compile, check that operator == is defined for every token type.
- assert(tokenizer.GetToken() == Token{ConstantToken{445}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"+"}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{BracketToken::CLOSE});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{QuoteToken{}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{DotToken{}});
- tokenizer.Next();
- assert(tokenizer.IsEnd());
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss{"-2 - 2"};
- Tokenizer tokenizer{&ss};
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{ConstantToken{-2}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"-"}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{ConstantToken{2}});
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss{"foo bar zog-zog?"};
- Tokenizer tokenizer{&ss};
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"foo"}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"bar"}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"zog-zog?"}});
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss{"1234+4"};
- Tokenizer tokenizer{&ss};
- assert(tokenizer.GetToken() == Token{ConstantToken{1234}});
- assert(tokenizer.GetToken() == Token{ConstantToken{1234}});
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss;
- ss << "2 ";
- Tokenizer tokenizer{&ss};
- assert(tokenizer.GetToken() == Token{ConstantToken{2}});
- ss << "* ";
- tokenizer.Next();
- assert(tokenizer.GetToken() == Token{SymbolToken{"*"}});
- ss << "2";
- tokenizer.Next();
- assert(tokenizer.GetToken() == Token{ConstantToken{2}});
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss{" "};
- Tokenizer tokenizer{&ss};
- assert(tokenizer.IsEnd());
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss{" 4 + "};
- Tokenizer tokenizer{&ss};
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{ConstantToken{4}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"+"}});
- tokenizer.Next();
- assert(tokenizer.IsEnd());
- }
- std::cout << ".............................\n";
- {
- std::string input = R"EOF(
- )EOF";
- std::stringstream ss{input};
- Tokenizer tokenizer{&ss};
- assert(tokenizer.IsEnd());
- }
- std::cout << ".............................\n";
- {
- std::string input = R"EOF(
- 4 +
- )EOF";
- std::stringstream ss{input};
- Tokenizer tokenizer{&ss};
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{ConstantToken{4}});
- tokenizer.Next();
- assert(!tokenizer.IsEnd());
- assert(tokenizer.GetToken() == Token{SymbolToken{"+"}});
- tokenizer.Next();
- assert(tokenizer.IsEnd());
- }
- std::cout << ".............................\n";
- {
- std::stringstream ss;
- Tokenizer tokenizer{&ss};
- assert(tokenizer.IsEnd());
- }
- std::cout << ".............................\n";
- try {
- ShouldThrow();
- } catch (SyntaxError& s) {
- std::cout << "Throws!\n";
- }
- std::cout << ".............................\n";
- {
- auto node = ReadFull("5");
- assert(Is<Number>(node));
- assert(As<Number>(node)->GetValue() == 5);
- node = ReadFull("-5");
- assert(Is<Number>(node));
- assert(As<Number>(node)->GetValue() == -5);
- }
- std::cout << ".............................\n";
- {
- auto node = ReadFull("+");
- assert(Is<Symbol>(node));
- assert(As<Symbol>(node)->GetName() == "+");
- }
- std::cout << ".............................\n";
- {
- std::default_random_engine rng{42};
- for (int i = 0; i < 10; ++i) {
- auto name = RandomSymbol(&rng);
- auto node = ReadFull(name);
- assert(Is<Symbol>(node));
- assert(As<Symbol>(node)->GetName() == name);
- }
- }
- std::cout << ".............................\n";
- {
- auto null = ReadFull("()");
- assert(!null);
- }
- std::cout << ".............................\n";
- {
- auto pair = ReadFull("(1 . 2)");
- assert(Is<Cell>(pair));
- auto first = As<Cell>(pair)->GetFirst();
- assert(Is<Number>(first));
- assert(As<Number>(first)->GetValue() == 1);
- auto second = As<Cell>(pair)->GetSecond();
- assert(Is<Number>(second));
- assert(As<Number>(second)->GetValue() == 2);
- }
- std::cout << ".............................\n";
- return 0;
- }
Add Comment
Please, Sign In to add comment