tokenizer/test.cpp at master · wolkdev/tokenizer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

#include "tokenizer.hpp"
#include <iostream>

int main(int argc, char const *argv[])
{
    const char* str =
        "class my_class\n"
        "{\n"
        "    private:\n"
        "    int* iptr = nullptr;\n"
        "    std::string str = \"\";\n"
        "\n"
        "    my_class(std::string&& _str);\n"
        "}";

    // tokenizer initialization

    tokenizer tok;

    tok.set_dropped_delimiters(" \t\r\n");

    tok.add_kept_delimiter("=");
    tok.add_kept_delimiter("*");
    tok.add_kept_delimiter("{");
    tok.add_kept_delimiter("}");
    tok.add_kept_delimiter("(");
    tok.add_kept_delimiter(")");
    tok.add_kept_delimiter(";");
    tok.add_kept_delimiter("&");
    tok.add_kept_delimiter("&&");
    tok.add_kept_delimiter(":");
    tok.add_kept_delimiter("::");
    tok.add_kept_delimiter("\"");

    // tokenize

    tok.start(str);

    while (!tok.finished())
    {
        const std::string& token = tok.next();

        std::cout << token << "\n";
    }

    return 0;
}