src/as_tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

#ifndef FMV_AS_TOKENIZER_H_
#define FMV_AS_TOKENIZER_H_

#include <stdint.h>
#include <stdio.h>

#include "utils.h"

enum token_type {
    TK_OP, TK_ARG, TK_TAG, TK_COLON, TK_NEWLINE, TK_ENDOFFILE
};

struct token {
    enum token_type type;
    int line;
    int col;
    char *sval;
    int64_t ival;
    double fval;
};

#define INPUT_STREAM_BUF_SIZE 1024

struct input_stream{
    FILE *fp;
    char *buf;
    int buf_pos;
    int cursor;
    int line;
    int col;
}; 

struct token_stream {
    struct token *buf;
    struct input_stream *s;
};

struct token* next_token(struct allocator * alct, struct token_stream * ts);
struct token* peek_token(struct allocator * alct, struct token_stream * ts);
void print_token(struct token *t);
struct token_stream * new_token_stream(struct allocator * alct, FILE* fp);

#endif // FMV_AS_TOKENIZER_H_