src/as_tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

#ifndef FMV_AS_TOKENIZER_H_
#define FMV_AS_TOKENIZER_H_

#include <stdint.h>
#include <stdio.h>

#include "utils.h"

enum token_type {
    TK_OP, TK_ARG, TK_TAG, TK_COLON, TK_NEWLINE, TK_ENDOFFILE
};

struct token {
    enum token_type type;
    int line;
    int col;
    char *sval;
    int64_t ival;
    double fval;
};
typedef struct token * token_t;

#define INPUT_STREAM_BUF_SIZE 1024

struct input_stream{
    FILE *fp;
    char *buf;
    int buf_pos;
    int cursor;
    int line;
    int col;
}; 
typedef struct input_stream *input_stream_t;

struct token_stream {
    token_t buf;
    input_stream_t s;
};
typedef struct token_stream *token_stream_t;

token_t next_token(allocator_t alct, token_stream_t ts);
token_t peek_token(allocator_t alct, token_stream_t ts);
void print_token(token_t t);
token_stream_t new_token_stream(allocator_t alct, FILE* fp);

#endif // FMV_AS_TOKENIZER_H_