src/as_tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

#ifndef FMV_AS_TOKENIZER_H_
#define FMV_AS_TOKENIZER_H_

#include <stdint.h>
#include <stdio.h>

#include "utils.h"

enum token_type {
    TK_OP, TK_ARG, TK_TAG, TK_COLON, TK_NEWLINE, TK_ENDOFFILE
};
typedef enum token_type token_type;

struct token {
    enum token_type type;
    int line;
    int col;
    char *sval;
    int64_t ival;
    double fval;
};
typedef struct token token;

#define INPUT_STREAM_BUF_SIZE 1024

struct input_stream{
    FILE *fp;
    char *buf;
    int buf_pos;
    int cursor;
    int line;
    int col;
}; 
typedef struct input_stream input_stream;

struct token_stream {
    token *buf;
    input_stream *s;
};
typedef struct token_stream token_stream;

// result<token*>
result next_token(allocator * alct, token_stream * ts);

// result<token*>
result peek_token(allocator * alct, token_stream * ts);

void print_token(struct token *t);

token_stream* new_token_stream(allocator * alct, FILE* fp);

#endif // FMV_AS_TOKENIZER_H_