src/as_tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

#ifndef FMV_AS_TOKENIZER_H_
#define FMV_AS_TOKENIZER_H_

#include <stdint.h>
#include <stdio.h>

#include "utils.h"

enum tokenType {
    OP, ARG, TAG, COLON, NEWLINE, ENDOFFILE
};

struct token {
    enum tokenType type;
    int line;
    int col;
    char *sval;
    int64_t ival;
    double fval;
};
typedef struct token *Token;

#define INPUT_STREAM_BUF_SIZE 1024

struct inputStream {
    FILE *fp;
    char *buf;
    int buf_pos;
    int cursor;
    int line;
    int col;
}; 
typedef struct inputStream *InputStream;

struct tokenStream {
    Token buf;
    InputStream s;
};
typedef struct tokenStream *TokenStream;

Token nextToken(Allocator alct, TokenStream ts);
Token peekToken(Allocator alct, TokenStream ts);
void printToken(Token t);
TokenStream makeTokenStream(Allocator alct, FILE* fp);

#endif // FMV_AS_TOKENIZER_H_