new assembler: tokenizer

1 files changed, 36 insertions, 0 deletions

diff --git a/src/as_tokenizer.h b/src/as_tokenizer.h
new file mode 100644
index 0000000..fef8625
--- /dev/null
+++ b/src/as_tokenizer.h
@@ -0,0 +1,36 @@
+#include <stdint.h>
+#include <stdio.h>
+
+typedef enum {
+    OP, ARG, LABEL, COLON, NEWLINE, ENDOFFILE
+} TokenType;
+
+typedef struct {
+    TokenType type;
+    int line;
+    int col;
+    char *sval;
+    int64_t ival;
+    double fval;
+} Token;
+
+#define INPUT_STREAM_BUF_SIZE 1024
+
+typedef struct {
+    FILE *fp;
+    char *buf;
+    int buf_pos;
+    int cursor;
+    int line;
+    int col;
+} InputStream;
+
+typedef struct {
+    Token* buf;
+    InputStream *s;
+} TokenStream;
+
+Token *nextToken(TokenStream *ts);
+Token *peekToken(TokenStream *ts);
+void printToken(Token *t);
+TokenStream* makeTokenStream(FILE* fp);