git.haldean.org wallbot / 7ff169d
tokenizer haldean 4 years ago
5 changed file(s) with 134 addition(s) and 10 deletion(s). Raw diff Collapse all Expand all
+24
-0
lang/.clang-format less more
0 ---
1 BasedOnStyle: Webkit
2 IndentWidth: 8
3 ---
4 Language: Cpp
5 AlignAfterOpenBracket: AlwaysBreak
6 AlignEscapedNewlines: Left
7 AlwaysBreakAfterReturnType: TopLevelDefinitions
8 BreakBeforeBraces: Custom
9 BreakBeforeBinaryOperators: NonAssignment
10 BraceWrapping:
11 AfterClass: true
12 AfterControlStatement: true
13 AfterEnum: true
14 AfterFunction: true
15 AfterNamespace: true
16 AfterStruct: true
17 AfterUnion: true
18 BeforeElse: true
19 SplitEmptyFunction: false
20 SplitEmptyRecord: false
21 ColumnLimit: 79
22 IndentCaseLabels: true
23 PointerAlignment: Right
+17
-0
lang/GNUmakefile less more
0 c_sources = meka.c
1
2 %.o: %.c
3 $(CC) -c -fPIC -MD -o $@ -I. $<
4
5 -include meka.d
6
7 libmeka.a: meka.o
8 $(AR) rcs $@ $<
9
10 test/tokenize: test/tokenize.o libmeka.a
11 $(CC) -o $@ $^
12
13 clean:
14 rm -f libmeka.a test/tokenize meka.o meka.d test/tokenize.o test/tokenize.d
15
16 .PHONY: clean
+41
-0
lang/meka.c less more
0 #include "meka.h"
1
2 #define _is_whitespace(c) ((c) == ' ' || (c) == '\n' || (c) == '\r')
3
4 void
5 meka_skip_to_token(struct meka_prog *prog)
6 {
7 char c;
8 while (prog->next != prog->end)
9 {
10 c = *prog->next;
11 if (_is_whitespace(c))
12 continue;
13 if (c == '#')
14 {
15 for (; prog->next != prog->end && *prog->next != '\n';
16 prog->next++);
17 continue;
18 }
19 return;
20 }
21 }
22
23 int
24 meka_next_token(struct meka_token *res, struct meka_prog *prog)
25 {
26 char c;
27
28 meka_skip_to_token(prog);
29 if (prog->next == prog->end)
30 return 0;
31
32 res->data = prog->next;
33 for (res->len = 0; prog->next != prog->end; res->len++)
34 {
35 c = *prog->next++;
36 if (_is_whitespace(c) || c == '#')
37 break;
38 }
39 return res->len > 0;
40 }
+26
-10
lang/meka.h less more
0 #pragma once
0 #include <stdint.h>
11
2 const uint32_t meka_tag_bool = 0x00000000
3 const uint32_t meka_tag_int = 0x00000001
4 const uint32_t meka_tag_qvec = 0x00000002
5 const uint32_t meka_tag_rvec = 0x00000003
6 const uint32_t meka_tag_reg = 0x00000004
7 const uint32_t meka_tag_sig = 0x00000005
8 const uint32_t meka_tag_chan = 0x00000006
9 const uint32_t meka_tag_cset = 0x00000007
10 const uint32_t meka_tag_mask = 0xFFFFFFF8
2 static const uint32_t meka_tag_bool = 0x00000000;
3 static const uint32_t meka_tag_int = 0x00000001;
4 static const uint32_t meka_tag_qvec = 0x00000002;
5 static const uint32_t meka_tag_rvec = 0x00000003;
6 static const uint32_t meka_tag_reg = 0x00000004;
7 static const uint32_t meka_tag_sig = 0x00000005;
8 static const uint32_t meka_tag_chan = 0x00000006;
9 static const uint32_t meka_tag_cset = 0x00000007;
10 static const uint32_t meka_tag_mask = 0xFFFFFFF8;
1111
1212 typedef uint32_t meka_value;
1313
1414 #define meka_unpack(v) ((v) >> 3)
1515 #define meka_pack(t, v) (((v) << 3) | meka_tag_ # v)
16
17 struct meka_token
18 {
19 char *data;
20 uint_least16_t len;
21 };
22
23 struct meka_prog
24 {
25 char *buf;
26 char *next;
27 char *end;
28 };
29
30 int
31 meka_next_token(struct meka_token *res, struct meka_prog *prog);
+26
-0
lang/test/tokenize.c less more
0 #include <inttypes.h>
1 #include <stdio.h>
2 #include "meka.h"
3
4 #define BUF_SIZE 65536
5 static char buf[BUF_SIZE];
6
7 int
8 main()
9 {
10 struct meka_prog prog;
11 struct meka_token tok;
12 size_t n;
13
14 n = fread(buf, 1, BUF_SIZE, stdin);
15 prog.buf = buf;
16 prog.next = prog.buf;
17 prog.end = prog.buf + n;
18
19 while (meka_next_token(&tok, &prog))
20 {
21 fwrite(tok.data, 1, tok.len, stdout);
22 printf(" (%" PRIuLEAST16 ")\n", tok.len);
23 }
24 return 0;
25 }