tokenizer
haldean
4 years ago
|
0 |
---
|
|
1 |
BasedOnStyle: Webkit
|
|
2 |
IndentWidth: 8
|
|
3 |
---
|
|
4 |
Language: Cpp
|
|
5 |
AlignAfterOpenBracket: AlwaysBreak
|
|
6 |
AlignEscapedNewlines: Left
|
|
7 |
AlwaysBreakAfterReturnType: TopLevelDefinitions
|
|
8 |
BreakBeforeBraces: Custom
|
|
9 |
BreakBeforeBinaryOperators: NonAssignment
|
|
10 |
BraceWrapping:
|
|
11 |
AfterClass: true
|
|
12 |
AfterControlStatement: true
|
|
13 |
AfterEnum: true
|
|
14 |
AfterFunction: true
|
|
15 |
AfterNamespace: true
|
|
16 |
AfterStruct: true
|
|
17 |
AfterUnion: true
|
|
18 |
BeforeElse: true
|
|
19 |
SplitEmptyFunction: false
|
|
20 |
SplitEmptyRecord: false
|
|
21 |
ColumnLimit: 79
|
|
22 |
IndentCaseLabels: true
|
|
23 |
PointerAlignment: Right
|
|
0 |
c_sources = meka.c
|
|
1 |
|
|
2 |
%.o: %.c
|
|
3 |
$(CC) -c -fPIC -MD -o $@ -I. $<
|
|
4 |
|
|
5 |
-include meka.d
|
|
6 |
|
|
7 |
libmeka.a: meka.o
|
|
8 |
$(AR) rcs $@ $<
|
|
9 |
|
|
10 |
test/tokenize: test/tokenize.o libmeka.a
|
|
11 |
$(CC) -o $@ $^
|
|
12 |
|
|
13 |
clean:
|
|
14 |
rm -f libmeka.a test/tokenize meka.o meka.d test/tokenize.o test/tokenize.d
|
|
15 |
|
|
16 |
.PHONY: clean
|
|
0 |
#include "meka.h"
|
|
1 |
|
|
2 |
#define _is_whitespace(c) ((c) == ' ' || (c) == '\n' || (c) == '\r')
|
|
3 |
|
|
4 |
void
|
|
5 |
meka_skip_to_token(struct meka_prog *prog)
|
|
6 |
{
|
|
7 |
char c;
|
|
8 |
while (prog->next != prog->end)
|
|
9 |
{
|
|
10 |
c = *prog->next;
|
|
11 |
if (_is_whitespace(c))
|
|
12 |
continue;
|
|
13 |
if (c == '#')
|
|
14 |
{
|
|
15 |
for (; prog->next != prog->end && *prog->next != '\n';
|
|
16 |
prog->next++);
|
|
17 |
continue;
|
|
18 |
}
|
|
19 |
return;
|
|
20 |
}
|
|
21 |
}
|
|
22 |
|
|
23 |
int
|
|
24 |
meka_next_token(struct meka_token *res, struct meka_prog *prog)
|
|
25 |
{
|
|
26 |
char c;
|
|
27 |
|
|
28 |
meka_skip_to_token(prog);
|
|
29 |
if (prog->next == prog->end)
|
|
30 |
return 0;
|
|
31 |
|
|
32 |
res->data = prog->next;
|
|
33 |
for (res->len = 0; prog->next != prog->end; res->len++)
|
|
34 |
{
|
|
35 |
c = *prog->next++;
|
|
36 |
if (_is_whitespace(c) || c == '#')
|
|
37 |
break;
|
|
38 |
}
|
|
39 |
return res->len > 0;
|
|
40 |
}
|
0 | |
#pragma once
|
|
0 |
#include <stdint.h>
|
1 | 1 |
|
2 | |
const uint32_t meka_tag_bool = 0x00000000
|
3 | |
const uint32_t meka_tag_int = 0x00000001
|
4 | |
const uint32_t meka_tag_qvec = 0x00000002
|
5 | |
const uint32_t meka_tag_rvec = 0x00000003
|
6 | |
const uint32_t meka_tag_reg = 0x00000004
|
7 | |
const uint32_t meka_tag_sig = 0x00000005
|
8 | |
const uint32_t meka_tag_chan = 0x00000006
|
9 | |
const uint32_t meka_tag_cset = 0x00000007
|
10 | |
const uint32_t meka_tag_mask = 0xFFFFFFF8
|
|
2 |
static const uint32_t meka_tag_bool = 0x00000000;
|
|
3 |
static const uint32_t meka_tag_int = 0x00000001;
|
|
4 |
static const uint32_t meka_tag_qvec = 0x00000002;
|
|
5 |
static const uint32_t meka_tag_rvec = 0x00000003;
|
|
6 |
static const uint32_t meka_tag_reg = 0x00000004;
|
|
7 |
static const uint32_t meka_tag_sig = 0x00000005;
|
|
8 |
static const uint32_t meka_tag_chan = 0x00000006;
|
|
9 |
static const uint32_t meka_tag_cset = 0x00000007;
|
|
10 |
static const uint32_t meka_tag_mask = 0xFFFFFFF8;
|
11 | 11 |
|
12 | 12 |
typedef uint32_t meka_value;
|
13 | 13 |
|
14 | 14 |
#define meka_unpack(v) ((v) >> 3)
|
15 | 15 |
#define meka_pack(t, v) (((v) << 3) | meka_tag_ # v)
|
|
16 |
|
|
17 |
struct meka_token
|
|
18 |
{
|
|
19 |
char *data;
|
|
20 |
uint_least16_t len;
|
|
21 |
};
|
|
22 |
|
|
23 |
struct meka_prog
|
|
24 |
{
|
|
25 |
char *buf;
|
|
26 |
char *next;
|
|
27 |
char *end;
|
|
28 |
};
|
|
29 |
|
|
30 |
int
|
|
31 |
meka_next_token(struct meka_token *res, struct meka_prog *prog);
|
|
0 |
#include <inttypes.h>
|
|
1 |
#include <stdio.h>
|
|
2 |
#include "meka.h"
|
|
3 |
|
|
4 |
#define BUF_SIZE 65536
|
|
5 |
static char buf[BUF_SIZE];
|
|
6 |
|
|
7 |
int
|
|
8 |
main()
|
|
9 |
{
|
|
10 |
struct meka_prog prog;
|
|
11 |
struct meka_token tok;
|
|
12 |
size_t n;
|
|
13 |
|
|
14 |
n = fread(buf, 1, BUF_SIZE, stdin);
|
|
15 |
prog.buf = buf;
|
|
16 |
prog.next = prog.buf;
|
|
17 |
prog.end = prog.buf + n;
|
|
18 |
|
|
19 |
while (meka_next_token(&tok, &prog))
|
|
20 |
{
|
|
21 |
fwrite(tok.data, 1, tok.len, stdout);
|
|
22 |
printf(" (%" PRIuLEAST16 ")\n", tok.len);
|
|
23 |
}
|
|
24 |
return 0;
|
|
25 |
}
|