Merge branch 'main' of https://github.com/washh/ivy
This commit is contained in:
@@ -1,6 +1,10 @@
|
|||||||
cmake_minimum_required(VERSION 3.5)
|
cmake_minimum_required(VERSION 3.5)
|
||||||
project(ivy C)
|
project(ivy C)
|
||||||
|
|
||||||
|
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
|
||||||
|
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
|
||||||
|
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
|
||||||
|
|
||||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
|
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
|
||||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
|
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
|
||||||
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
|
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
|
||||||
|
|||||||
@@ -44,7 +44,24 @@ static int compile_file(const char *path)
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
printf("read token!\n");
|
printf("%s", ivy_lex_token_type_to_string(tok->t_type));
|
||||||
|
|
||||||
|
switch (tok->t_type) {
|
||||||
|
case IVY_TOK_IDENT:
|
||||||
|
case IVY_TOK_LABEL:
|
||||||
|
printf("(%s)", tok->t_str);
|
||||||
|
break;
|
||||||
|
case IVY_TOK_SYMBOL:
|
||||||
|
printf("(%s)", ivy_symbol_to_string(tok->t_symbol));
|
||||||
|
break;
|
||||||
|
case IVY_TOK_KEYWORD:
|
||||||
|
printf("(%s)", ivy_keyword_to_string(tok->t_keyword));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
int r = (lex.lex_status == IVY_OK || lex.lex_status == IVY_ERR_EOF) ? 0
|
int r = (lex.lex_status == IVY_OK || lex.lex_status == IVY_ERR_EOF) ? 0
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ syn match ivyType /\<[A-Z]\{1,2\}\([a-z0-9]\+[A-Z]\{0,
|
|||||||
syn keyword ivyStorage class protocol
|
syn keyword ivyStorage class protocol
|
||||||
syn keyword ivyRepeat for while in break continue
|
syn keyword ivyRepeat for while in break continue
|
||||||
syn keyword ivyConditional if elif else unless match
|
syn keyword ivyConditional if elif else unless match
|
||||||
syn match ivySelectorLabel /\<[a-z]\([A-Za-z0-9_]\+\)\:/
|
syn match ivySelectorLabel /\<[a-z]\([A-Za-z0-9_]\+\)\:\(\:\)\@!/
|
||||||
syn match ivySelectorLabel /\<\([a-z]\([A-Za-z0-9_]\+\)\:\)\+/
|
syn match ivySelectorLabel /\<\([a-z]\([A-Za-z0-9_]\+\)\:\(\:\)\@!\)\+/
|
||||||
syn match ivyUnnamedVariable /\<_\>/
|
syn match ivyUnnamedVariable /\<_\>/
|
||||||
|
|
||||||
syn match ivyAtomName /#[a-z][a-z0-9_:/]*\>/
|
syn match ivyAtomName /#[a-z][a-z0-9_:/]*\>/
|
||||||
@@ -31,7 +31,8 @@ syn match ivyLineContinuation /\\\n/
|
|||||||
" Modifiers
|
" Modifiers
|
||||||
syn keyword ivyException try catch finally throw
|
syn keyword ivyException try catch finally throw
|
||||||
|
|
||||||
syn keyword ivyBuiltinVar self error cout cin cerr
|
syn match ivySelfVar /\<self\([^a-zA-Z0-9_]\)\@=/
|
||||||
|
syn keyword ivyBuiltinVar error cout cin cerr
|
||||||
|
|
||||||
syn keyword ivyUnspecifiedStatement end package use as then do get set
|
syn keyword ivyUnspecifiedStatement end package use as then do get set
|
||||||
|
|
||||||
@@ -137,6 +138,7 @@ hi def link ivyStorage Keyword
|
|||||||
hi def link ivyIsAs Keyword
|
hi def link ivyIsAs Keyword
|
||||||
hi def link ivyAccessor Keyword
|
hi def link ivyAccessor Keyword
|
||||||
hi def link ivyBuiltinVar @variable.builtin
|
hi def link ivyBuiltinVar @variable.builtin
|
||||||
|
hi def link ivySelfVar @variable.builtin
|
||||||
|
|
||||||
hi def link ivyStatement Statement
|
hi def link ivyStatement Statement
|
||||||
hi def link ivyRepeat Repeat
|
hi def link ivyRepeat Repeat
|
||||||
|
|||||||
@@ -57,17 +57,20 @@ enum ivy_symbol {
|
|||||||
IVY_SYM_COLON,
|
IVY_SYM_COLON,
|
||||||
IVY_SYM_DOUBLE_COLON,
|
IVY_SYM_DOUBLE_COLON,
|
||||||
IVY_SYM_PLUS,
|
IVY_SYM_PLUS,
|
||||||
IVY_SYM_MINUS,
|
IVY_SYM_HYPHEN,
|
||||||
|
IVY_SYM_DOUBLE_HYPHEN,
|
||||||
IVY_SYM_FORWARD_SLASH,
|
IVY_SYM_FORWARD_SLASH,
|
||||||
IVY_SYM_ASTERISK,
|
IVY_SYM_ASTERISK,
|
||||||
IVY_SYM_PERCENT,
|
IVY_SYM_PERCENT,
|
||||||
IVY_SYM_AMPERSAND,
|
IVY_SYM_AMPERSAND,
|
||||||
IVY_SYM_EQUAL,
|
IVY_SYM_EQUAL,
|
||||||
IVY_SYM_DOUBLE_EQUAL,
|
IVY_SYM_DOUBLE_EQUAL,
|
||||||
|
IVY_SYM_DOUBLE_LEFT_ANGLE,
|
||||||
|
IVY_SYM_DOUBLE_RIGHT_ANGLE,
|
||||||
IVY_SYM_DOUBLE_LEFT_ANGLE_EQUAL,
|
IVY_SYM_DOUBLE_LEFT_ANGLE_EQUAL,
|
||||||
IVY_SYM_DOUBLE_RIGHT_ANGLE_EQUAL,
|
IVY_SYM_DOUBLE_RIGHT_ANGLE_EQUAL,
|
||||||
IVY_SYM_PLUS_EQUAL,
|
IVY_SYM_PLUS_EQUAL,
|
||||||
IVY_SYM_MINUS_EQUAL,
|
IVY_SYM_HYPHEN_EQUAL,
|
||||||
IVY_SYM_FORWARD_SLASH_EQUAL,
|
IVY_SYM_FORWARD_SLASH_EQUAL,
|
||||||
IVY_SYM_ASTERISK_EQUAL,
|
IVY_SYM_ASTERISK_EQUAL,
|
||||||
IVY_SYM_AMPERSAND_EQUAL,
|
IVY_SYM_AMPERSAND_EQUAL,
|
||||||
@@ -98,10 +101,14 @@ struct ivy_token {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct ivy_lexer_symbol_node;
|
||||||
|
|
||||||
struct ivy_lexer {
|
struct ivy_lexer {
|
||||||
struct ivy_line_source *lex_source;
|
struct ivy_line_source *lex_source;
|
||||||
enum ivy_status lex_status;
|
enum ivy_status lex_status;
|
||||||
struct ivy_token *lex_queue;
|
struct ivy_token *lex_queue;
|
||||||
|
struct ivy_lexer_symbol_node *lex_sym_tree;
|
||||||
|
enum ivy_token_type lex_prev_token;
|
||||||
|
|
||||||
char *lex_linebuf;
|
char *lex_linebuf;
|
||||||
size_t lex_linebuf_len;
|
size_t lex_linebuf_len;
|
||||||
@@ -117,4 +124,8 @@ IVY_API struct ivy_token *ivy_lexer_read(struct ivy_lexer *lex);
|
|||||||
|
|
||||||
IVY_API void ivy_token_destroy(struct ivy_token *tok);
|
IVY_API void ivy_token_destroy(struct ivy_token *tok);
|
||||||
|
|
||||||
|
extern const char *ivy_lex_token_type_to_string(enum ivy_token_type type);
|
||||||
|
extern const char *ivy_keyword_to_string(enum ivy_keyword keyword);
|
||||||
|
extern const char *ivy_symbol_to_string(enum ivy_symbol sym);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
439
lang/lex.c
439
lang/lex.c
@@ -1,7 +1,10 @@
|
|||||||
|
#include <blue/core/hash.h>
|
||||||
|
#include <blue/core/queue.h>
|
||||||
#include <blue/object/string.h>
|
#include <blue/object/string.h>
|
||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
#include <ivy/lang/lex.h>
|
#include <ivy/lang/lex.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
@@ -12,9 +15,18 @@
|
|||||||
.id = (i), .name = (n) \
|
.id = (i), .name = (n) \
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct ivy_lexer_symbol_node {
|
||||||
|
char s_char;
|
||||||
|
enum ivy_symbol s_id;
|
||||||
|
|
||||||
|
b_queue_entry s_entry;
|
||||||
|
b_queue s_children;
|
||||||
|
};
|
||||||
|
|
||||||
struct lex_token_def {
|
struct lex_token_def {
|
||||||
int id;
|
int id;
|
||||||
const char *name;
|
const char *name;
|
||||||
|
uint64_t name_hash;
|
||||||
};
|
};
|
||||||
|
|
||||||
static struct lex_token_def keywords[] = {
|
static struct lex_token_def keywords[] = {
|
||||||
@@ -54,17 +66,22 @@ static struct lex_token_def symbols[] = {
|
|||||||
LEX_TOKEN_DEF(IVY_SYM_COLON, ":"),
|
LEX_TOKEN_DEF(IVY_SYM_COLON, ":"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_COLON, "::"),
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_COLON, "::"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_PLUS, "+"),
|
LEX_TOKEN_DEF(IVY_SYM_PLUS, "+"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_MINUS, "-"),
|
LEX_TOKEN_DEF(IVY_SYM_HYPHEN, "-"),
|
||||||
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_HYPHEN, "--"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_FORWARD_SLASH, "/"),
|
LEX_TOKEN_DEF(IVY_SYM_FORWARD_SLASH, "/"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_ASTERISK, "*"),
|
LEX_TOKEN_DEF(IVY_SYM_ASTERISK, "*"),
|
||||||
|
LEX_TOKEN_DEF(IVY_SYM_FORWARD_SLASH_ASTERISK, "/*"),
|
||||||
|
LEX_TOKEN_DEF(IVY_SYM_ASTERISK_FORWARD_SLASH, "*/"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_PERCENT, "%"),
|
LEX_TOKEN_DEF(IVY_SYM_PERCENT, "%"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_AMPERSAND, "&"),
|
LEX_TOKEN_DEF(IVY_SYM_AMPERSAND, "&"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_EQUAL, "="),
|
LEX_TOKEN_DEF(IVY_SYM_EQUAL, "="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_EQUAL, "=="),
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_EQUAL, "=="),
|
||||||
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_LEFT_ANGLE, "<<"),
|
||||||
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_RIGHT_ANGLE, ">>"),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_LEFT_ANGLE_EQUAL, "<<="),
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_LEFT_ANGLE_EQUAL, "<<="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_RIGHT_ANGLE_EQUAL, ">>="),
|
LEX_TOKEN_DEF(IVY_SYM_DOUBLE_RIGHT_ANGLE_EQUAL, ">>="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_PLUS_EQUAL, "+="),
|
LEX_TOKEN_DEF(IVY_SYM_PLUS_EQUAL, "+="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_MINUS_EQUAL, "-="),
|
LEX_TOKEN_DEF(IVY_SYM_HYPHEN_EQUAL, "-="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_FORWARD_SLASH_EQUAL, "/="),
|
LEX_TOKEN_DEF(IVY_SYM_FORWARD_SLASH_EQUAL, "/="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_ASTERISK_EQUAL, "*="),
|
LEX_TOKEN_DEF(IVY_SYM_ASTERISK_EQUAL, "*="),
|
||||||
LEX_TOKEN_DEF(IVY_SYM_AMPERSAND_EQUAL, "&="),
|
LEX_TOKEN_DEF(IVY_SYM_AMPERSAND_EQUAL, "&="),
|
||||||
@@ -82,15 +99,153 @@ static struct lex_token_def symbols[] = {
|
|||||||
};
|
};
|
||||||
static const size_t nr_symbols = sizeof symbols / sizeof symbols[0];
|
static const size_t nr_symbols = sizeof symbols / sizeof symbols[0];
|
||||||
|
|
||||||
|
static struct ivy_lexer_symbol_node *get_symbol_node(
|
||||||
|
struct ivy_lexer_symbol_node *node, char c)
|
||||||
|
{
|
||||||
|
b_queue_iterator it;
|
||||||
|
b_queue_foreach (&it, &node->s_children) {
|
||||||
|
struct ivy_lexer_symbol_node *child = b_unbox(
|
||||||
|
struct ivy_lexer_symbol_node, it.entry, s_entry);
|
||||||
|
if (child->s_char == c) {
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static enum ivy_status put_symbol(
|
||||||
|
struct ivy_lexer_symbol_node *tree, struct lex_token_def *sym)
|
||||||
|
{
|
||||||
|
for (size_t i = 0; sym->name[i]; i++) {
|
||||||
|
char c = sym->name[i];
|
||||||
|
struct ivy_lexer_symbol_node *child = get_symbol_node(tree, c);
|
||||||
|
if (child) {
|
||||||
|
tree = child;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
child = malloc(sizeof *child);
|
||||||
|
if (!child) {
|
||||||
|
return IVY_ERR_NO_MEMORY;
|
||||||
|
}
|
||||||
|
|
||||||
|
child->s_id = IVY_SYM_NONE;
|
||||||
|
child->s_char = c;
|
||||||
|
|
||||||
|
b_queue_push_back(&tree->s_children, &child->s_entry);
|
||||||
|
tree = child;
|
||||||
|
}
|
||||||
|
|
||||||
|
tree->s_id = sym->id;
|
||||||
|
return IVY_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void destroy_symbol_tree(struct ivy_lexer_symbol_node *tree)
|
||||||
|
{
|
||||||
|
b_queue_iterator it;
|
||||||
|
b_queue_iterator_begin(&tree->s_children, &it);
|
||||||
|
while (b_queue_iterator_is_valid(&it)) {
|
||||||
|
struct ivy_lexer_symbol_node *node = b_unbox(
|
||||||
|
struct ivy_lexer_symbol_node, it.entry, s_entry);
|
||||||
|
b_queue_iterator_erase(&it);
|
||||||
|
|
||||||
|
destroy_symbol_tree(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
free(tree);
|
||||||
|
}
|
||||||
|
|
||||||
|
static struct ivy_lexer_symbol_node *build_symbol_tree(void)
|
||||||
|
{
|
||||||
|
struct ivy_lexer_symbol_node *root = malloc(sizeof *root);
|
||||||
|
if (!root) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
memset(root, 0x0, sizeof *root);
|
||||||
|
root->s_id = IVY_SYM_NONE;
|
||||||
|
|
||||||
|
enum ivy_status status = IVY_OK;
|
||||||
|
for (size_t i = 0; i < nr_symbols; i++) {
|
||||||
|
status = put_symbol(root, &symbols[i]);
|
||||||
|
|
||||||
|
if (status != IVY_OK) {
|
||||||
|
destroy_symbol_tree(root);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void print_symbol_node(struct ivy_lexer_symbol_node *node, int depth)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < depth; i++) {
|
||||||
|
fputs(" ", stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("%c", node->s_char);
|
||||||
|
|
||||||
|
if (node->s_id != IVY_SYM_NONE) {
|
||||||
|
printf(" (%s)", ivy_symbol_to_string(node->s_id));
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("\n");
|
||||||
|
|
||||||
|
b_queue_iterator it;
|
||||||
|
b_queue_foreach (&it, &node->s_children) {
|
||||||
|
struct ivy_lexer_symbol_node *child = b_unbox(
|
||||||
|
struct ivy_lexer_symbol_node, it.entry, s_entry);
|
||||||
|
|
||||||
|
print_symbol_node(child, depth + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void init_keywords(void)
|
||||||
|
{
|
||||||
|
for (size_t i = 0; i < nr_keywords; i++) {
|
||||||
|
keywords[i].name_hash = b_hash_string(keywords[i].name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static enum ivy_keyword find_keyword_by_name(const char *s)
|
||||||
|
{
|
||||||
|
uint64_t s_hash = b_hash_string(s);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < nr_keywords; i++) {
|
||||||
|
struct lex_token_def *def = &keywords[i];
|
||||||
|
|
||||||
|
if (s_hash != def->name_hash) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (strcmp(s, def->name) != 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return def->id;
|
||||||
|
}
|
||||||
|
|
||||||
|
return IVY_KW_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
enum ivy_status ivy_lexer_init(struct ivy_lexer *lex)
|
enum ivy_status ivy_lexer_init(struct ivy_lexer *lex)
|
||||||
{
|
{
|
||||||
memset(lex, 0x0, sizeof *lex);
|
memset(lex, 0x0, sizeof *lex);
|
||||||
|
|
||||||
lex->lex_status = IVY_OK;
|
lex->lex_status = IVY_OK;
|
||||||
|
lex->lex_prev_token = IVY_TOK_NONE;
|
||||||
|
|
||||||
lex->lex_linebuf = malloc(LINEBUF_DEFAULT_CAPACITY);
|
lex->lex_linebuf = malloc(LINEBUF_DEFAULT_CAPACITY);
|
||||||
lex->lex_linebuf_cap = LINEBUF_DEFAULT_CAPACITY;
|
lex->lex_linebuf_cap = LINEBUF_DEFAULT_CAPACITY;
|
||||||
|
|
||||||
|
lex->lex_sym_tree = build_symbol_tree();
|
||||||
|
print_symbol_node(lex->lex_sym_tree, 0);
|
||||||
|
|
||||||
|
/* TODO only do keyword initialisation once */
|
||||||
|
init_keywords();
|
||||||
|
|
||||||
return IVY_OK;
|
return IVY_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,6 +261,10 @@ void ivy_lexer_finish(struct ivy_lexer *lex)
|
|||||||
free(lex->lex_linebuf);
|
free(lex->lex_linebuf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (lex->lex_sym_tree) {
|
||||||
|
destroy_symbol_tree(lex->lex_sym_tree);
|
||||||
|
}
|
||||||
|
|
||||||
memset(lex, 0x0, sizeof *lex);
|
memset(lex, 0x0, sizeof *lex);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,9 +274,15 @@ static enum ivy_status refill_linebuf(struct ivy_lexer *lex)
|
|||||||
return IVY_ERR_EOF;
|
return IVY_ERR_EOF;
|
||||||
}
|
}
|
||||||
|
|
||||||
return ivy_line_source_readline(
|
enum ivy_status status = ivy_line_source_readline(
|
||||||
lex->lex_source, lex->lex_linebuf, lex->lex_linebuf_cap,
|
lex->lex_source, lex->lex_linebuf, lex->lex_linebuf_cap,
|
||||||
&lex->lex_linebuf_len, NULL);
|
&lex->lex_linebuf_len, NULL);
|
||||||
|
|
||||||
|
if (status == IVY_OK) {
|
||||||
|
lex->lex_linebuf_ptr = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int peek(struct ivy_lexer *lex)
|
static int peek(struct ivy_lexer *lex)
|
||||||
@@ -140,6 +305,30 @@ static int peek(struct ivy_lexer *lex)
|
|||||||
return c;
|
return c;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int peek_next(struct ivy_lexer *lex)
|
||||||
|
{
|
||||||
|
enum ivy_status status = IVY_OK;
|
||||||
|
|
||||||
|
if (lex->lex_linebuf_ptr >= lex->lex_linebuf_len) {
|
||||||
|
status = refill_linebuf(lex);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status != IVY_OK) {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lex->lex_linebuf_len == 0) {
|
||||||
|
return IVY_ERR_EOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lex->lex_linebuf_ptr + 1 >= lex->lex_linebuf_len) {
|
||||||
|
return IVY_ERR_EOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
int c = lex->lex_linebuf[lex->lex_linebuf_ptr + 1];
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
|
||||||
static int advance(struct ivy_lexer *lex)
|
static int advance(struct ivy_lexer *lex)
|
||||||
{
|
{
|
||||||
enum ivy_status status = IVY_OK;
|
enum ivy_status status = IVY_OK;
|
||||||
@@ -193,11 +382,16 @@ static enum ivy_status push_token(struct ivy_lexer *lex, struct ivy_token *tok)
|
|||||||
}
|
}
|
||||||
|
|
||||||
*slot = tok;
|
*slot = tok;
|
||||||
|
lex->lex_prev_token = tok->t_type;
|
||||||
return IVY_OK;
|
return IVY_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
static enum ivy_status push_linefeed(struct ivy_lexer *lex)
|
static enum ivy_status push_linefeed(struct ivy_lexer *lex)
|
||||||
{
|
{
|
||||||
|
if (lex->lex_prev_token == IVY_TOK_LINEFEED) {
|
||||||
|
return IVY_OK;
|
||||||
|
}
|
||||||
|
|
||||||
struct ivy_token *tok = malloc(sizeof *tok);
|
struct ivy_token *tok = malloc(sizeof *tok);
|
||||||
if (!tok) {
|
if (!tok) {
|
||||||
return IVY_ERR_NO_MEMORY;
|
return IVY_ERR_NO_MEMORY;
|
||||||
@@ -223,22 +417,126 @@ static enum ivy_status push_symbol(struct ivy_lexer *lex, enum ivy_symbol sym)
|
|||||||
return push_token(lex, tok);
|
return push_token(lex, tok);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static enum ivy_status push_keyword(struct ivy_lexer *lex, enum ivy_keyword keyword)
|
||||||
|
{
|
||||||
|
struct ivy_token *tok = malloc(sizeof *tok);
|
||||||
|
if (!tok) {
|
||||||
|
return IVY_ERR_NO_MEMORY;
|
||||||
|
}
|
||||||
|
|
||||||
|
memset(tok, 0x0, sizeof *tok);
|
||||||
|
|
||||||
|
tok->t_type = IVY_TOK_KEYWORD;
|
||||||
|
tok->t_keyword = keyword;
|
||||||
|
return push_token(lex, tok);
|
||||||
|
}
|
||||||
|
|
||||||
|
static enum ivy_status read_line_comment(struct ivy_lexer *lex)
|
||||||
|
{
|
||||||
|
while (true) {
|
||||||
|
int c = advance(lex);
|
||||||
|
|
||||||
|
if (c == IVY_ERR_EOF || c == '\n') {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (c < 0) {
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return IVY_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static enum ivy_status read_block_comment(struct ivy_lexer *lex)
|
||||||
|
{
|
||||||
|
int depth = 1;
|
||||||
|
char buf[2] = {0};
|
||||||
|
|
||||||
|
while (depth > 0) {
|
||||||
|
int c = peek(lex);
|
||||||
|
if (c < 0) {
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!buf[0]) {
|
||||||
|
buf[0] = c;
|
||||||
|
} else if (!buf[1]) {
|
||||||
|
buf[1] = c;
|
||||||
|
} else {
|
||||||
|
buf[0] = buf[1];
|
||||||
|
buf[1] = c;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (buf[0] == '/' && buf[1] == '*') {
|
||||||
|
depth++;
|
||||||
|
} else if (buf[0] == '*' && buf[1] == '/') {
|
||||||
|
depth--;
|
||||||
|
}
|
||||||
|
|
||||||
|
advance(lex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return IVY_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static enum ivy_status read_symbol(struct ivy_lexer *lex)
|
||||||
|
{
|
||||||
|
char sym_buf[32];
|
||||||
|
unsigned int sym_len = 0;
|
||||||
|
struct ivy_lexer_symbol_node *node = lex->lex_sym_tree;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
int c = peek(lex);
|
||||||
|
|
||||||
|
struct ivy_lexer_symbol_node *next = get_symbol_node(node, c);
|
||||||
|
if (!next) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
node = next;
|
||||||
|
advance(lex);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!node || node->s_id == IVY_SYM_NONE) {
|
||||||
|
return IVY_ERR_BAD_SYNTAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (node->s_id == IVY_SYM_FORWARD_SLASH_ASTERISK) {
|
||||||
|
return read_block_comment(lex);
|
||||||
|
} else if (node->s_id == IVY_SYM_DOUBLE_HYPHEN) {
|
||||||
|
return read_line_comment(lex);
|
||||||
|
}
|
||||||
|
|
||||||
|
push_symbol(lex, node->s_id);
|
||||||
|
return IVY_OK;
|
||||||
|
}
|
||||||
|
|
||||||
static enum ivy_status read_ident(struct ivy_lexer *lex)
|
static enum ivy_status read_ident(struct ivy_lexer *lex)
|
||||||
{
|
{
|
||||||
b_string *str = b_string_create();
|
b_string *str = b_string_create();
|
||||||
int c = peek(lex);
|
bool label = false;
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
int c = peek(lex);
|
||||||
|
|
||||||
if (c < 0) {
|
if (c < 0) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (c == ':' && peek_next(lex) != ':') {
|
||||||
|
advance(lex);
|
||||||
|
label = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
if (!isalnum(c) && c != '_') {
|
if (!isalnum(c) && c != '_') {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
char s[2] = {c, 0};
|
char s[2] = {c, 0};
|
||||||
b_string_append_cstr(str, s);
|
b_string_append_cstr(str, s);
|
||||||
|
advance(lex);
|
||||||
}
|
}
|
||||||
|
|
||||||
const char *s = b_string_ptr(str);
|
const char *s = b_string_ptr(str);
|
||||||
@@ -247,12 +545,18 @@ static enum ivy_status read_ident(struct ivy_lexer *lex)
|
|||||||
push_symbol(lex, IVY_SYM_UNDERSCORE);
|
push_symbol(lex, IVY_SYM_UNDERSCORE);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct ivy_token *tok = create_token(IVY_TOK_IDENT);
|
enum ivy_keyword keyword = IVY_KW_NONE;
|
||||||
|
if (!label && (keyword = find_keyword_by_name(s)) != IVY_KW_NONE) {
|
||||||
|
b_string_release(str);
|
||||||
|
return push_keyword(lex, keyword);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ivy_token *tok
|
||||||
|
= create_token(label ? IVY_TOK_LABEL : IVY_TOK_IDENT);
|
||||||
tok->t_str = b_string_steal(str);
|
tok->t_str = b_string_steal(str);
|
||||||
b_string_release(str);
|
b_string_release(str);
|
||||||
|
|
||||||
push_token(lex, tok);
|
return push_token(lex, tok);
|
||||||
return IVY_OK;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static enum ivy_status pump_tokens(struct ivy_lexer *lex)
|
static enum ivy_status pump_tokens(struct ivy_lexer *lex)
|
||||||
@@ -277,6 +581,15 @@ static enum ivy_status pump_tokens(struct ivy_lexer *lex)
|
|||||||
return push_linefeed(lex);
|
return push_linefeed(lex);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
while (isspace(c)) {
|
||||||
|
advance(lex);
|
||||||
|
c = peek(lex);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (char_can_begin_symbol(c)) {
|
||||||
|
return read_symbol(lex);
|
||||||
|
}
|
||||||
|
|
||||||
if (isalpha(c) || c == '_') {
|
if (isalpha(c) || c == '_') {
|
||||||
return read_ident(lex);
|
return read_ident(lex);
|
||||||
}
|
}
|
||||||
@@ -288,15 +601,16 @@ struct ivy_token *ivy_lexer_peek(struct ivy_lexer *lex)
|
|||||||
{
|
{
|
||||||
enum ivy_status status = IVY_OK;
|
enum ivy_status status = IVY_OK;
|
||||||
|
|
||||||
if (!lex->lex_queue) {
|
while (!lex->lex_queue) {
|
||||||
status = pump_tokens(lex);
|
status = pump_tokens(lex);
|
||||||
}
|
|
||||||
|
|
||||||
if (status != IVY_OK) {
|
if (status != IVY_OK) {
|
||||||
lex->lex_status = status;
|
lex->lex_status = status;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lex->lex_status = status;
|
||||||
struct ivy_token *tok = lex->lex_queue;
|
struct ivy_token *tok = lex->lex_queue;
|
||||||
return tok;
|
return tok;
|
||||||
}
|
}
|
||||||
@@ -305,14 +619,14 @@ struct ivy_token *ivy_lexer_read(struct ivy_lexer *lex)
|
|||||||
{
|
{
|
||||||
enum ivy_status status = IVY_OK;
|
enum ivy_status status = IVY_OK;
|
||||||
|
|
||||||
if (!lex->lex_queue) {
|
while (!lex->lex_queue) {
|
||||||
status = pump_tokens(lex);
|
status = pump_tokens(lex);
|
||||||
}
|
|
||||||
|
|
||||||
if (status != IVY_OK) {
|
if (status != IVY_OK) {
|
||||||
lex->lex_status = status;
|
lex->lex_status = status;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
struct ivy_token *tok = lex->lex_queue;
|
struct ivy_token *tok = lex->lex_queue;
|
||||||
lex->lex_queue = lex->lex_queue->t_next;
|
lex->lex_queue = lex->lex_queue->t_next;
|
||||||
@@ -333,3 +647,106 @@ void ivy_token_destroy(struct ivy_token *tok)
|
|||||||
|
|
||||||
free(tok);
|
free(tok);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define ENUM_STR(x) \
|
||||||
|
case x: \
|
||||||
|
return #x
|
||||||
|
|
||||||
|
const char *ivy_lex_token_type_to_string(enum ivy_token_type type)
|
||||||
|
{
|
||||||
|
switch (type) {
|
||||||
|
ENUM_STR(IVY_TOK_NONE);
|
||||||
|
ENUM_STR(IVY_TOK_KEYWORD);
|
||||||
|
ENUM_STR(IVY_TOK_SYMBOL);
|
||||||
|
ENUM_STR(IVY_TOK_ATOM);
|
||||||
|
ENUM_STR(IVY_TOK_NUMBER);
|
||||||
|
ENUM_STR(IVY_TOK_LABEL);
|
||||||
|
ENUM_STR(IVY_TOK_IDENT);
|
||||||
|
ENUM_STR(IVY_TOK_STRING);
|
||||||
|
ENUM_STR(IVY_TOK_STR_START);
|
||||||
|
ENUM_STR(IVY_TOK_STR_END);
|
||||||
|
ENUM_STR(IVY_TOK_LINEFEED);
|
||||||
|
default:
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *ivy_keyword_to_string(enum ivy_keyword keyword)
|
||||||
|
{
|
||||||
|
switch (keyword) {
|
||||||
|
ENUM_STR(IVY_KW_NONE);
|
||||||
|
ENUM_STR(IVY_KW_PACKAGE);
|
||||||
|
ENUM_STR(IVY_KW_USE);
|
||||||
|
ENUM_STR(IVY_KW_CLASS);
|
||||||
|
ENUM_STR(IVY_KW_PROTOCOL);
|
||||||
|
ENUM_STR(IVY_KW_TRY);
|
||||||
|
ENUM_STR(IVY_KW_THROW);
|
||||||
|
ENUM_STR(IVY_KW_CATCH);
|
||||||
|
ENUM_STR(IVY_KW_IF);
|
||||||
|
ENUM_STR(IVY_KW_AND);
|
||||||
|
ENUM_STR(IVY_KW_OR);
|
||||||
|
ENUM_STR(IVY_KW_IS);
|
||||||
|
ENUM_STR(IVY_KW_NOT);
|
||||||
|
ENUM_STR(IVY_KW_ELSE);
|
||||||
|
ENUM_STR(IVY_KW_WHILE);
|
||||||
|
ENUM_STR(IVY_KW_FOR);
|
||||||
|
ENUM_STR(IVY_KW_MATCH);
|
||||||
|
ENUM_STR(IVY_KW_UNLESS);
|
||||||
|
ENUM_STR(IVY_KW_IN);
|
||||||
|
ENUM_STR(IVY_KW_DO);
|
||||||
|
ENUM_STR(IVY_KW_END);
|
||||||
|
default:
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *ivy_symbol_to_string(enum ivy_symbol sym)
|
||||||
|
{
|
||||||
|
switch (sym) {
|
||||||
|
ENUM_STR(IVY_SYM_NONE);
|
||||||
|
ENUM_STR(IVY_SYM_DOT);
|
||||||
|
ENUM_STR(IVY_SYM_LEFT_BRACE);
|
||||||
|
ENUM_STR(IVY_SYM_RIGHT_BRACE);
|
||||||
|
ENUM_STR(IVY_SYM_LEFT_BRACKET);
|
||||||
|
ENUM_STR(IVY_SYM_RIGHT_BRACKET);
|
||||||
|
ENUM_STR(IVY_SYM_LEFT_PAREN);
|
||||||
|
ENUM_STR(IVY_SYM_RIGHT_PAREN);
|
||||||
|
ENUM_STR(IVY_SYM_LEFT_ANGLE);
|
||||||
|
ENUM_STR(IVY_SYM_RIGHT_ANGLE);
|
||||||
|
ENUM_STR(IVY_SYM_COLON);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_COLON);
|
||||||
|
ENUM_STR(IVY_SYM_PLUS);
|
||||||
|
ENUM_STR(IVY_SYM_HYPHEN);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_HYPHEN);
|
||||||
|
ENUM_STR(IVY_SYM_FORWARD_SLASH);
|
||||||
|
ENUM_STR(IVY_SYM_ASTERISK);
|
||||||
|
ENUM_STR(IVY_SYM_PERCENT);
|
||||||
|
ENUM_STR(IVY_SYM_AMPERSAND);
|
||||||
|
ENUM_STR(IVY_SYM_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_LEFT_ANGLE);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_RIGHT_ANGLE);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_LEFT_ANGLE_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_DOUBLE_RIGHT_ANGLE_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_PLUS_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_HYPHEN_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_FORWARD_SLASH_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_ASTERISK_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_AMPERSAND_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_PIPE_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_PERCENT_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_CARET_EQUAL);
|
||||||
|
ENUM_STR(IVY_SYM_BANG);
|
||||||
|
ENUM_STR(IVY_SYM_PIPE);
|
||||||
|
ENUM_STR(IVY_SYM_CARET);
|
||||||
|
ENUM_STR(IVY_SYM_UNDERSCORE);
|
||||||
|
ENUM_STR(IVY_SYM_COMMA);
|
||||||
|
ENUM_STR(IVY_SYM_DOLLAR);
|
||||||
|
ENUM_STR(IVY_SYM_RIGHT_ARROW);
|
||||||
|
ENUM_STR(IVY_SYM_BIG_RIGHT_ARROW);
|
||||||
|
ENUM_STR(IVY_SYM_FORWARD_SLASH_ASTERISK);
|
||||||
|
ENUM_STR(IVY_SYM_ASTERISK_FORWARD_SLASH);
|
||||||
|
default:
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user