feat: basic lexer with flex

6 files changed, 160 insertions, 24 deletions

diff --git a/.clang-format b/.clang-format
index e7d632d..ddcf0b0 100644
--- a/.clang-format
+++ b/.clang-format
@@ -18,3 +18,4 @@ SortIncludes: false
 Cpp11BracedListStyle: false
 AlignOperands: DontAlign
 AlignAfterOpenBracket: Align
+RemoveBracesLLVM: true
diff --git a/Makefile b/Makefile
index 8d873b9..e1140cf 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,8 @@
 CC      = cc
-YACC    = bison
-CFLAGS  = -std=c99 -Wall -Wextra -Werror -Wpedantic -O2 -Ibin
-SOURCES = ast.c ast.h eval.c eval.h lexer.c lexer.h main.c
-OBJECTS = bin/parser.o bin/lexer.o bin/ast.o bin/eval.o bin/main.o
+LEX     = flex
+CFLAGS  = -std=c17 -Wall -Wextra -Werror -Wpedantic -O2
+SOURCES = lexer.h main.c
+OBJECTS = bin/lexer.o bin/main.o
 SANITIZE = -g -O0 -fsanitize=address,undefined
 
 .PHONY: all debug check format lint clean
@@ -15,42 +15,31 @@ bin:
 sfinae: $(OBJECTS)
 	$(CC) $(CFLAGS) -o $@ $(OBJECTS)
 
-bin/parser.c bin/parser.h: sfinae.y | bin
-	$(YACC) -d -o bin/parser.c sfinae.y
+bin/lexer.c: sfinae.l | bin
+	$(LEX) -o bin/lexer.c sfinae.l
 
-bin/parser.o: bin/parser.c
-	$(CC) -std=c99 -w -I. -c -o $@ bin/parser.c
+bin/lexer.o: bin/lexer.c lexer.h
+	$(CC) -std=c17 -w -I. -c -o $@ bin/lexer.c
 
 bin/%.o: %.c | bin
 	$(CC) $(CFLAGS) -c -o $@ $<
 
-bin/lexer.o: bin/parser.h ast.h lexer.h
-bin/main.o: ast.h eval.h lexer.h
-bin/ast.o: ast.h lexer.h
-bin/eval.o: eval.h ast.h
+bin/main.o: lexer.h
 
 debug: CFLAGS += $(SANITIZE)
 debug: clean sfinae
 
 check: sfinae
 	@for example in examples/*.sfinae; do \
-		expected="$${example%.sfinae}.expected"; \
-		if ./sfinae "$$example" | diff -u "$$expected" - >/dev/null; then \
-			echo "ok   $$example"; \
-		else \
-			echo "FAIL $$example"; ./sfinae "$$example" | diff -u "$$expected" -; exit 1; \
-		fi; \
+		echo "== $$example"; \
+		./sfinae "$$example" || exit 1; \
 	done
 
 format:
 	clang-format -i $(SOURCES)
 
-LINT_FILTER = '[{"name":"parser.h","lines":[[1,1]]}, \
-    {"name":"ast.c"},{"name":"eval.c"},{"name":"lexer.c"},{"name":"main.c"}, \
-    {"name":"ast.h"},{"name":"eval.h"},{"name":"lexer.h"}]'
-
-lint: bin/parser.h
-	clang-tidy --line-filter=$(LINT_FILTER) ast.c eval.c lexer.c main.c -- -std=c99 -Ibin
+lint:
+	clang-tidy main.c -- -std=c17
 
 clean:
 	rm -rf bin sfinae
diff --git a/examples/hello.sfinae b/examples/hello.sfinae
new file mode 100644
index 0000000..0be95fa
--- /dev/null
+++ b/examples/hello.sfinae
@@ -0,0 +1,2 @@
+// prints 5
+::emit<::decltype(::difference<'F', 'A'>)>::commit;
diff --git a/lexer.h b/lexer.h
new file mode 100644
index 0000000..eb7a4d3
--- /dev/null
+++ b/lexer.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stdio.h>
+
+// single characters stand for themselves
+// so the named tokens start past them
+enum Token
+{
+    TOK_END = 0,
+    TOK_COLONCOLON = 256,
+    TOK_EMIT,
+    TOK_COMMIT,
+    TOK_DECLTYPE,
+    TOK_DIFFERENCE,
+    TOK_CHAR
+};
+
+// provided by flex
+// the input
+// the current line
+// and the scanner itself
+extern FILE* yyin;
+extern int yylineno;
+int yylex(void);
+
+// true once the lexer has refused something
+bool lexer_failed(void);
+// the character inside the most recent TOK_CHAR
+char lexer_character(void);
diff --git a/main.c b/main.c
new file mode 100644
index 0000000..44d9025
--- /dev/null
+++ b/main.c
@@ -0,0 +1,56 @@
+#include "lexer.h"
+
+#include <stdio.h>
+
+static const char* token_name(int token)
+{
+    switch (token)
+    {
+        case TOK_COLONCOLON: return "::";
+        case TOK_EMIT: return "emit";
+        case TOK_COMMIT: return "commit";
+        case TOK_DECLTYPE: return "decltype";
+        case TOK_DIFFERENCE: return "difference";
+        default:
+            // single characters and character literals are printed by the caller
+            return NULL;
+    }
+}
+
+static int dump_tokens(void)
+{
+    int token = TOK_END;
+
+    while ((token = yylex()) != TOK_END)
+    {
+        const char* name = token_name(token);
+
+        if (token == TOK_CHAR)
+            (void)printf("'%c'\n", lexer_character());
+        else if (name != NULL)
+            (void)printf("%s\n", name);
+        else
+            (void)printf("%c\n", (char)token);
+    }
+    return lexer_failed() ? 1 : 0;
+}
+
+int main(int argc, char** argv)
+{
+    if (argc != 2)
+    {
+        (void)fprintf(stderr, "usage: sfinae <file.sfinae>\n");
+        return 2;
+    }
+
+    yyin = fopen(argv[1], "r");
+    if (yyin == NULL)
+    {
+        (void)fprintf(stderr, "cannot open %s, is it even real\n", argv[1]);
+        return 1;
+    }
+
+    int status = dump_tokens();
+    (void)fclose(yyin);
+    return status;
+}
diff --git a/sfinae.l b/sfinae.l
new file mode 100644
index 0000000..8453699
--- /dev/null
+++ b/sfinae.l
@@ -0,0 +1,58 @@
+%option noyywrap nounput noinput yylineno
+
+%{
+#include "lexer.h"
+
+#include <stdbool.h>
+#include <stdio.h>
+
+static bool failed = false;
+static char character = '\0';
+
+static char unescape(char escaped)
+{
+    switch (escaped)
+    {
+        case 'n': return '\n';
+        case 't': return '\t';
+        case '0': return '\0';
+        default: return escaped;
+    }
+}
+
+// complains, remembers, and hands back TOK_END so the scan stops where it went wrong
+static int fail(const char* message)
+{
+    (void)fprintf(stderr, "line %d: %s\n", yylineno, message);
+    failed = true;
+    return TOK_END;
+}
+%}
+
+%%
+
+"::"                    return TOK_COLONCOLON;
+"emit"                  return TOK_EMIT;
+"commit"                return TOK_COMMIT;
+"decltype"              return TOK_DECLTYPE;
+"difference"            return TOK_DIFFERENCE;
+[<>(),;]                return yytext[0];
+'\\.'                   character = unescape(yytext[2]); return TOK_CHAR;
+'.'                     character = yytext[1]; return TOK_CHAR;
+"//"[^\n]*              /* a comment */
+[ \t\r\n]+              /* blanks */
+[0-9]+                  return fail("numeric literals are for the weak, subtract two characters");
+[A-Za-z_][A-Za-z0-9_]*  return fail("that is not a notion this language recognises");
+.                       return fail("what is this character even");
+
+%%
+
+bool lexer_failed(void)
+{
+    return failed;
+}
+
+char lexer_character(void)
+{
+    return character;
+}