From 64fac5a9fbcfa809ac9e01197ac997b797a8b5e6 Mon Sep 17 00:00:00 2001 From: Brandon Dyck Date: Sun, 16 Jul 2023 23:30:20 -0600 Subject: [PATCH] Remove Name type and add quoted identifier token --- AST.cs | 17 +++------ Expr.g.cs | 6 ++-- Interpreter.cs | 39 +++++++++++---------- Parser.cs | 91 ++++++++++++++----------------------------------- Pattern.g.cs | 6 ++-- Program.cs | 50 +++++++++++++++++++++------ TODO.txt | 2 +- ast_classes.fsx | 12 +++---- 8 files changed, 103 insertions(+), 120 deletions(-) diff --git a/AST.cs b/AST.cs index 28a5ac9..187acd9 100644 --- a/AST.cs +++ b/AST.cs @@ -3,16 +3,7 @@ using System.Collections.Generic; namespace Finn.AST; -public record Name(string Value, bool Quoted) -{ - public override string ToString() - { - if (this.Quoted) return $"@\"{this.Value}\""; - return this.Value; - } -}; - -public record Field(Name Name, Expr? Value); +public record Field(Token Name, Expr? Value); public record BaseRecord(Expr Value, Field[] Updates); @@ -20,7 +11,7 @@ public partial record Variable { public override string ToString() { - return this.Value.ToString(); + return this.Value.lexeme; } } @@ -48,13 +39,13 @@ public partial record SimplePattern { return "_"; } - return this.Identifier.ToString(); + return this.Identifier.lexeme; } } public abstract record Binding(Expr Value); public record VarBinding(Pattern Pattern, Expr Value) : Binding(Value); -public record FuncBinding(Name Name, Pattern[] Params, Expr Value) : Binding(Value) +public record FuncBinding(Token Name, Pattern[] Params, Expr Value) : Binding(Value) { public override string ToString() { diff --git a/Expr.g.cs b/Expr.g.cs index 963fd75..5611b2e 100644 --- a/Expr.g.cs +++ b/Expr.g.cs @@ -69,7 +69,7 @@ public partial record If(Expr Condition, Expr Then, Expr Else) : Expr() return visitor.visitIfExpr(context, this); } } -public partial record Variable(Name Value) : Expr() +public partial record Variable(Token Value) : Expr() { public override TResult accept(TContext context, IExprVisitor visitor) { @@ -83,7 +83,7 @@ public partial record List(Expr[] Elements) : Expr() return visitor.visitListExpr(context, this); } } -public partial record Variant(Name Tag, Expr? Argument) : Expr() +public partial record Variant(Token Tag, Expr? Argument) : Expr() { public override TResult accept(TContext context, IExprVisitor visitor) { @@ -97,7 +97,7 @@ public partial record Record(Field[] Extensions, BaseRecord? Base) : Expr() return visitor.visitRecordExpr(context, this); } } -public partial record Selector(Expr Left, Name FieldName) : Expr() +public partial record Selector(Expr Left, Token FieldName) : Expr() { public override TResult accept(TContext context, IExprVisitor visitor) { diff --git a/Interpreter.cs b/Interpreter.cs index 72f8ec1..c08bda3 100644 --- a/Interpreter.cs +++ b/Interpreter.cs @@ -31,33 +31,33 @@ public class Env this.enclosing = enclosing; } - public object this[AST.Name name] + public object this[Token identifier] { set { - if (values.ContainsKey(name.Value)) + var name = (string)identifier.literal!; + if (values.ContainsKey(name)) { // TODO use real location info - var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1)); - throw new RuntimeError(tok, $"Cannot redefine variable {name} in same scope."); + throw new RuntimeError(identifier, $"Cannot redefine variable {name} in same scope."); } - values[name.Value] = value; + values[name] = value; } get { + var name = (string)identifier.literal!; try { - return values[name.Value]; + return values[name]; } catch { if (enclosing != null) { - return enclosing[name]; + return enclosing[identifier]; } // TODO use real location info - var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1)); - throw new RuntimeError(tok, $"Undefined variable {name}."); + throw new RuntimeError(identifier, $"Undefined variable {name}."); } } } @@ -98,8 +98,9 @@ public class Interpreter : AST.IExprVisitor var removedLabels = new List(); foreach (var field in pattern.Fields) { - removedLabels.Add(field.Name.Value); - var fieldValue = r.Get(field.Name.Value); + string name = (string)field.Name.literal!; + removedLabels.Add(name); + var fieldValue = r.Get(name); field.accept((fieldValue, env), this); } if (pattern.Rest != null) @@ -129,9 +130,10 @@ public class Interpreter : AST.IExprVisitor switch (obj) { case Variant v: - if (v.Tag != pattern.Tag.Value) + var tag = (string)pattern.Tag.literal!; + if (v.Tag != tag) { - throw new PatternTagMismatchException(pattern.Tag.Value, v.Tag); + throw new PatternTagMismatchException(tag, v.Tag); } if (v.Value == null && pattern.Argument == null) { @@ -502,7 +504,7 @@ public class Interpreter : AST.IExprVisitor HashSet updateLabels = new HashSet(); foreach (AST.Field update in expr.Base.Updates) { - var label = update.Name.Value; + var label = (string)update.Name.literal!; if (updateLabels.Contains(label)) { throw new RuntimeError(tok, "Record updates must be to unique fields."); @@ -525,7 +527,7 @@ public class Interpreter : AST.IExprVisitor HashSet extLabels = new HashSet(); foreach (AST.Field extension in expr.Extensions) { - var label = extension.Name.Value; + var label = (string)extension.Name.literal!; if (extLabels.Contains(label)) { throw new RuntimeError(tok, "Record extensions must have unique field names."); @@ -547,7 +549,7 @@ public class Interpreter : AST.IExprVisitor var r = checkRecordOperand(tok, left); try { - return r.Get(expr.FieldName.Value); + return r.Get((string)expr.FieldName.literal!); } catch { @@ -586,11 +588,12 @@ public class Interpreter : AST.IExprVisitor public object visitVariantExpr(Env env, AST.Variant expr) { + var tag = (string)expr.Tag.literal!; if (expr.Argument == null) { - return new Variant(expr.Tag.Value, null); + return new Variant(tag, null); } - return new Variant(expr.Tag.Value, evaluate(env, expr.Argument)); + return new Variant(tag, evaluate(env, expr.Argument)); } public object visitWhenExpr(Env env, AST.When expr) diff --git a/Parser.cs b/Parser.cs index 0b75ffc..5a8f219 100644 --- a/Parser.cs +++ b/Parser.cs @@ -71,9 +71,9 @@ class Parser return tokens[current - 1]; } - private Token consume(TokenType type, string message) + private Token consume(string message, params TokenType[] types) { - if (check(type)) return advance(); + if (check(types)) return advance(); throw error(peek(), message); } @@ -162,11 +162,7 @@ class Parser List fields = new List(); while (!check(TokenType.RBrace, TokenType.Pipe)) { - var fieldName = name(); - if (fieldName == null) - { - throw error(peek(), "Expect identifier as field name."); - } + var fieldName = consume("Expect identifier as field name.", TokenType.Identifier, TokenType.QuotedIdentifier); var pat = match(TokenType.Equal) ? pattern() : null; fields.Add(new(fieldName, pat)); if (!match(TokenType.Comma)) @@ -175,7 +171,7 @@ class Parser } } var restPattern = match(TokenType.Pipe) ? simplePattern() : null; - consume(TokenType.RBrace, "Expect '}' at end of record pattern."); + consume("Expect '}' at end of record pattern.", TokenType.RBrace); return new(fields.ToArray(), restPattern); } @@ -185,17 +181,13 @@ class Parser { return null; } - Name? tag = name(); - if (tag == null) - { - throw error(peek(), "Expect identifier as tag name."); - } + var tag = consume("Expect identifier as tag name.", TokenType.Identifier, TokenType.QuotedIdentifier); if (!match(TokenType.LParen)) { return new(tag, null); } Pattern argument = pattern(); - consume(TokenType.RParen, "Expect ')' after variant argument."); + consume("Expect ')' after variant argument.", TokenType.RParen); return new(tag, argument); } @@ -206,13 +198,7 @@ class Parser return new SimplePattern(null); } - var identifier = name(); - if (identifier != null) - { - return new SimplePattern(identifier); - } - - return null; + return match(TokenType.Identifier, TokenType.QuotedIdentifier) ? new(previous()) : null; } private Pattern pattern() @@ -254,7 +240,7 @@ class Parser { bindings.Add(parseBinding()); } - consume(TokenType.In, "Expect 'in' after let-bindings."); + consume("Expect 'in' after let-bindings.", TokenType.In); Expr body = expression(); return new Let(bindings.ToArray(), body); @@ -273,11 +259,11 @@ class Parser break; } } - consume(TokenType.RParen, "Expect ')' at end of parameters."); - consume(TokenType.Equal, "Expect '=' after parameters."); + consume("Expect ')' at end of parameters.", TokenType.RParen); + consume("Expect '=' after parameters.", TokenType.Equal); return new FuncBinding(funcName, funcParams.ToArray(), expression()); default: - consume(TokenType.Equal, "Expect '=' after pattern."); + consume("Expect '=' after pattern.", TokenType.Equal); return new VarBinding(p, expression()); } } @@ -292,9 +278,9 @@ class Parser return when(); } Expr condition = expression(); - consume(TokenType.Then, "Expect 'then' after condition."); + consume("Expect 'then' after condition.", TokenType.Then); Expr thenCase = expression(); - consume(TokenType.Else, "Expect 'else' after 'then' case."); + consume("Expect 'else' after 'then' case.", TokenType.Else); Expr elseCase = expression(); return new If(condition, thenCase, elseCase); } @@ -306,7 +292,7 @@ class Parser return primary(); } Expr head = expression(); - consume(TokenType.Is, "Expect 'is' after expression."); + consume("Expect 'is' after expression.", TokenType.Is); List cases = new List(); cases.Add(parseCase()); @@ -319,44 +305,26 @@ class Parser VarBinding parseCase() { Pattern pat = pattern(); - consume(TokenType.DoubleArrow, "Expect '=>' after pattern."); + consume("Expect '=>' after pattern.", TokenType.DoubleArrow); Expr value = expression(); return new VarBinding(pat, value); } } - private Name? name() - { - if (match(TokenType.Identifier)) - { - return new(previous().lexeme, false); - } - if (match(TokenType.At)) - { - Token literal = consume(TokenType.String, "Expect string literal after '@'."); - return new((string)(literal.literal!), true); - } - return null; - } - private Expr primary() { Expr expr = operand(); if (match(TokenType.Period)) { - Name? ident = name(); - if (ident == null) - { - throw error(advance(), "Expect identifier after dot."); - } + var ident = consume("Expect identifier after dot.", TokenType.Identifier, TokenType.QuotedIdentifier); return new Selector(expr, ident); } if (match(TokenType.LBracket)) { var index = expression(); - consume(TokenType.RBracket, "Expect '[' after expression."); + consume("Expect '[' after expression.", TokenType.RBracket); return new Indexer(expr, index); } @@ -378,7 +346,7 @@ class Parser break; } } - consume(TokenType.RParen, "Expect ')' after arguments."); + consume("Expect ')' after arguments.", TokenType.RParen); return new Call(expr, args.ToArray()); } @@ -393,16 +361,15 @@ class Parser return new Literal(previous().literal!); } - var ident = name(); - if (ident != null) + if (match(TokenType.Identifier, TokenType.QuotedIdentifier)) { - return new Variable(ident); + return new Variable(previous()); } if (match(TokenType.LParen)) { Expr groupedExpr = expression(); - consume(TokenType.RParen, "Expect ')' after expression."); + consume("Expect ')' after expression.", TokenType.RParen); return new Grouping(groupedExpr); } @@ -463,18 +430,14 @@ class Parser { return null; } - Name? tag = name(); + var tag = consume("Expect identifier as tag name.", TokenType.QuotedIdentifier, TokenType.Identifier); Expr? argument = null; - if (tag == null) - { - throw error(peek(), "Expect identifier as tag name."); - } if (match(TokenType.LParen)) { if (!match(TokenType.RParen)) { argument = expression(); - consume(TokenType.RParen, "Expect ')' after variant argument."); + consume("Expect ')' after variant argument.", TokenType.RParen); } } return new Variant(tag, argument); @@ -497,7 +460,7 @@ class Parser baseRecord = new(baseExpr, updates); } - consume(TokenType.RBrace, "Expect '}' at end of record literal."); + consume("Expect '}' at end of record literal.", TokenType.RBrace); return new Record(extensions, baseRecord); Field[] parseFields(params TokenType[] endAt) @@ -506,11 +469,7 @@ class Parser while (!check(endAt)) { - var fieldName = name(); - if (fieldName == null) - { - throw error(peek(), "Expect identifier as field name."); - } + var fieldName = consume("Expect identifier as field name.", TokenType.Identifier, TokenType.QuotedIdentifier); var value = match(TokenType.Equal) ? expression() : null; fields.Add(new(fieldName, value)); if (!match(TokenType.Comma)) diff --git a/Pattern.g.cs b/Pattern.g.cs index 78e8b50..992a0db 100644 --- a/Pattern.g.cs +++ b/Pattern.g.cs @@ -16,21 +16,21 @@ public interface IPatternVisitor { TResult visitFieldPatternPattern(TContext context, FieldPattern pattern); TResult visitRecordPatternPattern(TContext context, RecordPattern pattern); } -public partial record SimplePattern(Name? Identifier) : Pattern() +public partial record SimplePattern(Token? Identifier) : Pattern() { public override TResult accept(TContext context, IPatternVisitor visitor) { return visitor.visitSimplePatternPattern(context, this); } } -public partial record VariantPattern(Name Tag, Pattern? Argument) : Pattern() +public partial record VariantPattern(Token Tag, Pattern? Argument) : Pattern() { public override TResult accept(TContext context, IPatternVisitor visitor) { return visitor.visitVariantPatternPattern(context, this); } } -public partial record FieldPattern(Name Name, Pattern? Pattern) : Pattern() +public partial record FieldPattern(Token Name, Pattern? Pattern) : Pattern() { public override TResult accept(TContext context, IPatternVisitor visitor) { diff --git a/Program.cs b/Program.cs index f2ff70b..90acdb4 100644 --- a/Program.cs +++ b/Program.cs @@ -41,8 +41,16 @@ class Program { var scanner = new Scanner(src); List tokens = scanner.scanTokens(); + Console.WriteLine("TOKENS\n======="); + foreach (var token in tokens) + { + Console.WriteLine(token); + } Parser parser = new Parser(tokens); Expr? expression = parser.parse(); + Console.WriteLine("\nAST\n======="); + Console.WriteLine(expression); + Console.WriteLine(); if (hadError) { @@ -132,7 +140,7 @@ public enum TokenType Recurse, Def, Blank, - Identifier, Number, String, + Identifier, QuotedIdentifier, Number, String, EOF } @@ -207,10 +215,14 @@ class Scanner addToken(type, null); } + private string lexeme + { + get => source.Substring(start.Offset, current.Offset - start.Offset); + } + private void addToken(TokenType type, Object? literal) { - String text = source.Substring(start.Offset, current.Offset - start.Offset); - tokens.Add(new Token(type, text, literal, start)); + tokens.Add(new Token(type, lexeme, literal, start)); } private bool match(char expected) @@ -237,8 +249,9 @@ class Scanner return source[current.Offset]; } - private void stringLiteral() + private string? _stringLiteral(string errorName) { + var valueStart = current.Offset; while (peek() != '"' && !isAtEnd()) { if (peek() == '\n') @@ -248,15 +261,32 @@ class Scanner if (isAtEnd()) { - Program.error(current, "Unterminated string."); - return; + Program.error(current, $"Unterminated {errorName}."); + return null; } // The closing ". advance(); - // Trim the surrounding quotes. - String value = source.Substring(start.Offset + 1, current.Offset - start.Offset - 2); + // Trim the closing quote. + return source.Substring(valueStart, current.Offset - valueStart - 1); + } + + private void quotedIdentifier() + { + if (!match('"')) + { + Program.error(current, "Expect \" after @."); + } + var value = _stringLiteral("quoted identifier"); + if (value == null) return; + addToken(TokenType.QuotedIdentifier, value); + } + + private void stringLiteral() + { + var value = _stringLiteral("string"); + if (value == null) return; addToken(TokenType.String, value); } @@ -300,7 +330,7 @@ class Scanner String text = source.Substring(start.Offset, current.Offset - start.Offset); TokenType type; var isKeyword = keywords.TryGetValue(text, out type); - addToken(isKeyword ? type : TokenType.Identifier); + addToken(isKeyword ? type : TokenType.Identifier, text); } private void scanToken() @@ -317,7 +347,6 @@ class Scanner case ':': addToken(TokenType.Colon); break; case '\'': addToken(TokenType.Tick); break; case '`': addToken(TokenType.Backtick); break; - case '@': addToken(TokenType.At); break; case ',': addToken(TokenType.Comma); break; case ';': addToken(TokenType.Semicolon); break; case '.': addToken(TokenType.Period); break; @@ -344,6 +373,7 @@ class Scanner case '>': addToken(match('=') ? TokenType.GreaterEqual : TokenType.Greater); break; + case '@': quotedIdentifier(); break; case '"': stringLiteral(); break; case '#': while (peek() != '\n' && !isAtEnd()) advance(); diff --git a/TODO.txt b/TODO.txt index a81dd49..b6ae52f 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,3 +1,3 @@ -Unify identifier types Include tokens in AST nodes Figure out multiple-binding let-expr semantics +Inject error handling into parser and scanner \ No newline at end of file diff --git a/ast_classes.fsx b/ast_classes.fsx index d6c7c19..f4eb5ce 100644 --- a/ast_classes.fsx +++ b/ast_classes.fsx @@ -23,18 +23,18 @@ let exprTypes = { Type = "Expr"; Name = "Then" } { Type = "Expr"; Name = "Else" } ] } { Name = "Variable" - Fields = [ { Type = "Name"; Name = "Value" } ] } + Fields = [ { Type = "Token"; Name = "Value" } ] } { Name = "List" Fields = [ { Type = "Expr[]"; Name = "Elements" } ] } { Name = "Variant" - Fields = [ { Type = "Name"; Name = "Tag" }; { Type = "Expr?"; Name = "Argument" } ] } + Fields = [ { Type = "Token"; Name = "Tag" }; { Type = "Expr?"; Name = "Argument" } ] } { Name = "Record" Fields = [ { Type = "Field[]" Name = "Extensions" } { Type = "BaseRecord?"; Name = "Base" } ] } { Name = "Selector" - Fields = [ { Type = "Expr"; Name = "Left" }; { Type = "Name"; Name = "FieldName" } ] } + Fields = [ { Type = "Expr"; Name = "Left" }; { Type = "Token"; Name = "FieldName" } ] } { Name = "Indexer" Fields = [ { Type = "Expr"; Name = "Left" }; { Type = "Expr"; Name = "Index" } ] } { Name = "Call" @@ -49,11 +49,11 @@ let exprTypes = let patternTypes = [ { Name = "SimplePattern" - Fields = [ { Type = "Name?"; Name = "Identifier" } ] } + Fields = [ { Type = "Token?"; Name = "Identifier" } ] } { Name = "VariantPattern" - Fields = [ { Type = "Name"; Name = "Tag" }; { Type = "Pattern?"; Name = "Argument" } ] } + Fields = [ { Type = "Token"; Name = "Tag" }; { Type = "Pattern?"; Name = "Argument" } ] } { Name = "FieldPattern" - Fields = [ { Type = "Name"; Name = "Name" }; { Type = "Pattern?"; Name = "Pattern" } ] } + Fields = [ { Type = "Token"; Name = "Name" }; { Type = "Pattern?"; Name = "Pattern" } ] } { Name = "RecordPattern" Fields = [ { Type = "FieldPattern[]"