Include richer position data in tokens

This commit is contained in:
Brandon Dyck 2023-07-16 19:47:42 -06:00
parent 9ea27888cb
commit de2fa22869
2 changed files with 60 additions and 37 deletions

View File

@ -38,7 +38,7 @@ public class Env
if (values.ContainsKey(name.Value))
{
// TODO use real location info
var tok = new Token(TokenType.Identifier, name.Value, null, 1);
var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
throw new RuntimeError(tok, $"Cannot redefine variable {name} in same scope.");
}
values[name.Value] = value;
@ -56,7 +56,7 @@ public class Env
return enclosing[name];
}
// TODO use real location info
var tok = new Token(TokenType.Identifier, name.Value, null, 1);
var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
throw new RuntimeError(tok, $"Undefined variable {name}.");
}
}
@ -409,7 +409,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
{
var cond = evaluate(env, expr.Condition);
// TODO Maybe I should token info in the AST.
var vb = checkBoolOperand(new Token(TokenType.If, "if", null, 1), cond);
var vb = checkBoolOperand(new Token(TokenType.If, "if", null, new(0, 1, 1)), cond);
if (vb == Variant.True)
{
return evaluate(env, expr.Then);
@ -423,7 +423,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
public object visitIndexerExpr(Env env, AST.Indexer expr)
{
// TODO use real token
var tok = new Token(TokenType.LBracket, "[", null, 1);
var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
var left = checkListOperand(tok, evaluate(env, expr.Left));
var index = checkNumberOperand(tok, evaluate(env, expr.Index));
try
@ -450,7 +450,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
catch (Exception e)
{
// TODO use real info
var tok = new Token(TokenType.Let, "let", null, 1);
var tok = new Token(TokenType.Let, "let", null, new(0, 1, 1));
throw new RuntimeError(tok, e.Message);
}
break;
@ -464,7 +464,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
public object visitListExpr(Env env, AST.List expr)
{
// TODO use real token
var tok = new Token(TokenType.LBracket, "[", null, 1);
var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
List l = List.Empty;
foreach (var itemExpr in expr.Elements)
{
@ -487,7 +487,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
public object visitRecordExpr(Env env, AST.Record expr)
{
// TODO use real token
Token tok = new Token(TokenType.LBrace, "{", null, 1);
Token tok = new Token(TokenType.LBrace, "{", null, new(0, 1, 1));
Record rec = Record.Empty;
if (expr.Base != null)
{
@ -543,7 +543,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
{
var left = evaluate(env, expr.Left);
// TODO Use real token.
var tok = new Token(TokenType.Period, ".", null, 1);
var tok = new Token(TokenType.Period, ".", null, new(0, 1, 1));
var r = checkRecordOperand(tok, left);
try
{
@ -597,7 +597,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
{
var head = evaluate(env, expr.Head);
// TODO use real info
var tok = new Token(TokenType.When, "when", null, 1);
var tok = new Token(TokenType.When, "when", null, new(0, 1, 1));
foreach (var c in expr.Cases)
{
try

View File

@ -67,32 +67,32 @@ class Program
}
}
public static void error(int line, String message)
public static void error(Position position, String message)
{
report(line, "", message);
report(position, "", message);
}
public static void error(Token token, String message)
{
if (token.type == TokenType.EOF)
{
report(token.line, " at end", message);
report(token.position, " at end", message);
}
else
{
report(token.line, " at '" + token.lexeme + "'", message);
report(token.position, " at '" + token.lexeme + "'", message);
}
}
public static void runtimeError(RuntimeError err)
{
Console.Error.WriteLine($"{err.Message}\n[line {err.Token.line}]");
Console.Error.WriteLine($"{err.Message}\n[{err.Token.position}]");
hadRuntimeError = true;
}
static void report(int line, String where, String message)
static void report(Position position, String where, String message)
{
Console.WriteLine($"[line {line}] Error{where}: {message}");
Console.WriteLine($"[{position}] Error{where}: {message}");
hadError = true;
}
}
@ -136,15 +136,37 @@ public enum TokenType
EOF
}
public record Token(TokenType type, String lexeme, Object? literal, int line);
public record struct Position(int Offset, int Line, int Column)
{
public override string ToString()
{
return $"{this.Line}:{this.Column}";
}
public static Position operator ++(Position p)
{
return p.Next();
}
public Position Next()
{
return this with { Offset = Offset + 1, Column = Column + 1 };
}
public Position NewLine()
{
return this with { Line = Line + 1, Column = 1 };
}
}
public record Token(TokenType type, String lexeme, Object? literal, Position position);
class Scanner
{
private readonly String source;
private readonly List<Token> tokens = new List<Token>();
private int start = 0;
private int current = 0;
private int line = 1;
private Position start = new(0, 1, 1);
private Position current = new(0, 1, 1);
private static readonly Dictionary<String, TokenType> keywords = new Dictionary<string, TokenType>()
{
@ -172,12 +194,12 @@ class Scanner
private bool isAtEnd()
{
return current >= source.Length;
return current.Offset >= source.Length;
}
private char advance()
{
return source[current++];
return source[(current++).Offset];
}
private void addToken(TokenType type)
@ -187,14 +209,14 @@ class Scanner
private void addToken(TokenType type, Object? literal)
{
String text = source.Substring(start, current - start);
tokens.Add(new Token(type, text, literal, line));
String text = source.Substring(start.Offset, current.Offset - start.Offset);
tokens.Add(new Token(type, text, literal, start));
}
private bool match(char expected)
{
if (isAtEnd()) return false;
if (source[current] != expected) return false;
if (source[current.Offset] != expected) return false;
current++;
return true;
@ -203,7 +225,7 @@ class Scanner
private bool match(Predicate<char> pred)
{
if (isAtEnd()) return false;
if (!pred(source[current])) return false;
if (!pred(source[current.Offset])) return false;
current++;
return true;
@ -212,20 +234,21 @@ class Scanner
private char? peek()
{
if (isAtEnd()) return null;
return source[current];
return source[current.Offset];
}
private void stringLiteral()
{
while (peek() != '"' && !isAtEnd())
{
if (peek() == '\n') line++;
if (peek() == '\n')
current = current.NewLine();
advance();
}
if (isAtEnd())
{
Program.error(line, "Unterminated string.");
Program.error(current, "Unterminated string.");
return;
}
@ -233,7 +256,7 @@ class Scanner
advance();
// Trim the surrounding quotes.
String value = source.Substring(start + 1, current - start - 2);
String value = source.Substring(start.Offset + 1, current.Offset - start.Offset - 2);
addToken(TokenType.String, value);
}
@ -244,8 +267,8 @@ class Scanner
private char? peekNext()
{
if (current + 1 >= source.Length) return null;
return source[current + 1];
if (current.Offset + 1 >= source.Length) return null;
return source[current.Offset + 1];
}
private void numberLiteral()
{
@ -257,7 +280,7 @@ class Scanner
match('.');
while (match(isDigitOrSeparator)) ;
}
double value = Double.Parse(source.Substring(start, current - start));
double value = Double.Parse(source.Substring(start.Offset, current.Offset - start.Offset));
addToken(TokenType.Number, value);
}
@ -274,7 +297,7 @@ class Scanner
private void identifier()
{
while (match(isIdentifierChar)) ;
String text = source.Substring(start, current - start);
String text = source.Substring(start.Offset, current.Offset - start.Offset);
TokenType type;
var isKeyword = keywords.TryGetValue(text, out type);
addToken(isKeyword ? type : TokenType.Identifier);
@ -330,7 +353,7 @@ class Scanner
case '\t':
break;
case '\n':
line++;
current = current.NewLine();
break;
default:
if (Char.IsAsciiDigit(c))
@ -343,7 +366,7 @@ class Scanner
}
else
{
Program.error(line, "Unexpected character.");
Program.error(current, "Unexpected character.");
}
break;
}
@ -356,7 +379,7 @@ class Scanner
scanToken();
}
tokens.Add(new Token(TokenType.EOF, "", null, line));
tokens.Add(new Token(TokenType.EOF, "", null, current));
return tokens;
}
}