Include richer position data in tokens

This commit is contained in:
Brandon Dyck 2023-07-16 19:47:42 -06:00
parent 9ea27888cb
commit de2fa22869
2 changed files with 60 additions and 37 deletions

View File

@ -38,7 +38,7 @@ public class Env
if (values.ContainsKey(name.Value)) if (values.ContainsKey(name.Value))
{ {
// TODO use real location info // TODO use real location info
var tok = new Token(TokenType.Identifier, name.Value, null, 1); var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
throw new RuntimeError(tok, $"Cannot redefine variable {name} in same scope."); throw new RuntimeError(tok, $"Cannot redefine variable {name} in same scope.");
} }
values[name.Value] = value; values[name.Value] = value;
@ -56,7 +56,7 @@ public class Env
return enclosing[name]; return enclosing[name];
} }
// TODO use real location info // TODO use real location info
var tok = new Token(TokenType.Identifier, name.Value, null, 1); var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
throw new RuntimeError(tok, $"Undefined variable {name}."); throw new RuntimeError(tok, $"Undefined variable {name}.");
} }
} }
@ -409,7 +409,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
{ {
var cond = evaluate(env, expr.Condition); var cond = evaluate(env, expr.Condition);
// TODO Maybe I should token info in the AST. // TODO Maybe I should token info in the AST.
var vb = checkBoolOperand(new Token(TokenType.If, "if", null, 1), cond); var vb = checkBoolOperand(new Token(TokenType.If, "if", null, new(0, 1, 1)), cond);
if (vb == Variant.True) if (vb == Variant.True)
{ {
return evaluate(env, expr.Then); return evaluate(env, expr.Then);
@ -423,7 +423,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
public object visitIndexerExpr(Env env, AST.Indexer expr) public object visitIndexerExpr(Env env, AST.Indexer expr)
{ {
// TODO use real token // TODO use real token
var tok = new Token(TokenType.LBracket, "[", null, 1); var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
var left = checkListOperand(tok, evaluate(env, expr.Left)); var left = checkListOperand(tok, evaluate(env, expr.Left));
var index = checkNumberOperand(tok, evaluate(env, expr.Index)); var index = checkNumberOperand(tok, evaluate(env, expr.Index));
try try
@ -450,7 +450,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
catch (Exception e) catch (Exception e)
{ {
// TODO use real info // TODO use real info
var tok = new Token(TokenType.Let, "let", null, 1); var tok = new Token(TokenType.Let, "let", null, new(0, 1, 1));
throw new RuntimeError(tok, e.Message); throw new RuntimeError(tok, e.Message);
} }
break; break;
@ -464,7 +464,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
public object visitListExpr(Env env, AST.List expr) public object visitListExpr(Env env, AST.List expr)
{ {
// TODO use real token // TODO use real token
var tok = new Token(TokenType.LBracket, "[", null, 1); var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
List l = List.Empty; List l = List.Empty;
foreach (var itemExpr in expr.Elements) foreach (var itemExpr in expr.Elements)
{ {
@ -487,7 +487,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
public object visitRecordExpr(Env env, AST.Record expr) public object visitRecordExpr(Env env, AST.Record expr)
{ {
// TODO use real token // TODO use real token
Token tok = new Token(TokenType.LBrace, "{", null, 1); Token tok = new Token(TokenType.LBrace, "{", null, new(0, 1, 1));
Record rec = Record.Empty; Record rec = Record.Empty;
if (expr.Base != null) if (expr.Base != null)
{ {
@ -543,7 +543,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
{ {
var left = evaluate(env, expr.Left); var left = evaluate(env, expr.Left);
// TODO Use real token. // TODO Use real token.
var tok = new Token(TokenType.Period, ".", null, 1); var tok = new Token(TokenType.Period, ".", null, new(0, 1, 1));
var r = checkRecordOperand(tok, left); var r = checkRecordOperand(tok, left);
try try
{ {
@ -597,7 +597,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
{ {
var head = evaluate(env, expr.Head); var head = evaluate(env, expr.Head);
// TODO use real info // TODO use real info
var tok = new Token(TokenType.When, "when", null, 1); var tok = new Token(TokenType.When, "when", null, new(0, 1, 1));
foreach (var c in expr.Cases) foreach (var c in expr.Cases)
{ {
try try

View File

@ -67,32 +67,32 @@ class Program
} }
} }
public static void error(int line, String message) public static void error(Position position, String message)
{ {
report(line, "", message); report(position, "", message);
} }
public static void error(Token token, String message) public static void error(Token token, String message)
{ {
if (token.type == TokenType.EOF) if (token.type == TokenType.EOF)
{ {
report(token.line, " at end", message); report(token.position, " at end", message);
} }
else else
{ {
report(token.line, " at '" + token.lexeme + "'", message); report(token.position, " at '" + token.lexeme + "'", message);
} }
} }
public static void runtimeError(RuntimeError err) public static void runtimeError(RuntimeError err)
{ {
Console.Error.WriteLine($"{err.Message}\n[line {err.Token.line}]"); Console.Error.WriteLine($"{err.Message}\n[{err.Token.position}]");
hadRuntimeError = true; hadRuntimeError = true;
} }
static void report(int line, String where, String message) static void report(Position position, String where, String message)
{ {
Console.WriteLine($"[line {line}] Error{where}: {message}"); Console.WriteLine($"[{position}] Error{where}: {message}");
hadError = true; hadError = true;
} }
} }
@ -136,15 +136,37 @@ public enum TokenType
EOF EOF
} }
public record Token(TokenType type, String lexeme, Object? literal, int line); public record struct Position(int Offset, int Line, int Column)
{
public override string ToString()
{
return $"{this.Line}:{this.Column}";
}
public static Position operator ++(Position p)
{
return p.Next();
}
public Position Next()
{
return this with { Offset = Offset + 1, Column = Column + 1 };
}
public Position NewLine()
{
return this with { Line = Line + 1, Column = 1 };
}
}
public record Token(TokenType type, String lexeme, Object? literal, Position position);
class Scanner class Scanner
{ {
private readonly String source; private readonly String source;
private readonly List<Token> tokens = new List<Token>(); private readonly List<Token> tokens = new List<Token>();
private int start = 0; private Position start = new(0, 1, 1);
private int current = 0; private Position current = new(0, 1, 1);
private int line = 1;
private static readonly Dictionary<String, TokenType> keywords = new Dictionary<string, TokenType>() private static readonly Dictionary<String, TokenType> keywords = new Dictionary<string, TokenType>()
{ {
@ -172,12 +194,12 @@ class Scanner
private bool isAtEnd() private bool isAtEnd()
{ {
return current >= source.Length; return current.Offset >= source.Length;
} }
private char advance() private char advance()
{ {
return source[current++]; return source[(current++).Offset];
} }
private void addToken(TokenType type) private void addToken(TokenType type)
@ -187,14 +209,14 @@ class Scanner
private void addToken(TokenType type, Object? literal) private void addToken(TokenType type, Object? literal)
{ {
String text = source.Substring(start, current - start); String text = source.Substring(start.Offset, current.Offset - start.Offset);
tokens.Add(new Token(type, text, literal, line)); tokens.Add(new Token(type, text, literal, start));
} }
private bool match(char expected) private bool match(char expected)
{ {
if (isAtEnd()) return false; if (isAtEnd()) return false;
if (source[current] != expected) return false; if (source[current.Offset] != expected) return false;
current++; current++;
return true; return true;
@ -203,7 +225,7 @@ class Scanner
private bool match(Predicate<char> pred) private bool match(Predicate<char> pred)
{ {
if (isAtEnd()) return false; if (isAtEnd()) return false;
if (!pred(source[current])) return false; if (!pred(source[current.Offset])) return false;
current++; current++;
return true; return true;
@ -212,20 +234,21 @@ class Scanner
private char? peek() private char? peek()
{ {
if (isAtEnd()) return null; if (isAtEnd()) return null;
return source[current]; return source[current.Offset];
} }
private void stringLiteral() private void stringLiteral()
{ {
while (peek() != '"' && !isAtEnd()) while (peek() != '"' && !isAtEnd())
{ {
if (peek() == '\n') line++; if (peek() == '\n')
current = current.NewLine();
advance(); advance();
} }
if (isAtEnd()) if (isAtEnd())
{ {
Program.error(line, "Unterminated string."); Program.error(current, "Unterminated string.");
return; return;
} }
@ -233,7 +256,7 @@ class Scanner
advance(); advance();
// Trim the surrounding quotes. // Trim the surrounding quotes.
String value = source.Substring(start + 1, current - start - 2); String value = source.Substring(start.Offset + 1, current.Offset - start.Offset - 2);
addToken(TokenType.String, value); addToken(TokenType.String, value);
} }
@ -244,8 +267,8 @@ class Scanner
private char? peekNext() private char? peekNext()
{ {
if (current + 1 >= source.Length) return null; if (current.Offset + 1 >= source.Length) return null;
return source[current + 1]; return source[current.Offset + 1];
} }
private void numberLiteral() private void numberLiteral()
{ {
@ -257,7 +280,7 @@ class Scanner
match('.'); match('.');
while (match(isDigitOrSeparator)) ; while (match(isDigitOrSeparator)) ;
} }
double value = Double.Parse(source.Substring(start, current - start)); double value = Double.Parse(source.Substring(start.Offset, current.Offset - start.Offset));
addToken(TokenType.Number, value); addToken(TokenType.Number, value);
} }
@ -274,7 +297,7 @@ class Scanner
private void identifier() private void identifier()
{ {
while (match(isIdentifierChar)) ; while (match(isIdentifierChar)) ;
String text = source.Substring(start, current - start); String text = source.Substring(start.Offset, current.Offset - start.Offset);
TokenType type; TokenType type;
var isKeyword = keywords.TryGetValue(text, out type); var isKeyword = keywords.TryGetValue(text, out type);
addToken(isKeyword ? type : TokenType.Identifier); addToken(isKeyword ? type : TokenType.Identifier);
@ -330,7 +353,7 @@ class Scanner
case '\t': case '\t':
break; break;
case '\n': case '\n':
line++; current = current.NewLine();
break; break;
default: default:
if (Char.IsAsciiDigit(c)) if (Char.IsAsciiDigit(c))
@ -343,7 +366,7 @@ class Scanner
} }
else else
{ {
Program.error(line, "Unexpected character."); Program.error(current, "Unexpected character.");
} }
break; break;
} }
@ -356,7 +379,7 @@ class Scanner
scanToken(); scanToken();
} }
tokens.Add(new Token(TokenType.EOF, "", null, line)); tokens.Add(new Token(TokenType.EOF, "", null, current));
return tokens; return tokens;
} }
} }