Include richer position data in tokens
This commit is contained in:
parent
9ea27888cb
commit
de2fa22869
@ -38,7 +38,7 @@ public class Env
|
||||
if (values.ContainsKey(name.Value))
|
||||
{
|
||||
// TODO use real location info
|
||||
var tok = new Token(TokenType.Identifier, name.Value, null, 1);
|
||||
var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
|
||||
throw new RuntimeError(tok, $"Cannot redefine variable {name} in same scope.");
|
||||
}
|
||||
values[name.Value] = value;
|
||||
@ -56,7 +56,7 @@ public class Env
|
||||
return enclosing[name];
|
||||
}
|
||||
// TODO use real location info
|
||||
var tok = new Token(TokenType.Identifier, name.Value, null, 1);
|
||||
var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
|
||||
throw new RuntimeError(tok, $"Undefined variable {name}.");
|
||||
}
|
||||
}
|
||||
@ -409,7 +409,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
{
|
||||
var cond = evaluate(env, expr.Condition);
|
||||
// TODO Maybe I should token info in the AST.
|
||||
var vb = checkBoolOperand(new Token(TokenType.If, "if", null, 1), cond);
|
||||
var vb = checkBoolOperand(new Token(TokenType.If, "if", null, new(0, 1, 1)), cond);
|
||||
if (vb == Variant.True)
|
||||
{
|
||||
return evaluate(env, expr.Then);
|
||||
@ -423,7 +423,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
public object visitIndexerExpr(Env env, AST.Indexer expr)
|
||||
{
|
||||
// TODO use real token
|
||||
var tok = new Token(TokenType.LBracket, "[", null, 1);
|
||||
var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
|
||||
var left = checkListOperand(tok, evaluate(env, expr.Left));
|
||||
var index = checkNumberOperand(tok, evaluate(env, expr.Index));
|
||||
try
|
||||
@ -450,7 +450,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
catch (Exception e)
|
||||
{
|
||||
// TODO use real info
|
||||
var tok = new Token(TokenType.Let, "let", null, 1);
|
||||
var tok = new Token(TokenType.Let, "let", null, new(0, 1, 1));
|
||||
throw new RuntimeError(tok, e.Message);
|
||||
}
|
||||
break;
|
||||
@ -464,7 +464,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
public object visitListExpr(Env env, AST.List expr)
|
||||
{
|
||||
// TODO use real token
|
||||
var tok = new Token(TokenType.LBracket, "[", null, 1);
|
||||
var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
|
||||
List l = List.Empty;
|
||||
foreach (var itemExpr in expr.Elements)
|
||||
{
|
||||
@ -487,7 +487,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
public object visitRecordExpr(Env env, AST.Record expr)
|
||||
{
|
||||
// TODO use real token
|
||||
Token tok = new Token(TokenType.LBrace, "{", null, 1);
|
||||
Token tok = new Token(TokenType.LBrace, "{", null, new(0, 1, 1));
|
||||
Record rec = Record.Empty;
|
||||
if (expr.Base != null)
|
||||
{
|
||||
@ -543,7 +543,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
{
|
||||
var left = evaluate(env, expr.Left);
|
||||
// TODO Use real token.
|
||||
var tok = new Token(TokenType.Period, ".", null, 1);
|
||||
var tok = new Token(TokenType.Period, ".", null, new(0, 1, 1));
|
||||
var r = checkRecordOperand(tok, left);
|
||||
try
|
||||
{
|
||||
@ -597,7 +597,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
|
||||
{
|
||||
var head = evaluate(env, expr.Head);
|
||||
// TODO use real info
|
||||
var tok = new Token(TokenType.When, "when", null, 1);
|
||||
var tok = new Token(TokenType.When, "when", null, new(0, 1, 1));
|
||||
foreach (var c in expr.Cases)
|
||||
{
|
||||
try
|
||||
|
||||
79
Program.cs
79
Program.cs
@ -67,32 +67,32 @@ class Program
|
||||
}
|
||||
}
|
||||
|
||||
public static void error(int line, String message)
|
||||
public static void error(Position position, String message)
|
||||
{
|
||||
report(line, "", message);
|
||||
report(position, "", message);
|
||||
}
|
||||
|
||||
public static void error(Token token, String message)
|
||||
{
|
||||
if (token.type == TokenType.EOF)
|
||||
{
|
||||
report(token.line, " at end", message);
|
||||
report(token.position, " at end", message);
|
||||
}
|
||||
else
|
||||
{
|
||||
report(token.line, " at '" + token.lexeme + "'", message);
|
||||
report(token.position, " at '" + token.lexeme + "'", message);
|
||||
}
|
||||
}
|
||||
|
||||
public static void runtimeError(RuntimeError err)
|
||||
{
|
||||
Console.Error.WriteLine($"{err.Message}\n[line {err.Token.line}]");
|
||||
Console.Error.WriteLine($"{err.Message}\n[{err.Token.position}]");
|
||||
hadRuntimeError = true;
|
||||
}
|
||||
|
||||
static void report(int line, String where, String message)
|
||||
static void report(Position position, String where, String message)
|
||||
{
|
||||
Console.WriteLine($"[line {line}] Error{where}: {message}");
|
||||
Console.WriteLine($"[{position}] Error{where}: {message}");
|
||||
hadError = true;
|
||||
}
|
||||
}
|
||||
@ -136,15 +136,37 @@ public enum TokenType
|
||||
EOF
|
||||
}
|
||||
|
||||
public record Token(TokenType type, String lexeme, Object? literal, int line);
|
||||
public record struct Position(int Offset, int Line, int Column)
|
||||
{
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{this.Line}:{this.Column}";
|
||||
}
|
||||
|
||||
public static Position operator ++(Position p)
|
||||
{
|
||||
return p.Next();
|
||||
}
|
||||
|
||||
public Position Next()
|
||||
{
|
||||
return this with { Offset = Offset + 1, Column = Column + 1 };
|
||||
}
|
||||
|
||||
public Position NewLine()
|
||||
{
|
||||
return this with { Line = Line + 1, Column = 1 };
|
||||
}
|
||||
}
|
||||
|
||||
public record Token(TokenType type, String lexeme, Object? literal, Position position);
|
||||
|
||||
class Scanner
|
||||
{
|
||||
private readonly String source;
|
||||
private readonly List<Token> tokens = new List<Token>();
|
||||
private int start = 0;
|
||||
private int current = 0;
|
||||
private int line = 1;
|
||||
private Position start = new(0, 1, 1);
|
||||
private Position current = new(0, 1, 1);
|
||||
|
||||
private static readonly Dictionary<String, TokenType> keywords = new Dictionary<string, TokenType>()
|
||||
{
|
||||
@ -172,12 +194,12 @@ class Scanner
|
||||
|
||||
private bool isAtEnd()
|
||||
{
|
||||
return current >= source.Length;
|
||||
return current.Offset >= source.Length;
|
||||
}
|
||||
|
||||
private char advance()
|
||||
{
|
||||
return source[current++];
|
||||
return source[(current++).Offset];
|
||||
}
|
||||
|
||||
private void addToken(TokenType type)
|
||||
@ -187,14 +209,14 @@ class Scanner
|
||||
|
||||
private void addToken(TokenType type, Object? literal)
|
||||
{
|
||||
String text = source.Substring(start, current - start);
|
||||
tokens.Add(new Token(type, text, literal, line));
|
||||
String text = source.Substring(start.Offset, current.Offset - start.Offset);
|
||||
tokens.Add(new Token(type, text, literal, start));
|
||||
}
|
||||
|
||||
private bool match(char expected)
|
||||
{
|
||||
if (isAtEnd()) return false;
|
||||
if (source[current] != expected) return false;
|
||||
if (source[current.Offset] != expected) return false;
|
||||
|
||||
current++;
|
||||
return true;
|
||||
@ -203,7 +225,7 @@ class Scanner
|
||||
private bool match(Predicate<char> pred)
|
||||
{
|
||||
if (isAtEnd()) return false;
|
||||
if (!pred(source[current])) return false;
|
||||
if (!pred(source[current.Offset])) return false;
|
||||
|
||||
current++;
|
||||
return true;
|
||||
@ -212,20 +234,21 @@ class Scanner
|
||||
private char? peek()
|
||||
{
|
||||
if (isAtEnd()) return null;
|
||||
return source[current];
|
||||
return source[current.Offset];
|
||||
}
|
||||
|
||||
private void stringLiteral()
|
||||
{
|
||||
while (peek() != '"' && !isAtEnd())
|
||||
{
|
||||
if (peek() == '\n') line++;
|
||||
if (peek() == '\n')
|
||||
current = current.NewLine();
|
||||
advance();
|
||||
}
|
||||
|
||||
if (isAtEnd())
|
||||
{
|
||||
Program.error(line, "Unterminated string.");
|
||||
Program.error(current, "Unterminated string.");
|
||||
return;
|
||||
}
|
||||
|
||||
@ -233,7 +256,7 @@ class Scanner
|
||||
advance();
|
||||
|
||||
// Trim the surrounding quotes.
|
||||
String value = source.Substring(start + 1, current - start - 2);
|
||||
String value = source.Substring(start.Offset + 1, current.Offset - start.Offset - 2);
|
||||
addToken(TokenType.String, value);
|
||||
}
|
||||
|
||||
@ -244,8 +267,8 @@ class Scanner
|
||||
|
||||
private char? peekNext()
|
||||
{
|
||||
if (current + 1 >= source.Length) return null;
|
||||
return source[current + 1];
|
||||
if (current.Offset + 1 >= source.Length) return null;
|
||||
return source[current.Offset + 1];
|
||||
}
|
||||
private void numberLiteral()
|
||||
{
|
||||
@ -257,7 +280,7 @@ class Scanner
|
||||
match('.');
|
||||
while (match(isDigitOrSeparator)) ;
|
||||
}
|
||||
double value = Double.Parse(source.Substring(start, current - start));
|
||||
double value = Double.Parse(source.Substring(start.Offset, current.Offset - start.Offset));
|
||||
addToken(TokenType.Number, value);
|
||||
}
|
||||
|
||||
@ -274,7 +297,7 @@ class Scanner
|
||||
private void identifier()
|
||||
{
|
||||
while (match(isIdentifierChar)) ;
|
||||
String text = source.Substring(start, current - start);
|
||||
String text = source.Substring(start.Offset, current.Offset - start.Offset);
|
||||
TokenType type;
|
||||
var isKeyword = keywords.TryGetValue(text, out type);
|
||||
addToken(isKeyword ? type : TokenType.Identifier);
|
||||
@ -330,7 +353,7 @@ class Scanner
|
||||
case '\t':
|
||||
break;
|
||||
case '\n':
|
||||
line++;
|
||||
current = current.NewLine();
|
||||
break;
|
||||
default:
|
||||
if (Char.IsAsciiDigit(c))
|
||||
@ -343,7 +366,7 @@ class Scanner
|
||||
}
|
||||
else
|
||||
{
|
||||
Program.error(line, "Unexpected character.");
|
||||
Program.error(current, "Unexpected character.");
|
||||
}
|
||||
break;
|
||||
}
|
||||
@ -356,7 +379,7 @@ class Scanner
|
||||
scanToken();
|
||||
}
|
||||
|
||||
tokens.Add(new Token(TokenType.EOF, "", null, line));
|
||||
tokens.Add(new Token(TokenType.EOF, "", null, current));
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user