Include richer position data in tokens

Interpreter.cs

@@ -38,7 +38,7 @@ public class Env
 			if (values.ContainsKey(name.Value))
 			{
 				// TODO use real location info
-				var tok = new Token(TokenType.Identifier, name.Value, null, 1);
+				var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
 				throw new RuntimeError(tok, $"Cannot redefine variable {name} in same scope.");
 			}
 			values[name.Value] = value;
@@ -56,7 +56,7 @@ public class Env
 					return enclosing[name];
 				}
 				// TODO use real location info
-				var tok = new Token(TokenType.Identifier, name.Value, null, 1);
+				var tok = new Token(TokenType.Identifier, name.Value, null, new(0, 1, 1));
 				throw new RuntimeError(tok, $"Undefined variable {name}.");
 			}
 		}
@@ -409,7 +409,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 	{
 		var cond = evaluate(env, expr.Condition);
 		// TODO Maybe I should token info in the AST.
-		var vb = checkBoolOperand(new Token(TokenType.If, "if", null, 1), cond);
+		var vb = checkBoolOperand(new Token(TokenType.If, "if", null, new(0, 1, 1)), cond);
 		if (vb == Variant.True)
 		{
 			return evaluate(env, expr.Then);
@@ -423,7 +423,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 	public object visitIndexerExpr(Env env, AST.Indexer expr)
 	{
 		// TODO use real token
-		var tok = new Token(TokenType.LBracket, "[", null, 1);
+		var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
 		var left = checkListOperand(tok, evaluate(env, expr.Left));
 		var index = checkNumberOperand(tok, evaluate(env, expr.Index));
 		try
@@ -450,7 +450,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 					catch (Exception e)
 					{
 						// TODO use real info
-						var tok = new Token(TokenType.Let, "let", null, 1);
+						var tok = new Token(TokenType.Let, "let", null, new(0, 1, 1));
 						throw new RuntimeError(tok, e.Message);
 					}
 					break;
@@ -464,7 +464,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 	public object visitListExpr(Env env, AST.List expr)
 	{
 		// TODO use real token
-		var tok = new Token(TokenType.LBracket, "[", null, 1);
+		var tok = new Token(TokenType.LBracket, "[", null, new(0, 1, 1));
 		List l = List.Empty;
 		foreach (var itemExpr in expr.Elements)
 		{
@@ -487,7 +487,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 	public object visitRecordExpr(Env env, AST.Record expr)
 	{
 		// TODO use real token
-		Token tok = new Token(TokenType.LBrace, "{", null, 1);
+		Token tok = new Token(TokenType.LBrace, "{", null, new(0, 1, 1));
 		Record rec = Record.Empty;
 		if (expr.Base != null)
 		{
@@ -543,7 +543,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 	{
 		var left = evaluate(env, expr.Left);
 		// TODO Use real token.
-		var tok = new Token(TokenType.Period, ".", null, 1);
+		var tok = new Token(TokenType.Period, ".", null, new(0, 1, 1));
 		var r = checkRecordOperand(tok, left);
 		try
 		{
@@ -597,7 +597,7 @@ public class Interpreter : AST.IExprVisitor<Env, object>
 	{
 		var head = evaluate(env, expr.Head);
 		// TODO use real info
-		var tok = new Token(TokenType.When, "when", null, 1);
+		var tok = new Token(TokenType.When, "when", null, new(0, 1, 1));
 		foreach (var c in expr.Cases)
 		{
 			try

Program.cs

@@ -67,32 +67,32 @@ class Program
 		}
 	}
 
-	public static void error(int line, String message)
+	public static void error(Position position, String message)
 	{
-		report(line, "", message);
+		report(position, "", message);
 	}
 
 	public static void error(Token token, String message)
 	{
 		if (token.type == TokenType.EOF)
 		{
-			report(token.line, " at end", message);
+			report(token.position, " at end", message);
 		}
 		else
 		{
-			report(token.line, " at '" + token.lexeme + "'", message);
+			report(token.position, " at '" + token.lexeme + "'", message);
 		}
 	}
 
 	public static void runtimeError(RuntimeError err)
 	{
-		Console.Error.WriteLine($"{err.Message}\n[line {err.Token.line}]");
+		Console.Error.WriteLine($"{err.Message}\n[{err.Token.position}]");
 		hadRuntimeError = true;
 	}
 
-	static void report(int line, String where, String message)
+	static void report(Position position, String where, String message)
 	{
-		Console.WriteLine($"[line {line}] Error{where}: {message}");
+		Console.WriteLine($"[{position}] Error{where}: {message}");
 		hadError = true;
 	}
 }
@@ -136,15 +136,37 @@ public enum TokenType
 	EOF
 }
 
-public record Token(TokenType type, String lexeme, Object? literal, int line);
+public record struct Position(int Offset, int Line, int Column)
+{
+	public override string ToString()
+	{
+		return $"{this.Line}:{this.Column}";
+	}
+
+	public static Position operator ++(Position p)
+	{
+		return p.Next();
+	}
+
+	public Position Next()
+	{
+		return this with { Offset = Offset + 1, Column = Column + 1 };
+	}
+
+	public Position NewLine()
+	{
+		return this with { Line = Line + 1, Column = 1 };
+	}
+}
+
+public record Token(TokenType type, String lexeme, Object? literal, Position position);
 
 class Scanner
 {
 	private readonly String source;
 	private readonly List<Token> tokens = new List<Token>();
-	private int start = 0;
+	private Position start = new(0, 1, 1);
-	private int current = 0;
+	private Position current = new(0, 1, 1);
-	private int line = 1;
 
 	private static readonly Dictionary<String, TokenType> keywords = new Dictionary<string, TokenType>()
 	{
@@ -172,12 +194,12 @@ class Scanner
 
 	private bool isAtEnd()
 	{
-		return current >= source.Length;
+		return current.Offset >= source.Length;
 	}
 
 	private char advance()
 	{
-		return source[current++];
+		return source[(current++).Offset];
 	}
 
 	private void addToken(TokenType type)
@@ -187,14 +209,14 @@ class Scanner
 
 	private void addToken(TokenType type, Object? literal)
 	{
-		String text = source.Substring(start, current - start);
+		String text = source.Substring(start.Offset, current.Offset - start.Offset);
-		tokens.Add(new Token(type, text, literal, line));
+		tokens.Add(new Token(type, text, literal, start));
 	}
 
 	private bool match(char expected)
 	{
 		if (isAtEnd()) return false;
-		if (source[current] != expected) return false;
+		if (source[current.Offset] != expected) return false;
 
 		current++;
 		return true;
@@ -203,7 +225,7 @@ class Scanner
 	private bool match(Predicate<char> pred)
 	{
 		if (isAtEnd()) return false;
-		if (!pred(source[current])) return false;
+		if (!pred(source[current.Offset])) return false;
 
 		current++;
 		return true;
@@ -212,20 +234,21 @@ class Scanner
 	private char? peek()
 	{
 		if (isAtEnd()) return null;
-		return source[current];
+		return source[current.Offset];
 	}
 
 	private void stringLiteral()
 	{
 		while (peek() != '"' && !isAtEnd())
 		{
-			if (peek() == '\n') line++;
+			if (peek() == '\n')
+				current = current.NewLine();
 			advance();
 		}
 
 		if (isAtEnd())
 		{
-			Program.error(line, "Unterminated string.");
+			Program.error(current, "Unterminated string.");
 			return;
 		}
 
@@ -233,7 +256,7 @@ class Scanner
 		advance();
 
 		// Trim the surrounding quotes.
-		String value = source.Substring(start + 1, current - start - 2);
+		String value = source.Substring(start.Offset + 1, current.Offset - start.Offset - 2);
 		addToken(TokenType.String, value);
 	}
 
@@ -244,8 +267,8 @@ class Scanner
 
 	private char? peekNext()
 	{
-		if (current + 1 >= source.Length) return null;
+		if (current.Offset + 1 >= source.Length) return null;
-		return source[current + 1];
+		return source[current.Offset + 1];
 	}
 	private void numberLiteral()
 	{
@@ -257,7 +280,7 @@ class Scanner
 			match('.');
 			while (match(isDigitOrSeparator)) ;
 		}
-		double value = Double.Parse(source.Substring(start, current - start));
+		double value = Double.Parse(source.Substring(start.Offset, current.Offset - start.Offset));
 		addToken(TokenType.Number, value);
 	}
 
@@ -274,7 +297,7 @@ class Scanner
 	private void identifier()
 	{
 		while (match(isIdentifierChar)) ;
-		String text = source.Substring(start, current - start);
+		String text = source.Substring(start.Offset, current.Offset - start.Offset);
 		TokenType type;
 		var isKeyword = keywords.TryGetValue(text, out type);
 		addToken(isKeyword ? type : TokenType.Identifier);
@@ -330,7 +353,7 @@ class Scanner
 			case '\t':
 				break;
 			case '\n':
-				line++;
+				current = current.NewLine();
 				break;
 			default:
 				if (Char.IsAsciiDigit(c))
@@ -343,7 +366,7 @@ class Scanner
 				}
 				else
 				{
-					Program.error(line, "Unexpected character.");
+					Program.error(current, "Unexpected character.");
 				}
 				break;
 		}
@@ -356,7 +379,7 @@ class Scanner
 			scanToken();
 		}
 
-		tokens.Add(new Token(TokenType.EOF, "", null, line));
+		tokens.Add(new Token(TokenType.EOF, "", null, current));
 		return tokens;
 	}
 }