From 6b201fbb69d54630e40d636dfc16849872175811 Mon Sep 17 00:00:00 2001
From: Brandon Dyck <brandon@dyck.us>
Date: Thu, 20 Aug 2026 22:54:31 -0600
Subject: [PATCH] Finish and test Token combinator

---
 TODO.txt            |  1 -
 bytes/bytes.go      | 17 +++++++++++++++--
 bytes/bytes_test.go | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 3 deletions(-)
 create mode 100644 bytes/bytes_test.go

diff --git a/TODO.txt b/TODO.txt
index 78ee180..038c6cc 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -3,4 +3,3 @@ Think about changing "consume" to "commit"
 Should MakeState be private now that there's Run?
 What's Megaparsec got that we ain't got?
 chainl
-whitespace handling
diff --git a/bytes/bytes.go b/bytes/bytes.go
index cf76b56..eee9a3f 100644
--- a/bytes/bytes.go
+++ b/bytes/bytes.go
@@ -1,11 +1,15 @@
+// SPDX-License-Identifier: Unlicense
+
 package bytes
 
 import (
 	"git.codemonkeysoftware.net/b/gigaparsec"
 )
 
-func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
-	mappedWS := gigaparsec.Map(whitespace, func(WSOut) struct{} { return struct{}{} })
+// Token wraps a parser to discard all whitespace after the parsed input.
+// whitespace is a parser that will consume non-empty whitespace.
+func Token[Out any](whitespace gigaparsec.Parser[byte, struct{}]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
+	mappedWS := gigaparsec.Map(whitespace, func(struct{}) struct{} { return struct{}{} })
 	var ignoreWS gigaparsec.Parser[byte, struct{}] = func(s gigaparsec.State[byte]) (gigaparsec.Result[byte, struct{}], error) {
 		result, err := mappedWS(s)
 		return result.Consume(false), err
@@ -14,3 +18,12 @@ func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gig
 		return gigaparsec.Seq(p, gigaparsec.Repeat(0, ignoreWS), func(val Out, _ []struct{}) Out { return val })
 	}
 }
+
+func isWhitespaceASCII(b byte) bool {
+	return b == ' ' || (b >= '\t' && b <= '\r')
+}
+
+// WhitespaceASCII consumes one ASCII whitespace character: space, tab, LF, CR, vertical tab, or form feed.
+func WhitespaceASCII() gigaparsec.Parser[byte, struct{}] {
+	return gigaparsec.Map(gigaparsec.Satisfy(isWhitespaceASCII), func(byte) struct{} { return struct{}{} })
+}
diff --git a/bytes/bytes_test.go b/bytes/bytes_test.go
new file mode 100644
index 0000000..b3c4fe2
--- /dev/null
+++ b/bytes/bytes_test.go
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Unlicense
+
+package bytes_test
+
+import (
+	"fmt"
+	"strings"
+
+	"git.codemonkeysoftware.net/b/gigaparsec"
+	"git.codemonkeysoftware.net/b/gigaparsec/bytes"
+)
+
+func ExampleToken() {
+	ptokens := gigaparsec.Choose(
+		bytes.MatchString("("),
+		bytes.MatchString(")"),
+		bytes.MatchString("+"),
+		bytes.MatchString("-"),
+		bytes.Regexp(`[a-zA-Z][a-zA-Z0-9_]*`).Label("identifier"),
+		bytes.Regexp(`[0-9]+`).Label("number"),
+	)
+
+	tokenize := gigaparsec.Repeat(0, bytes.Token[string](bytes.WhitespaceASCII())(ptokens))
+	tokens, _ := gigaparsec.Run(tokenize, strings.NewReader("(alpha + 42) -5  "))
+	for _, token := range tokens {
+		fmt.Println(token)
+	}
+
+	// Output:
+	// (
+	// alpha
+	// +
+	// 42
+	// )
+	// -
+	// 5
+}