Finish and test Token combinator

TODO.txt

@@ -3,4 +3,3 @@ Think about changing "consume" to "commit"
 Should MakeState be private now that there's Run?
 What's Megaparsec got that we ain't got?
 chainl
-whitespace handling

bytes/bytes.go

@@ -1,11 +1,15 @@
+// SPDX-License-Identifier: Unlicense
+
 package bytes
 
 import (
 	"git.codemonkeysoftware.net/b/gigaparsec"
 )
 
-func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
+// Token wraps a parser to discard all whitespace after the parsed input.
-	mappedWS := gigaparsec.Map(whitespace, func(WSOut) struct{} { return struct{}{} })
+// whitespace is a parser that will consume non-empty whitespace.
+func Token[Out any](whitespace gigaparsec.Parser[byte, struct{}]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
+	mappedWS := gigaparsec.Map(whitespace, func(struct{}) struct{} { return struct{}{} })
 	var ignoreWS gigaparsec.Parser[byte, struct{}] = func(s gigaparsec.State[byte]) (gigaparsec.Result[byte, struct{}], error) {
 		result, err := mappedWS(s)
 		return result.Consume(false), err
@@ -14,3 +18,12 @@ func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gig
 		return gigaparsec.Seq(p, gigaparsec.Repeat(0, ignoreWS), func(val Out, _ []struct{}) Out { return val })
 	}
 }
+
+func isWhitespaceASCII(b byte) bool {
+	return b == ' ' || (b >= '\t' && b <= '\r')
+}
+
+// WhitespaceASCII consumes one ASCII whitespace character: space, tab, LF, CR, vertical tab, or form feed.
+func WhitespaceASCII() gigaparsec.Parser[byte, struct{}] {
+	return gigaparsec.Map(gigaparsec.Satisfy(isWhitespaceASCII), func(byte) struct{} { return struct{}{} })
+}

bytes/bytes_test.go

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Unlicense
+
+package bytes_test
+
+import (
+	"fmt"
+	"strings"
+
+	"git.codemonkeysoftware.net/b/gigaparsec"
+	"git.codemonkeysoftware.net/b/gigaparsec/bytes"
+)
+
+func ExampleToken() {
+	ptokens := gigaparsec.Choose(
+		bytes.MatchString("("),
+		bytes.MatchString(")"),
+		bytes.MatchString("+"),
+		bytes.MatchString("-"),
+		bytes.Regexp(`[a-zA-Z][a-zA-Z0-9_]*`).Label("identifier"),
+		bytes.Regexp(`[0-9]+`).Label("number"),
+	)
+
+	tokenize := gigaparsec.Repeat(0, bytes.Token[string](bytes.WhitespaceASCII())(ptokens))
+	tokens, _ := gigaparsec.Run(tokenize, strings.NewReader("(alpha + 42) -5  "))
+	for _, token := range tokens {
+		fmt.Println(token)
+	}
+
+	// Output:
+	// (
+	// alpha
+	// +
+	// 42
+	// )
+	// -
+	// 5
+}