Finish and test Token combinator

bytes/bytes.go

@@ -1,11 +1,15 @@
+// SPDX-License-Identifier: Unlicense
+
 package bytes
 
 import (
 	"git.codemonkeysoftware.net/b/gigaparsec"
 )
 
-func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
-	mappedWS := gigaparsec.Map(whitespace, func(WSOut) struct{} { return struct{}{} })
+// Token wraps a parser to discard all whitespace after the parsed input.
+// whitespace is a parser that will consume non-empty whitespace.
+func Token[Out any](whitespace gigaparsec.Parser[byte, struct{}]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
+	mappedWS := gigaparsec.Map(whitespace, func(struct{}) struct{} { return struct{}{} })
 	var ignoreWS gigaparsec.Parser[byte, struct{}] = func(s gigaparsec.State[byte]) (gigaparsec.Result[byte, struct{}], error) {
 		result, err := mappedWS(s)
 		return result.Consume(false), err
@@ -14,3 +18,12 @@ func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gig
 		return gigaparsec.Seq(p, gigaparsec.Repeat(0, ignoreWS), func(val Out, _ []struct{}) Out { return val })
 	}
 }
+
+func isWhitespaceASCII(b byte) bool {
+	return b == ' ' || (b >= '\t' && b <= '\r')
+}
+
+// WhitespaceASCII consumes one ASCII whitespace character: space, tab, LF, CR, vertical tab, or form feed.
+func WhitespaceASCII() gigaparsec.Parser[byte, struct{}] {
+	return gigaparsec.Map(gigaparsec.Satisfy(isWhitespaceASCII), func(byte) struct{} { return struct{}{} })
+}