Move Map and Run onto Parser, and document stuff

TODO.txt

@@ -3,4 +3,3 @@ Think about changing "consume" to "commit"
 Should MakeState be private now that there's Run?
 What's Megaparsec got that we ain't got?
 chainl
-whitespace handling

bytes/bytes.go

@@ -1,11 +1,15 @@
+// SPDX-License-Identifier: Unlicense
+
 package bytes
 
 import (
 	"git.codemonkeysoftware.net/b/gigaparsec"
 )
 
-func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
-	mappedWS := gigaparsec.Map(whitespace, func(WSOut) struct{} { return struct{}{} })
+// Token wraps a parser to discard all whitespace after the parsed input.
+// whitespace is a parser that will consume non-empty whitespace.
+func Token[Out any](whitespace gigaparsec.Parser[byte, struct{}]) func(p gigaparsec.Parser[byte, Out]) gigaparsec.Parser[byte, Out] {
+	mappedWS := whitespace.Map(func(struct{}) struct{} { return struct{}{} })
 	var ignoreWS gigaparsec.Parser[byte, struct{}] = func(s gigaparsec.State[byte]) (gigaparsec.Result[byte, struct{}], error) {
 		result, err := mappedWS(s)
 		return result.Consume(false), err
@@ -14,3 +18,12 @@ func Token[Out, WSOut any](whitespace gigaparsec.Parser[byte, WSOut]) func(p gig
 		return gigaparsec.Seq(p, gigaparsec.Repeat(0, ignoreWS), func(val Out, _ []struct{}) Out { return val })
 	}
 }
+
+func isWhitespaceASCII(b byte) bool {
+	return b == ' ' || (b >= '\t' && b <= '\r')
+}
+
+// WhitespaceASCII consumes one ASCII whitespace character: space, tab, LF, CR, vertical tab, or form feed.
+func WhitespaceASCII() gigaparsec.Parser[byte, struct{}] {
+	return gigaparsec.Satisfy(isWhitespaceASCII).Map(func(byte) struct{} { return struct{}{} })
+}

bytes/bytes_test.go

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Unlicense
+
+package bytes_test
+
+import (
+	"fmt"
+	"strings"
+
+	"git.codemonkeysoftware.net/b/gigaparsec"
+	"git.codemonkeysoftware.net/b/gigaparsec/bytes"
+)
+
+func ExampleToken() {
+	ptokens := gigaparsec.Choose(
+		bytes.MatchString("("),
+		bytes.MatchString(")"),
+		bytes.MatchString("+"),
+		bytes.MatchString("-"),
+		bytes.Regexp(`[a-zA-Z][a-zA-Z0-9_]*`).Label("identifier"),
+		bytes.Regexp(`[0-9]+`).Label("number"),
+	)
+
+	tokenize := gigaparsec.Repeat(0, bytes.Token[string](bytes.WhitespaceASCII())(ptokens))
+	tokens, _ := tokenize.Run(strings.NewReader("(alpha + 42) -5  "))
+	for _, token := range tokens {
+		fmt.Println(token)
+	}
+
+	// Output:
+	// (
+	// alpha
+	// +
+	// 42
+	// )
+	// -
+	// 5
+}

gigaparsec.go

@@ -194,6 +194,7 @@ func (p Parser[In, Out]) Label(label string) Parser[In, Out] {
 	}
 }
 
+// Where creates a parser that fails if pred does not hold.
 func (p Parser[In, Out]) Where(pred func(Out) bool) Parser[In, Out] {
 	return func(s State[In]) (Result[In, Out], error) {
 		result, err := p(s)
@@ -205,13 +206,15 @@ func (p Parser[In, Out]) Where(pred func(Out) bool) Parser[In, Out] {
 	}
 }
 
-type ParseError Message
-
-func (pe ParseError) Error() string {
-	return Message(pe).String()
+// Map creates a parser that converts the output of p from Out1 to Out2.
+func (p Parser[In, Out]) Map[Out2 any](f func(Out) Out2) Parser[In, Out2] {
+	return Bind(p, func(out Out) Parser[In, Out2] {
+		return Return[In](f(out))
+	})
 }
 
-func Run[In, Out any](p Parser[In, Out], r ReaderAt[In]) (out Out, err error) {
+// Run executes a parser on r and returns the result.
+func (p Parser[In, Out]) Run(r ReaderAt[In]) (out Out, err error) {
 	start := MakeState(r)
 	result, err := p(start)
 	if err != nil {
@@ -226,6 +229,12 @@ func Run[In, Out any](p Parser[In, Out], r ReaderAt[In]) (out Out, err error) {
 	return
 }
 
+type ParseError Message
+
+func (pe ParseError) Error() string {
+	return Message(pe).String()
+}
+
 // Return creates a parser that always succeeds and returns value without consuming any input.
 func Return[In, Out any](value Out) Parser[In, Out] {
 	return func(state State[In]) (Result[In, Out], error) {
@@ -355,13 +364,6 @@ func Try[In, Out any](p Parser[In, Out]) Parser[In, Out] {
 	}
 }
 
-// Map creates a parser that converts the output of p from Out1 to Out2.
-func Map[In, Out1, Out2 any](p Parser[In, Out1], f func(Out1) Out2) Parser[In, Out2] {
-	return Bind(p, func(out Out1) Parser[In, Out2] {
-		return Return[In](f(out))
-	})
-}
-
 func end[In any](s State[In]) (Result[In, struct{}], error) {
 	_, _, err := s.Read([]In{})
 	if errors.Is(err, io.EOF) {
@@ -424,6 +426,7 @@ func Lazy[In, Out any](p func() Parser[In, Out]) Parser[In, Out] {
 	}
 }
 
+// Bracket parses p, flanked by left and right.
 func Bracket[In, Out, LOut, ROut any](left Parser[In, LOut], p Parser[In, Out], right Parser[In, ROut]) Parser[In, Out] {
 	return Seq3(left, p, right, func(_ LOut, val Out, _ ROut) Out { return val })
 }

go.mod

@@ -1,6 +1,6 @@
 module git.codemonkeysoftware.net/b/gigaparsec
 
-go 1.23
+go 1.27
 
 require (
 	github.com/shoenig/test v1.11.0

naive/naive_test.go

@@ -22,13 +22,13 @@ func BenchmarkBind5(b *testing.B) {
 	input := gigaparsec.SliceReaderAt[byte]{0}
 	b.Run("gigaparsec.Bind5", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(gigaparsec.Bind5), input)
+			p(gigaparsec.Bind5).Run(input)
 		}
 
 	})
 	b.Run("naïve.Bind5", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(naive.Bind5), input)
+			p(naive.Bind5).Run(input)
 		}
 	})
 }
@@ -46,13 +46,13 @@ func BenchmarkBind9(b *testing.B) {
 	input := gigaparsec.SliceReaderAt[byte]{0}
 	b.Run("gigaparsec.Bind9", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(gigaparsec.Bind9), input)
+			p(gigaparsec.Bind9).Run(input)
 		}
 
 	})
 	b.Run("naive.Bind9", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(naive.Bind9), input)
+			p(naive.Bind9).Run(input)
 		}
 	})
 }
@@ -72,12 +72,12 @@ func BenchmarkSeq5(b *testing.B) {
 	input := gigaparsec.SliceReaderAt[byte]{0}
 	b.Run("gigaparsec.Seq5", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(gigaparsec.Seq5), input)
+			p(gigaparsec.Seq5).Run(input)
 		}
 	})
 	b.Run("naive.Seq5", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(naive.Seq5), input)
+			p(naive.Seq5).Run(input)
 		}
 	})
 }
@@ -97,12 +97,12 @@ func BenchmarkSeq9(b *testing.B) {
 	input := gigaparsec.SliceReaderAt[byte]{0}
 	b.Run("gigaparsec.Seq9", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(gigaparsec.Seq9), input)
+			p(gigaparsec.Seq9).Run(input)
 		}
 	})
 	b.Run("naive.Seq9", func(b *testing.B) {
 		for range b.N {
-			gigaparsec.Run(p(naive.Seq9), input)
+			p(naive.Seq9).Run(input)
 		}
 	})
 }