Finish and test Token combinator

2026-08-20 22:54:31 -06:00
parent 5d1f4940ce
commit 6b201fbb69
3 changed files with 52 additions and 3 deletions
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Unlicense
+
+package bytes_test
+
+import (
+	"fmt"
+	"strings"
+
+	"git.codemonkeysoftware.net/b/gigaparsec"
+	"git.codemonkeysoftware.net/b/gigaparsec/bytes"
+)
+
+func ExampleToken() {
+	ptokens := gigaparsec.Choose(
+		bytes.MatchString("("),
+		bytes.MatchString(")"),
+		bytes.MatchString("+"),
+		bytes.MatchString("-"),
+		bytes.Regexp(`[a-zA-Z][a-zA-Z0-9_]*`).Label("identifier"),
+		bytes.Regexp(`[0-9]+`).Label("number"),
+	)
+
+	tokenize := gigaparsec.Repeat(0, bytes.Token[string](bytes.WhitespaceASCII())(ptokens))
+	tokens, _ := gigaparsec.Run(tokenize, strings.NewReader("(alpha + 42) -5  "))
+	for _, token := range tokens {
+		fmt.Println(token)
+	}
+
+	// Output:
+	// (
+	// alpha
+	// +
+	// 42
+	// )
+	// -
+	// 5
+}