Updated README

Makefile

@@ -8,6 +8,6 @@ vet: fmt
 buildLib: vet
 	go build -gcflags="all=-N -l" ./...
 buildCmd: buildLib
-	go build -C cmd/ -gcflags="all=-N -l" -o re ./...
+	go build -C kg/ -gcflags="all=-N -l" -o kg ./...
 test: buildCmd
 	go test -v ./...

README.md

@@ -15,3 +15,7 @@ It also includes features not present in regexp, such as lookarounds and backref
 The syntax is, for the most part, a superset of Go's regexp. A full overview of the syntax can be found [here](https://pkg.go.dev/gitea.twomorecents.org/Rockingcool/kleingrep/regex#hdr-Syntax).
 
 __For more information, see https://pkg.go.dev/gitea.twomorecents.org/Rockingcool/kleingrep/regex__.
+
+### How it works
+
+I've written about the inner workings of the engine [on my blog](https://twomorecents.org/writing-regex-engine/index.html).

cmd/main.go

@@ -1,245 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/fatih/color"
-
-	reg "gitea.twomorecents.org/Rockingcool/kleingrep/regex"
-)
-
-func main() {
-	// Flags for the regex Compile function
-	flagsToCompile := make([]reg.ReFlag, 0)
-
-	invertFlag := flag.Bool("v", false, "Invert match.")
-	// This flag has two 'modes':
-	// 1. Without '-v': Prints only matches. Prints a newline after every match.
-	// 2. With '-v': Substitutes all matches with empty string.
-	onlyFlag := flag.Bool("o", false, "Print only colored content. Overrides -l.")
-	lineFlag := flag.Bool("l", false, "Only print lines with a match (or with no matches, if -v is enabled). Similar to grep's default.")
-	multiLineFlag := flag.Bool("t", false, "Multi-line mode. Treats newline just like any character.")
-	printMatchesFlag := flag.Bool("p", false, "Prints start and end index of each match. Can only be used with '-t' for multi-line mode.")
-	caseInsensitiveFlag := flag.Bool("i", false, "Case-insensitive. Disregard the case of all characters.")
-	matchNum := flag.Int("m", 0, "Print the match with the given index. Eg. -m 3 prints the third match.")
-	substituteText := flag.String("s", "", "Substitute the contents of each match with the given string. Overrides -o and -v")
-	flag.Parse()
-
-	// These flags have to be passed to the Compile function
-	if *multiLineFlag {
-		flagsToCompile = append(flagsToCompile, reg.RE_MULTILINE, reg.RE_SINGLE_LINE)
-	}
-	if *caseInsensitiveFlag {
-		flagsToCompile = append(flagsToCompile, reg.RE_CASE_INSENSITIVE)
-	}
-
-	// -l and -o are mutually exclusive: -o overrides -l
-	if *onlyFlag {
-		*lineFlag = false
-	}
-	// Check if substitute and matchNum flags have been enabled
-	substituteFlagEnabled := false
-	matchNumFlagEnabled := false
-	flag.Visit(func(f *flag.Flag) {
-		if f.Name == "s" {
-			substituteFlagEnabled = true
-		}
-		if f.Name == "m" {
-			matchNumFlagEnabled = true
-		}
-	})
-
-	// Validate matchNumFlag - must be positive integer
-	if matchNumFlagEnabled && *matchNum < 1 {
-		panic("Invalid match number to print.")
-	}
-
-	// Process:
-	// 1. Convert regex into postfix notation (Shunting-Yard algorithm)
-	// 		a. Add explicit concatenation operators to facilitate this
-	// 2. Build NFA from postfix representation (Thompson's algorithm)
-	// 3. Run the string against the NFA
-
-	if len(flag.Args()) < 1 || len(flag.Args()) > 2 { // flag.Args() also strips out program name
-		fmt.Println("ERROR: Missing cmdline args")
-		os.Exit(22)
-	}
-	var re string
-	re = flag.Args()[0]
-	var inputFile *os.File
-	if len(flag.Args()) == 1 || flag.Args()[1] == "-" { // Either no file argument, or file argument is "-"
-		inputFile = os.Stdin
-	} else {
-		var err error
-		inputFile, err = os.Open(flag.Args()[1])
-		if err != nil {
-			fmt.Printf("%s: No such file or directory\n", flag.Args()[1])
-			os.Exit(2)
-		}
-	}
-
-	var test_str string
-	var err error
-	var linesRead bool // Whether or not we have read the lines in the file
-	lineNum := 0       // Current line number
-	// Create reader for stdin and writer for stdout
-	reader := bufio.NewReader(inputFile)
-	out := bufio.NewWriter(os.Stdout)
-
-	regComp, err := reg.Compile(re, flagsToCompile...)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	for true {
-		if linesRead {
-			break
-		}
-		if !(*multiLineFlag) {
-			// Read every string from stdin until we encounter an error. If the error isn't EOF, panic.
-			test_str, err = reader.ReadString('\n')
-			lineNum++
-			if err != nil {
-				if err == io.EOF {
-					linesRead = true
-				} else {
-					panic(err)
-				}
-			}
-			if len(test_str) > 0 && test_str[len(test_str)-1] == '\n' {
-				test_str = test_str[:len(test_str)-1]
-			}
-		} else {
-			// Multi-line mode - read every line of input into a temp. string.
-			// test_str will contain all lines of input (including newline characters)
-			// as one string.
-			var temp string
-			for temp, err = reader.ReadString('\n'); err == nil; temp, err = reader.ReadString('\n') {
-				test_str += temp
-			}
-			// Assuming err != nil
-			if err == io.EOF {
-				if len(temp) > 0 {
-					test_str += temp // Add the last line (if it is non-empty)
-				}
-				linesRead = true
-			} else {
-				panic(err)
-			}
-		}
-		matchIndices := make([]reg.Match, 0)
-		if matchNumFlagEnabled {
-			tmp, err := regComp.FindNthMatch(test_str, *matchNum)
-			if err == nil {
-				matchIndices = append(matchIndices, tmp)
-			}
-		} else {
-			matchIndices = regComp.FindAllSubmatch(test_str)
-		}
-
-		test_str_runes := []rune(test_str) // Converting to runes preserves unicode characters
-
-		if *printMatchesFlag {
-			// if we are in single line mode, print the line on which
-			// the matches occur
-			if len(matchIndices) > 0 {
-				if !(*multiLineFlag) {
-					fmt.Fprintf(out, "Line %d:\n", lineNum)
-				}
-				for _, m := range matchIndices {
-					fmt.Fprintf(out, "%s\n", m.String())
-				}
-				err := out.Flush()
-				if err != nil {
-					panic(err)
-				}
-			}
-			continue
-		}
-		// Decompose the array of matchIndex structs into a flat unique array of ints - if matchIndex is {4,7}, flat array will contain 4,5,6
-		// This should make checking O(1) instead of O(n)
-		indicesToPrint := new_uniq_arr[int]()
-		for _, idx := range matchIndices {
-			indicesToPrint.add(genRange(idx[0].StartIdx, idx[0].EndIdx)...)
-		}
-		// If we are inverting, then we should print the indices which _didn't_ match
-		// in color.
-		if *invertFlag {
-			oldIndices := indicesToPrint.values()
-			indicesToPrint = new_uniq_arr[int]()
-			// Explanation:
-			// Find all numbers from 0 to len(test_str_runes) that are NOT in oldIndices.
-			// These are the values we want to print, now that we have inverted the match.
-			// Re-initialize indicesToPrint and add all of these values to it.
-			indicesToPrint.add(setDifference(genRange(0, len(test_str_runes)), oldIndices)...)
-
-		}
-		// If lineFlag is enabled, we should only print something if:
-		// 		a. We are not inverting, and have at least one match on the current line
-		// 		OR
-		// 		b. We are inverting, and have no matches at all on the current line.
-		// This checks for the inverse, and continues if it is true.
-		if *lineFlag {
-			if !(*invertFlag) && len(matchIndices) == 0 || *invertFlag && len(matchIndices) > 0 {
-				continue
-			}
-		}
-
-		// If we are substituting, we need a different behavior, as follows:
-		// For every character in the test string:
-		// 		1. Check if the index is the start of any matchIndex
-		// 		2. If so, print the substitute text, and set our index to
-		//			the corresponding end index.
-		// 		3. If not, just print the character.
-		if substituteFlagEnabled {
-			for i := range test_str_runes {
-				inMatchIndex := false
-				for _, m := range matchIndices {
-					if i == m[0].StartIdx {
-						fmt.Fprintf(out, "%s", *substituteText)
-						i = m[0].EndIdx
-						inMatchIndex = true
-						break
-					}
-				}
-				if !inMatchIndex {
-					fmt.Fprintf(out, "%c", test_str_runes[i])
-				}
-			}
-		} else {
-			for i, c := range test_str_runes {
-				if indicesToPrint.contains(i) {
-					color.New(color.FgRed).Fprintf(out, "%c", c)
-					// Newline after every match - only if -o is enabled and -v is disabled.
-					if *onlyFlag && !(*invertFlag) {
-						for matchIdxNum, idx := range matchIndices {
-							if matchIdxNum < len(matchIndices)-1 { // Only print a newline afte printing a match, if there are multiple matches on the line, and we aren't on the last one. This is because the newline that gets added at the end will take care of that.
-								if i+1 == idx[0].EndIdx { // End index is one more than last index of match
-									fmt.Fprintf(out, "\n")
-									break
-								}
-							}
-						}
-					}
-				} else {
-					if !(*onlyFlag) {
-						fmt.Fprintf(out, "%c", c)
-					}
-				}
-			}
-		}
-		err = out.Flush()
-		if err != nil {
-			panic(err)
-		}
-		// If the last character in the string wasn't a newline, AND we either have don't -o set or we do (and we've matched something), then print a newline
-		if (len(test_str_runes) > 0 && test_str_runes[len(test_str_runes)-1] != '\n') &&
-			(!*onlyFlag || indicesToPrint.len() > 0) {
-			fmt.Println()
-		}
-	}
-}

kg/main.go

@@ -0,0 +1,284 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+
+	"github.com/fatih/color"
+
+	reg "gitea.twomorecents.org/Rockingcool/kleingrep/regex"
+)
+
+func main() {
+	// Flags for the regex Compile function
+	flagsToCompile := make([]reg.ReFlag, 0)
+
+	invertFlag := flag.Bool("v", false, "Invert match.")
+	// This flag has two 'modes':
+	// 1. Without '-v': Prints only matches. Prints a newline after every match.
+	// 2. With '-v': Substitutes all matches with empty string.
+	onlyFlag := flag.Bool("o", false, "Print only colored content. Overrides -l.")
+	lineFlag := flag.Bool("l", false, "Only print lines with a match (or with no matches, if -v is enabled). Similar to grep's default.")
+	multiLineFlag := flag.Bool("t", false, "Multi-line mode. Treats newline just like any character.")
+	printMatchesFlag := flag.Bool("p", false, "Prints start and end index of each match. Can only be used with '-t' for multi-line mode.")
+	caseInsensitiveFlag := flag.Bool("i", false, "Case-insensitive. Disregard the case of all characters.")
+	recursiveFlag := flag.Bool("r", false, "Recursively search all files in the given directory.")
+	lineNumFlag := flag.Bool("n", false, "For each line with a match, print the line number. Implies -l.")
+	matchNum := flag.Int("m", 0, "Print the match with the given index. Eg. -m 3 prints the third match.")
+	substituteText := flag.String("s", "", "Substitute the contents of each match with the given string. Overrides -o and -v")
+	flag.Parse()
+
+	// These flags have to be passed to the Compile function
+	if *multiLineFlag {
+		flagsToCompile = append(flagsToCompile, reg.RE_MULTILINE, reg.RE_SINGLE_LINE)
+	}
+	if *caseInsensitiveFlag {
+		flagsToCompile = append(flagsToCompile, reg.RE_CASE_INSENSITIVE)
+	}
+
+	// -l and -o are mutually exclusive: -o overrides -l
+	if *onlyFlag {
+		*lineFlag = false
+	}
+	// Check if substitute and matchNum flags have been enabled
+	substituteFlagEnabled := false
+	matchNumFlagEnabled := false
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == "s" {
+			substituteFlagEnabled = true
+		}
+		if f.Name == "m" {
+			matchNumFlagEnabled = true
+		}
+	})
+
+	// Validate matchNumFlag - must be positive integer
+	if matchNumFlagEnabled && *matchNum < 1 {
+		panic("Invalid match number to print.")
+	}
+
+	// Enable lineFlag if lineNumFlag is enabled
+	if *lineNumFlag {
+		*lineFlag = true
+	}
+
+	// Process:
+	// 1. Convert regex into postfix notation (Shunting-Yard algorithm)
+	// 		a. Add explicit concatenation operators to facilitate this
+	// 2. Build NFA from postfix representation (Thompson's algorithm)
+	// 3. Run the string against the NFA
+
+	if len(flag.Args()) < 1 { // flag.Args() also strips out program name
+		fmt.Printf("%s: ERROR: Missing cmdline args\n", os.Args[0])
+		os.Exit(22)
+	}
+	if *recursiveFlag && len(flag.Args()) < 2 { // File/Directory must be provided with '-r'
+		fmt.Printf("%s: ERROR: Missing cmdline args\n", os.Args[0])
+		os.Exit(22)
+	}
+	var re string
+	re = flag.Args()[0]
+	var inputFiles []*os.File
+	if len(flag.Args()) == 1 || flag.Args()[1] == "-" { // Either no file argument, or file argument is "-"
+		if !slices.Contains(inputFiles, os.Stdin) {
+			inputFiles = append(inputFiles, os.Stdin) // os.Stdin cannot be entered more than once into the file list
+		}
+	} else {
+		inputFilenames := flag.Args()[1:]
+		for _, inputFilename := range inputFilenames {
+			inputFile, err := os.Open(inputFilename)
+			if err != nil {
+				fmt.Printf("%s: %s: No such file or directory\n", os.Args[0], inputFilename)
+			} else {
+				fileStat, err := inputFile.Stat()
+				if err != nil {
+					fmt.Printf("%v\n", err)
+					os.Exit(2)
+				} else {
+					if fileStat.Mode().IsDir() {
+						fmt.Printf("%s: %s: Is a directory\n", os.Args[0], inputFilename)
+					} else {
+						inputFiles = append(inputFiles, inputFile)
+					}
+				}
+			}
+		}
+	}
+
+	var test_str string
+	var err error
+	var linesRead bool // Whether or not we have read the lines in the file
+	lineNum := 0       // Current line number
+	// Create writer for stdout
+	out := bufio.NewWriter(os.Stdout)
+	// Compile regex
+	regComp, err := reg.Compile(re, flagsToCompile...)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for _, inputFile := range inputFiles {
+		lineNum = 0
+		reader := bufio.NewReader(inputFile)
+		linesRead = false
+		for true {
+			if linesRead {
+				break
+			}
+			if !(*multiLineFlag) {
+				// Read every string from stdin until we encounter an error. If the error isn't EOF, panic.
+				test_str, err = reader.ReadString('\n')
+				lineNum++
+				if err != nil {
+					if err == io.EOF {
+						linesRead = true
+					} else {
+						panic(err)
+					}
+				}
+				if len(test_str) > 0 && test_str[len(test_str)-1] == '\n' {
+					test_str = test_str[:len(test_str)-1]
+				}
+			} else {
+				// Multi-line mode - read every line of input into a temp. string.
+				// test_str will contain all lines of input (including newline characters)
+				// as one string.
+				var temp string
+				for temp, err = reader.ReadString('\n'); err == nil; temp, err = reader.ReadString('\n') {
+					test_str += temp
+				}
+				// Assuming err != nil
+				if err == io.EOF {
+					if len(temp) > 0 {
+						test_str += temp // Add the last line (if it is non-empty)
+					}
+					linesRead = true
+				} else {
+					panic(err)
+				}
+			}
+			matchIndices := make([]reg.Match, 0)
+			if matchNumFlagEnabled {
+				tmp, err := regComp.FindNthMatch(test_str, *matchNum)
+				if err == nil {
+					matchIndices = append(matchIndices, tmp)
+				}
+			} else {
+				matchIndices = regComp.FindAllSubmatch(test_str)
+			}
+
+			test_str_runes := []rune(test_str) // Converting to runes preserves unicode characters
+
+			if *printMatchesFlag {
+				// if we are in single line mode, print the line on which
+				// the matches occur
+				if len(matchIndices) > 0 {
+					if !(*multiLineFlag) {
+						fmt.Fprintf(out, "Line %d:\n", lineNum)
+					}
+					for _, m := range matchIndices {
+						fmt.Fprintf(out, "%s\n", m.String())
+					}
+					err := out.Flush()
+					if err != nil {
+						panic(err)
+					}
+				}
+				continue
+			}
+			// Decompose the array of matchIndex structs into a flat unique array of ints - if matchIndex is {4,7}, flat array will contain 4,5,6
+			// This should make checking O(1) instead of O(n)
+			indicesToPrint := new_uniq_arr[int]()
+			for _, idx := range matchIndices {
+				indicesToPrint.add(genRange(idx[0].StartIdx, idx[0].EndIdx)...)
+			}
+			// If we are inverting, then we should print the indices which _didn't_ match
+			// in color.
+			if *invertFlag {
+				oldIndices := indicesToPrint.values()
+				indicesToPrint = new_uniq_arr[int]()
+				// Explanation:
+				// Find all numbers from 0 to len(test_str_runes) that are NOT in oldIndices.
+				// These are the values we want to print, now that we have inverted the match.
+				// Re-initialize indicesToPrint and add all of these values to it.
+				indicesToPrint.add(setDifference(genRange(0, len(test_str_runes)), oldIndices)...)
+
+			}
+			// If lineFlag is enabled, we should only print something if:
+			// 		a. We are not inverting, and have at least one match on the current line
+			// 		OR
+			// 		b. We are inverting, and have no matches at all on the current line.
+			// This checks for the inverse, and continues if it is true.
+			if *lineFlag {
+				if !(*invertFlag) && len(matchIndices) == 0 || *invertFlag && len(matchIndices) > 0 {
+					continue
+				} else {
+					if *recursiveFlag || len(flag.Args()) > 2 { // If we have 2 args, then we're only searching 1 file. We should only print the filename if there's more than 1 file.
+						color.New(color.FgMagenta).Fprintf(out, "%s:", inputFile.Name()) // Print filename
+					}
+					if *lineNumFlag {
+						color.New(color.FgGreen).Fprintf(out, "%d:", lineNum) // Print filename
+					}
+				}
+			}
+
+			// If we are substituting, we need a different behavior, as follows:
+			// For every character in the test string:
+			// 		1. Check if the index is the start of any matchIndex
+			// 		2. If so, print the substitute text, and set our index to
+			//			the corresponding end index.
+			// 		3. If not, just print the character.
+			if substituteFlagEnabled {
+				for i := range test_str_runes {
+					inMatchIndex := false
+					for _, m := range matchIndices {
+						if i == m[0].StartIdx {
+							fmt.Fprintf(out, "%s", *substituteText)
+							i = m[0].EndIdx
+							inMatchIndex = true
+							break
+						}
+					}
+					if !inMatchIndex {
+						fmt.Fprintf(out, "%c", test_str_runes[i])
+					}
+				}
+			} else {
+				for i, c := range test_str_runes {
+					if indicesToPrint.contains(i) {
+						color.New(color.FgRed, color.Bold).Fprintf(out, "%c", c)
+						// Newline after every match - only if -o is enabled and -v is disabled.
+						if *onlyFlag && !(*invertFlag) {
+							for matchIdxNum, idx := range matchIndices {
+								if matchIdxNum < len(matchIndices)-1 { // Only print a newline afte printing a match, if there are multiple matches on the line, and we aren't on the last one. This is because the newline that gets added at the end will take care of that.
+									if i+1 == idx[0].EndIdx { // End index is one more than last index of match
+										fmt.Fprintf(out, "\n")
+										break
+									}
+								}
+							}
+						}
+					} else {
+						if !(*onlyFlag) {
+							fmt.Fprintf(out, "%c", c)
+						}
+					}
+				}
+			}
+			err = out.Flush()
+			if err != nil {
+				panic(err)
+			}
+			// If the last character in the string wasn't a newline, AND we either have don't -o set or we do (and we've matched something), then print a newline
+			if (len(test_str_runes) > 0 && test_str_runes[len(test_str_runes)-1] != '\n') &&
+				(!*onlyFlag || indicesToPrint.len() > 0) {
+				fmt.Println()
+			}
+		}
+	}
+}

regex/compile.go

@@ -47,6 +47,7 @@ func (re *Reg) UnmarshalText(text []byte) error {
 	return err
 }
 
+// Longest makes future searches prefer the longest branch of an alternation, as opposed to the leftmost branch.
 func (re *Reg) Longest() {
 	re.preferLongest = true
 }