Compare commits
No commits in common. 'master' and 'implementUnicodeCharClass' have entirely different histories.
master
...
implementU
@ -1 +1,2 @@
|
||||
kg/kg
|
||||
re
|
||||
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
## Kleingrep
|
||||
|
||||
Kleingrep is a regular expression engine, providing a library and command-line tool written in Go.
|
||||
|
||||
It aims to provide a more featureful engine, compared to the one in Go's
|
||||
[regexp](https://pkg.go.dev/regexp), while retaining some semblance of efficiency.
|
||||
|
||||
The engine does __not__ use backtracking, relying on the NFA-based method described in
|
||||
[Russ Cox's articles](https://swtch.com/~rsc/regexp). As such, it is immune to catastrophic backtracking.
|
||||
|
||||
It also includes features not present in regexp, such as lookarounds and backreferences.
|
||||
|
||||
### Syntax
|
||||
|
||||
The syntax is, for the most part, a superset of Go's regexp. A full overview of the syntax can be found [here](https://pkg.go.dev/gitea.twomorecents.org/Rockingcool/kleingrep/regex#hdr-Syntax).
|
||||
|
||||
__For more information, see https://pkg.go.dev/gitea.twomorecents.org/Rockingcool/kleingrep/regex__.
|
||||
|
||||
### How it works
|
||||
|
||||
I've written about the inner workings of the engine [on my blog](https://twomorecents.org/writing-regex-engine/index.html).
|
||||
@ -0,0 +1,227 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/fatih/color"
|
||||
|
||||
reg "gitea.twomorecents.org/Rockingcool/kleingrep/regex"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Flags for the regex Compile function
|
||||
flagsToCompile := make([]reg.ReFlag, 0)
|
||||
|
||||
invertFlag := flag.Bool("v", false, "Invert match.")
|
||||
// This flag has two 'modes':
|
||||
// 1. Without '-v': Prints only matches. Prints a newline after every match.
|
||||
// 2. With '-v': Substitutes all matches with empty string.
|
||||
onlyFlag := flag.Bool("o", false, "Print only colored content. Overrides -l.")
|
||||
lineFlag := flag.Bool("l", false, "Only print lines with a match (or with no matches, if -v is enabled). Similar to grep's default.")
|
||||
multiLineFlag := flag.Bool("t", false, "Multi-line mode. Treats newline just like any character.")
|
||||
printMatchesFlag := flag.Bool("p", false, "Prints start and end index of each match. Can only be used with '-t' for multi-line mode.")
|
||||
caseInsensitiveFlag := flag.Bool("i", false, "Case-insensitive. Disregard the case of all characters.")
|
||||
matchNum := flag.Int("m", 0, "Print the match with the given index. Eg. -m 3 prints the third match.")
|
||||
substituteText := flag.String("s", "", "Substitute the contents of each match with the given string. Overrides -o and -v")
|
||||
flag.Parse()
|
||||
|
||||
// These flags have to be passed to the Compile function
|
||||
if *multiLineFlag {
|
||||
flagsToCompile = append(flagsToCompile, reg.RE_MULTILINE, reg.RE_SINGLE_LINE)
|
||||
}
|
||||
if *caseInsensitiveFlag {
|
||||
flagsToCompile = append(flagsToCompile, reg.RE_CASE_INSENSITIVE)
|
||||
}
|
||||
|
||||
// -l and -o are mutually exclusive: -o overrides -l
|
||||
if *onlyFlag {
|
||||
*lineFlag = false
|
||||
}
|
||||
// Check if substitute and matchNum flags have been enabled
|
||||
substituteFlagEnabled := false
|
||||
matchNumFlagEnabled := false
|
||||
flag.Visit(func(f *flag.Flag) {
|
||||
if f.Name == "s" {
|
||||
substituteFlagEnabled = true
|
||||
}
|
||||
if f.Name == "m" {
|
||||
matchNumFlagEnabled = true
|
||||
}
|
||||
})
|
||||
|
||||
// Validate matchNumFlag - must be positive integer
|
||||
if matchNumFlagEnabled && *matchNum < 1 {
|
||||
panic("Invalid match number to print.")
|
||||
}
|
||||
|
||||
// Process:
|
||||
// 1. Convert regex into postfix notation (Shunting-Yard algorithm)
|
||||
// a. Add explicit concatenation operators to facilitate this
|
||||
// 2. Build NFA from postfix representation (Thompson's algorithm)
|
||||
// 3. Run the string against the NFA
|
||||
|
||||
if len(flag.Args()) != 1 { // flag.Args() also strips out program name
|
||||
fmt.Println("ERROR: Missing cmdline args")
|
||||
os.Exit(22)
|
||||
}
|
||||
var re string
|
||||
re = flag.Args()[0]
|
||||
var test_str string
|
||||
var err error
|
||||
var linesRead bool // Whether or not we have read the lines in the file
|
||||
lineNum := 0 // Current line number
|
||||
// Create reader for stdin and writer for stdout
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
out := bufio.NewWriter(os.Stdout)
|
||||
|
||||
regComp, err := reg.Compile(re, flagsToCompile...)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
for true {
|
||||
if linesRead {
|
||||
break
|
||||
}
|
||||
if !(*multiLineFlag) {
|
||||
// Read every string from stdin until we encounter an error. If the error isn't EOF, panic.
|
||||
test_str, err = reader.ReadString('\n')
|
||||
lineNum++
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
linesRead = true
|
||||
} else {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if len(test_str) > 0 && test_str[len(test_str)-1] == '\n' {
|
||||
test_str = test_str[:len(test_str)-1]
|
||||
}
|
||||
} else {
|
||||
// Multi-line mode - read every line of input into a temp. string.
|
||||
// test_str will contain all lines of input (including newline characters)
|
||||
// as one string.
|
||||
var temp string
|
||||
for temp, err = reader.ReadString('\n'); err == nil; temp, err = reader.ReadString('\n') {
|
||||
test_str += temp
|
||||
}
|
||||
// Assuming err != nil
|
||||
if err == io.EOF {
|
||||
if len(temp) > 0 {
|
||||
test_str += temp // Add the last line (if it is non-empty)
|
||||
}
|
||||
linesRead = true
|
||||
} else {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
matchIndices := make([]reg.Match, 0)
|
||||
if matchNumFlagEnabled {
|
||||
tmp, err := regComp.FindNthMatch(test_str, *matchNum)
|
||||
if err == nil {
|
||||
matchIndices = append(matchIndices, tmp)
|
||||
}
|
||||
} else {
|
||||
matchIndices = regComp.FindAllSubmatch(test_str)
|
||||
}
|
||||
|
||||
test_str_runes := []rune(test_str) // Converting to runes preserves unicode characters
|
||||
|
||||
if *printMatchesFlag {
|
||||
// if we are in single line mode, print the line on which
|
||||
// the matches occur
|
||||
if len(matchIndices) > 0 {
|
||||
if !(*multiLineFlag) {
|
||||
fmt.Fprintf(out, "Line %d:\n", lineNum)
|
||||
}
|
||||
for _, m := range matchIndices {
|
||||
fmt.Fprintf(out, "%s\n", m.String())
|
||||
}
|
||||
err := out.Flush()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Decompose the array of matchIndex structs into a flat unique array of ints - if matchIndex is {4,7}, flat array will contain 4,5,6
|
||||
// This should make checking O(1) instead of O(n)
|
||||
indicesToPrint := new_uniq_arr[int]()
|
||||
for _, idx := range matchIndices {
|
||||
indicesToPrint.add(genRange(idx[0].StartIdx, idx[0].EndIdx)...)
|
||||
}
|
||||
// If we are inverting, then we should print the indices which _didn't_ match
|
||||
// in color.
|
||||
if *invertFlag {
|
||||
oldIndices := indicesToPrint.values()
|
||||
indicesToPrint = new_uniq_arr[int]()
|
||||
// Explanation:
|
||||
// Find all numbers from 0 to len(test_str_runes) that are NOT in oldIndices.
|
||||
// These are the values we want to print, now that we have inverted the match.
|
||||
// Re-initialize indicesToPrint and add all of these values to it.
|
||||
indicesToPrint.add(setDifference(genRange(0, len(test_str_runes)), oldIndices)...)
|
||||
|
||||
}
|
||||
// If lineFlag is enabled, we should only print something if:
|
||||
// a. We are not inverting, and have at least one match on the current line
|
||||
// OR
|
||||
// b. We are inverting, and have no matches at all on the current line.
|
||||
// This checks for the inverse, and continues if it is true.
|
||||
if *lineFlag {
|
||||
if !(*invertFlag) && len(matchIndices) == 0 || *invertFlag && len(matchIndices) > 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// If we are substituting, we need a different behavior, as follows:
|
||||
// For every character in the test string:
|
||||
// 1. Check if the index is the start of any matchIndex
|
||||
// 2. If so, print the substitute text, and set our index to
|
||||
// the corresponding end index.
|
||||
// 3. If not, just print the character.
|
||||
if substituteFlagEnabled {
|
||||
for i := range test_str_runes {
|
||||
inMatchIndex := false
|
||||
for _, m := range matchIndices {
|
||||
if i == m[0].StartIdx {
|
||||
fmt.Fprintf(out, "%s", *substituteText)
|
||||
i = m[0].EndIdx
|
||||
inMatchIndex = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inMatchIndex {
|
||||
fmt.Fprintf(out, "%c", test_str_runes[i])
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for i, c := range test_str_runes {
|
||||
if indicesToPrint.contains(i) {
|
||||
color.New(color.FgRed).Fprintf(out, "%c", c)
|
||||
// Newline after every match - only if -o is enabled and -v is disabled.
|
||||
if *onlyFlag && !(*invertFlag) {
|
||||
for _, idx := range matchIndices {
|
||||
if i+1 == idx[0].EndIdx { // End index is one more than last index of match
|
||||
fmt.Fprintf(out, "\n")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !(*onlyFlag) {
|
||||
fmt.Fprintf(out, "%c", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
err = out.Flush()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
@ -1,298 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/fatih/color"
|
||||
|
||||
reg "gitea.twomorecents.org/Rockingcool/kleingrep/regex"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Flags for the regex Compile function
|
||||
flagsToCompile := make([]reg.ReFlag, 0)
|
||||
|
||||
invertFlag := flag.Bool("v", false, "Invert match.")
|
||||
// This flag has two 'modes':
|
||||
// 1. Without '-v': Prints only matches. Prints a newline after every match.
|
||||
// 2. With '-v': Substitutes all matches with empty string.
|
||||
onlyFlag := flag.Bool("o", false, "Print only colored content. Overrides -l.")
|
||||
lineFlag := flag.Bool("l", false, "Only print lines with a match (or with no matches, if -v is enabled). Similar to grep's default.")
|
||||
multiLineFlag := flag.Bool("t", false, "Multi-line mode. Treats newline just like any character.")
|
||||
printMatchesFlag := flag.Bool("p", false, "Prints start and end index of each match. Can only be used with '-t' for multi-line mode.")
|
||||
caseInsensitiveFlag := flag.Bool("i", false, "Case-insensitive. Disregard the case of all characters.")
|
||||
recursiveFlag := flag.Bool("r", false, "Recursively search all files in the given directory.")
|
||||
lineNumFlag := flag.Bool("n", false, "For each line with a match, print the line number. Implies -l.")
|
||||
matchNum := flag.Int("m", 0, "Print the match with the given index. Eg. -m 3 prints the third match.")
|
||||
substituteText := flag.String("s", "", "Substitute the contents of each match with the given string. Overrides -o and -v")
|
||||
flag.Parse()
|
||||
|
||||
// These flags have to be passed to the Compile function
|
||||
if *multiLineFlag {
|
||||
flagsToCompile = append(flagsToCompile, reg.RE_MULTILINE, reg.RE_SINGLE_LINE)
|
||||
}
|
||||
if *caseInsensitiveFlag {
|
||||
flagsToCompile = append(flagsToCompile, reg.RE_CASE_INSENSITIVE)
|
||||
}
|
||||
|
||||
// -l and -o are mutually exclusive: -o overrides -l
|
||||
if *onlyFlag {
|
||||
*lineFlag = false
|
||||
}
|
||||
// Check if substitute and matchNum flags have been enabled
|
||||
substituteFlagEnabled := false
|
||||
matchNumFlagEnabled := false
|
||||
flag.Visit(func(f *flag.Flag) {
|
||||
if f.Name == "s" {
|
||||
substituteFlagEnabled = true
|
||||
}
|
||||
if f.Name == "m" {
|
||||
matchNumFlagEnabled = true
|
||||
}
|
||||
})
|
||||
|
||||
// Validate matchNumFlag - must be positive integer
|
||||
if matchNumFlagEnabled && *matchNum < 1 {
|
||||
panic("Invalid match number to print.")
|
||||
}
|
||||
|
||||
// Enable lineFlag if lineNumFlag is enabled
|
||||
if *lineNumFlag {
|
||||
*lineFlag = true
|
||||
}
|
||||
|
||||
// Process:
|
||||
// 1. Convert regex into postfix notation (Shunting-Yard algorithm)
|
||||
// a. Add explicit concatenation operators to facilitate this
|
||||
// 2. Build NFA from postfix representation (Thompson's algorithm)
|
||||
// 3. Run the string against the NFA
|
||||
|
||||
if len(flag.Args()) < 1 { // flag.Args() also strips out program name
|
||||
fmt.Printf("%s: ERROR: Missing cmdline args\n", os.Args[0])
|
||||
os.Exit(22)
|
||||
}
|
||||
if *recursiveFlag && len(flag.Args()) < 2 { // File/Directory must be provided with '-r'
|
||||
fmt.Printf("%s: ERROR: Missing cmdline args\n", os.Args[0])
|
||||
os.Exit(22)
|
||||
}
|
||||
var re string
|
||||
re = flag.Args()[0]
|
||||
var inputFiles []*os.File
|
||||
if len(flag.Args()) == 1 || flag.Args()[1] == "-" { // Either no file argument, or file argument is "-"
|
||||
if !slices.Contains(inputFiles, os.Stdin) {
|
||||
inputFiles = append(inputFiles, os.Stdin) // os.Stdin cannot be entered more than once into the file list
|
||||
}
|
||||
} else {
|
||||
inputFilenames := flag.Args()[1:]
|
||||
for _, inputFilename := range inputFilenames {
|
||||
inputFile, err := os.Open(inputFilename)
|
||||
defer inputFile.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("%s: %s: No such file or directory\n", os.Args[0], inputFilename)
|
||||
} else {
|
||||
fileStat, err := inputFile.Stat()
|
||||
if err != nil {
|
||||
fmt.Printf("%v\n", err)
|
||||
os.Exit(2)
|
||||
} else {
|
||||
if fileStat.Mode().IsDir() {
|
||||
fmt.Printf("%s: %s: Is a directory\n", os.Args[0], inputFilename)
|
||||
} else {
|
||||
var nullCharPresent bool
|
||||
if nullCharPresent, err = fileContainsNullChar(inputFilename); nullCharPresent {
|
||||
if err != nil {
|
||||
fmt.Printf("%v\n", err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
fmt.Printf("%s: %s: Is a binary file\n", os.Args[0], inputFilename)
|
||||
}
|
||||
} else {
|
||||
inputFiles = append(inputFiles, inputFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(inputFiles) == 0 { // No valid files given
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
var test_str string
|
||||
var err error
|
||||
var linesRead bool // Whether or not we have read the lines in the file
|
||||
lineNum := 0 // Current line number
|
||||
// Create writer for stdout
|
||||
out := bufio.NewWriter(os.Stdout)
|
||||
// Compile regex
|
||||
regComp, err := reg.Compile(re, flagsToCompile...)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, inputFile := range inputFiles {
|
||||
lineNum = 0
|
||||
reader := bufio.NewReader(inputFile)
|
||||
linesRead = false
|
||||
for true {
|
||||
if linesRead {
|
||||
break
|
||||
}
|
||||
if !(*multiLineFlag) {
|
||||
// Read every string from stdin until we encounter an error. If the error isn't EOF, panic.
|
||||
test_str, err = reader.ReadString('\n')
|
||||
lineNum++
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
linesRead = true
|
||||
} else {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if len(test_str) > 0 && test_str[len(test_str)-1] == '\n' {
|
||||
test_str = test_str[:len(test_str)-1]
|
||||
}
|
||||
} else {
|
||||
// Multi-line mode - read every line of input into a temp. string.
|
||||
// test_str will contain all lines of input (including newline characters)
|
||||
// as one string.
|
||||
var temp string
|
||||
for temp, err = reader.ReadString('\n'); err == nil; temp, err = reader.ReadString('\n') {
|
||||
test_str += temp
|
||||
}
|
||||
// Assuming err != nil
|
||||
if err == io.EOF {
|
||||
if len(temp) > 0 {
|
||||
test_str += temp // Add the last line (if it is non-empty)
|
||||
}
|
||||
linesRead = true
|
||||
} else {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
matchIndices := make([]reg.Match, 0)
|
||||
if matchNumFlagEnabled {
|
||||
tmp, err := regComp.FindNthMatch(test_str, *matchNum)
|
||||
if err == nil {
|
||||
matchIndices = append(matchIndices, tmp)
|
||||
}
|
||||
} else {
|
||||
matchIndices = regComp.FindAllSubmatch(test_str)
|
||||
}
|
||||
|
||||
test_str_runes := []rune(test_str) // Converting to runes preserves unicode characters
|
||||
|
||||
if *printMatchesFlag {
|
||||
// if we are in single line mode, print the line on which
|
||||
// the matches occur
|
||||
if len(matchIndices) > 0 {
|
||||
if !(*multiLineFlag) {
|
||||
fmt.Fprintf(out, "Line %d:\n", lineNum)
|
||||
}
|
||||
for _, m := range matchIndices {
|
||||
fmt.Fprintf(out, "%s\n", m.String())
|
||||
}
|
||||
err := out.Flush()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Decompose the array of matchIndex structs into a flat unique array of ints - if matchIndex is {4,7}, flat array will contain 4,5,6
|
||||
// This should make checking O(1) instead of O(n)
|
||||
indicesToPrint := new_uniq_arr[int]()
|
||||
for _, idx := range matchIndices {
|
||||
indicesToPrint.add(genRange(idx[0].StartIdx, idx[0].EndIdx)...)
|
||||
}
|
||||
// If we are inverting, then we should print the indices which _didn't_ match
|
||||
// in color.
|
||||
if *invertFlag {
|
||||
oldIndices := indicesToPrint.values()
|
||||
indicesToPrint = new_uniq_arr[int]()
|
||||
// Explanation:
|
||||
// Find all numbers from 0 to len(test_str_runes) that are NOT in oldIndices.
|
||||
// These are the values we want to print, now that we have inverted the match.
|
||||
// Re-initialize indicesToPrint and add all of these values to it.
|
||||
indicesToPrint.add(setDifference(genRange(0, len(test_str_runes)), oldIndices)...)
|
||||
|
||||
}
|
||||
// If lineFlag is enabled, we should only print something if:
|
||||
// a. We are not inverting, and have at least one match on the current line
|
||||
// OR
|
||||
// b. We are inverting, and have no matches at all on the current line.
|
||||
// This checks for the inverse, and continues if it is true.
|
||||
if *lineFlag {
|
||||
if !(*invertFlag) && len(matchIndices) == 0 || *invertFlag && len(matchIndices) > 0 {
|
||||
continue
|
||||
} else {
|
||||
if *recursiveFlag || len(flag.Args()) > 2 { // If we have 2 args, then we're only searching 1 file. We should only print the filename if there's more than 1 file.
|
||||
color.New(color.FgMagenta).Fprintf(out, "%s:", inputFile.Name()) // Print filename
|
||||
}
|
||||
if *lineNumFlag {
|
||||
color.New(color.FgGreen).Fprintf(out, "%d:", lineNum) // Print filename
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we are substituting, we need a different behavior, as follows:
|
||||
// For every character in the test string:
|
||||
// 1. Check if the index is the start of any matchIndex
|
||||
// 2. If so, print the substitute text, and set our index to
|
||||
// the corresponding end index.
|
||||
// 3. If not, just print the character.
|
||||
if substituteFlagEnabled {
|
||||
for i := range test_str_runes {
|
||||
inMatchIndex := false
|
||||
for _, m := range matchIndices {
|
||||
if i == m[0].StartIdx {
|
||||
fmt.Fprintf(out, "%s", *substituteText)
|
||||
i = m[0].EndIdx
|
||||
inMatchIndex = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inMatchIndex {
|
||||
fmt.Fprintf(out, "%c", test_str_runes[i])
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for i, c := range test_str_runes {
|
||||
if indicesToPrint.contains(i) {
|
||||
color.New(color.FgRed, color.Bold).Fprintf(out, "%c", c)
|
||||
// Newline after every match - only if -o is enabled and -v is disabled.
|
||||
if *onlyFlag && !(*invertFlag) {
|
||||
for matchIdxNum, idx := range matchIndices {
|
||||
if matchIdxNum < len(matchIndices)-1 { // Only print a newline afte printing a match, if there are multiple matches on the line, and we aren't on the last one. This is because the newline that gets added at the end will take care of that.
|
||||
if i+1 == idx[0].EndIdx { // End index is one more than last index of match
|
||||
fmt.Fprintf(out, "\n")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !(*onlyFlag) {
|
||||
fmt.Fprintf(out, "%c", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
err = out.Flush()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// If the last character in the string wasn't a newline, AND we either have don't -o set or we do (and we've matched something), then print a newline
|
||||
if (len(test_str_runes) > 0 && test_str_runes[len(test_str_runes)-1] != '\n') &&
|
||||
(!*onlyFlag || indicesToPrint.len() > 0) {
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue