Improved submatch tracking by storing all group indices as a part of the state, which is viewed as a 'thread'
This commit is contained in:
90
matching.go
90
matching.go
@@ -62,15 +62,21 @@ func (g Group) isValid() bool {
|
||||
// the second ret val is true.
|
||||
// The third ret val is a list of all the group numbers of all the opening parentheses we crossed,
|
||||
// and the fourth is a list of all the closing parentheses we passed
|
||||
func takeZeroState(states []*State) (rtv []*State, isZero bool, openParenGroups []int, closeParenGroups []int) {
|
||||
func takeZeroState(states []*State, numGroups int, idx int) (rtv []*State, isZero bool) {
|
||||
for _, state := range states {
|
||||
if len(state.transitions[EPSILON]) > 0 {
|
||||
for _, s := range state.transitions[EPSILON] {
|
||||
if s.threadGroups == nil {
|
||||
s.threadGroups = newMatch(numGroups + 1)
|
||||
}
|
||||
copy(s.threadGroups, state.threadGroups)
|
||||
if s.groupBegin {
|
||||
openParenGroups = append(openParenGroups, s.groupNum)
|
||||
s.threadGroups[s.groupNum].startIdx = idx
|
||||
// openParenGroups = append(openParenGroups, s.groupNum)
|
||||
}
|
||||
if s.groupEnd {
|
||||
closeParenGroups = append(closeParenGroups, s.groupNum)
|
||||
s.threadGroups[s.groupNum].endIdx = idx
|
||||
// closeParenGroups = append(closeParenGroups, s.groupNum)
|
||||
}
|
||||
}
|
||||
rtv = append(rtv, state.transitions[EPSILON]...)
|
||||
@@ -78,10 +84,10 @@ func takeZeroState(states []*State) (rtv []*State, isZero bool, openParenGroups
|
||||
}
|
||||
for _, state := range rtv {
|
||||
if len(state.transitions[EPSILON]) > 0 {
|
||||
return rtv, true, openParenGroups, closeParenGroups
|
||||
return rtv, true
|
||||
}
|
||||
}
|
||||
return rtv, false, openParenGroups, closeParenGroups
|
||||
return rtv, false
|
||||
}
|
||||
|
||||
// zeroMatchPossible returns true if a zero-length match is possible
|
||||
@@ -90,20 +96,16 @@ func takeZeroState(states []*State) (rtv []*State, isZero bool, openParenGroups
|
||||
// so I should probably put it in a function.
|
||||
// It also returns all the capturing groups that both begin and end at the current index.
|
||||
// This is because, by definition, zero-states don't move forward in the string.
|
||||
func zeroMatchPossible(str []rune, idx int, states ...*State) (bool, []int, []int) {
|
||||
func zeroMatchPossible(str []rune, idx int, numGroups int, states ...*State) (bool, []int, []int) {
|
||||
allOpenParenGroups := make([]int, 0)
|
||||
allCloseParenGroups := make([]int, 0)
|
||||
zeroStates, isZero, openParenGroups, closeParenGroups := takeZeroState(states)
|
||||
allOpenParenGroups = append(allOpenParenGroups, openParenGroups...)
|
||||
allCloseParenGroups = append(allCloseParenGroups, closeParenGroups...)
|
||||
zeroStates, isZero := takeZeroState(states, numGroups, idx)
|
||||
tempstates := make([]*State, 0, len(zeroStates)+len(states))
|
||||
tempstates = append(tempstates, states...)
|
||||
tempstates = append(tempstates, zeroStates...)
|
||||
num_appended := 0 // number of unique states addded to tempstates
|
||||
for isZero == true {
|
||||
zeroStates, isZero, openParenGroups, closeParenGroups = takeZeroState(tempstates)
|
||||
allOpenParenGroups = append(allOpenParenGroups, openParenGroups...)
|
||||
allCloseParenGroups = append(allCloseParenGroups, closeParenGroups...)
|
||||
zeroStates, isZero = takeZeroState(tempstates, numGroups, idx)
|
||||
tempstates, num_appended = unique_append(tempstates, zeroStates...)
|
||||
if num_appended == 0 { // break if we haven't appended any more unique values
|
||||
break
|
||||
@@ -180,7 +182,7 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
if start.groupBegin {
|
||||
to_return[start.groupNum].startIdx = offset
|
||||
}
|
||||
if ok, openGrps, closeGrps := zeroMatchPossible(str, offset, start); ok {
|
||||
if ok, openGrps, closeGrps := zeroMatchPossible(str, offset, numGroups, start); ok {
|
||||
for _, gIdx := range openGrps {
|
||||
to_return[gIdx].startIdx = offset
|
||||
}
|
||||
@@ -227,9 +229,11 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
i++ // Advance to next character (if we aren't at a 0-state, which doesn't match anything), so that we can check for transitions. If we advance at a 0-state, we will never get a chance to match the first character
|
||||
}
|
||||
|
||||
start.threadGroups = newMatch(numGroups + 1)
|
||||
// Check if the start state begins a group - if so, add the start index to our list
|
||||
if start.groupBegin {
|
||||
tempIndices[start.groupNum].startIdx = i
|
||||
start.threadGroups[start.groupNum].startIdx = i
|
||||
// tempIndices[start.groupNum].startIdx = i
|
||||
}
|
||||
|
||||
currentStates = append(currentStates, start)
|
||||
@@ -241,23 +245,11 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
zeroStates := make([]*State, 0)
|
||||
// Keep taking zero-states, until there are no more left to take
|
||||
// Objective: If any of our current states have transitions to 0-states, replace them with the 0-state. Do this until there are no more transitions to 0-states, or there are no more unique 0-states to take.
|
||||
zeroStates, isZero, openParenGroups, closeParenGroups := takeZeroState(currentStates)
|
||||
for _, val := range openParenGroups {
|
||||
tempIndices[val].startIdx = i
|
||||
}
|
||||
for _, val := range closeParenGroups {
|
||||
tempIndices[val].endIdx = i
|
||||
}
|
||||
zeroStates, isZero := takeZeroState(currentStates, numGroups, i)
|
||||
tempStates = append(tempStates, zeroStates...)
|
||||
num_appended := 0
|
||||
for isZero == true {
|
||||
zeroStates, isZero, openParenGroups, closeParenGroups = takeZeroState(tempStates)
|
||||
for _, val := range openParenGroups {
|
||||
tempIndices[val].startIdx = i
|
||||
}
|
||||
for _, val := range closeParenGroups {
|
||||
tempIndices[val].endIdx = i
|
||||
}
|
||||
zeroStates, isZero = takeZeroState(tempStates, numGroups, i)
|
||||
tempStates, num_appended = unique_append(tempStates, zeroStates...)
|
||||
if num_appended == 0 { // Break if we haven't appended any more unique values
|
||||
break
|
||||
@@ -271,6 +263,7 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
numStatesMatched := 0 // The number of states which had at least 1 match for this round
|
||||
assertionFailed := false // Whether or not an assertion failed for this round
|
||||
lastStateInList := false // Whether or not a last state was in our list of states
|
||||
var lastStatePtr *State = nil // Pointer to the last-state, if it was found
|
||||
lastLookaroundInList := false // Whether or not a last state (that is a lookaround) was in our list of states
|
||||
for _, state := range currentStates {
|
||||
matches, numMatches := state.matchesFor(str, i)
|
||||
@@ -278,6 +271,12 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
numStatesMatched++
|
||||
tempStates = append(tempStates, matches...)
|
||||
foundPath = true
|
||||
for _, m := range matches {
|
||||
if m.threadGroups == nil {
|
||||
m.threadGroups = newMatch(numGroups + 1)
|
||||
}
|
||||
copy(m.threadGroups, state.threadGroups)
|
||||
}
|
||||
}
|
||||
if numMatches < 0 {
|
||||
assertionFailed = true
|
||||
@@ -287,6 +286,7 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
lastLookaroundInList = true
|
||||
}
|
||||
lastStateInList = true
|
||||
lastStatePtr = state
|
||||
}
|
||||
}
|
||||
|
||||
@@ -309,14 +309,27 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
return false, []Group{}, i
|
||||
}
|
||||
}
|
||||
// Check if we can find a state in our list that is:
|
||||
// a. A last-state
|
||||
// b. Empty
|
||||
// c. Doesn't assert anything
|
||||
for _, s := range currentStates {
|
||||
if s.isLast && s.isEmpty && s.assert == NONE {
|
||||
lastStatePtr = s
|
||||
lastStateInList = true
|
||||
}
|
||||
}
|
||||
if lastStateInList { // A last-state was in the list of states. add the matchIndex to our MatchIndex list
|
||||
for j := 1; j < numGroups+1; j++ {
|
||||
tempIndices[j] = lastStatePtr.threadGroups[j]
|
||||
}
|
||||
endIdx = i
|
||||
tempIndices[0] = Group{startIdx, endIdx}
|
||||
}
|
||||
|
||||
// Check if we can find a zero-length match
|
||||
if foundPath == false {
|
||||
if ok, _, _ := zeroMatchPossible(str, i, currentStates...); ok {
|
||||
if ok, _, _ := zeroMatchPossible(str, i, numGroups, currentStates...); ok {
|
||||
if tempIndices[0].isValid() == false {
|
||||
tempIndices[0] = Group{startIdx, startIdx}
|
||||
}
|
||||
@@ -345,23 +358,11 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
|
||||
// End-of-string reached. Go to any 0-states, until there are no more 0-states to go to. Then check if any of our states are in the end position.
|
||||
// This is the exact same algorithm used inside the loop, so I should probably put it in a function.
|
||||
zeroStates, isZero, openParenGroups, closeParenGroups := takeZeroState(currentStates)
|
||||
for _, val := range openParenGroups {
|
||||
tempIndices[val].startIdx = i
|
||||
}
|
||||
for _, val := range closeParenGroups {
|
||||
tempIndices[val].endIdx = i
|
||||
}
|
||||
zeroStates, isZero := takeZeroState(currentStates, numGroups, i)
|
||||
tempStates = append(tempStates, zeroStates...)
|
||||
num_appended := 0 // Number of unique states addded to tempStates
|
||||
for isZero == true {
|
||||
zeroStates, isZero, openParenGroups, closeParenGroups = takeZeroState(tempStates)
|
||||
for _, val := range openParenGroups {
|
||||
tempIndices[val].startIdx = i
|
||||
}
|
||||
for _, val := range closeParenGroups {
|
||||
tempIndices[val].endIdx = i
|
||||
}
|
||||
zeroStates, isZero = takeZeroState(tempStates, numGroups, i)
|
||||
tempStates, num_appended = unique_append(tempStates, zeroStates...)
|
||||
if num_appended == 0 { // Break if we haven't appended any more unique values
|
||||
break
|
||||
@@ -376,6 +377,9 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) (
|
||||
// make sure the assertion checks out.
|
||||
if state.isLast && startIdx < len(str) {
|
||||
if state.assert == NONE || state.checkAssertion(str, i) {
|
||||
for j := 1; j < numGroups+1; j++ {
|
||||
tempIndices[j] = state.threadGroups[j]
|
||||
}
|
||||
endIdx = i
|
||||
tempIndices[0] = Group{startIdx, endIdx}
|
||||
}
|
||||
|
3
nfa.go
3
nfa.go
@@ -33,10 +33,11 @@ type State struct {
|
||||
except []rune // Only valid if allChars is true - match all characters _except_ the ones in this block. Useful for inverting character classes.
|
||||
lookaroundRegex string // Only for lookaround states - Contents of the regex that the lookaround state holds
|
||||
lookaroundNFA *State // Holds the NFA of the lookaroundRegex - if it exists
|
||||
lookaroundNumCaptureGroups int // Number of capturing groups if current node is a lookaround
|
||||
lookaroundNumCaptureGroups int // Number of capturing groups in lookaround regex if current node is a lookaround
|
||||
groupBegin bool // Whether or not the node starts a capturing group
|
||||
groupEnd bool // Whether or not the node ends a capturing group
|
||||
groupNum int // Which capturing group the node starts / ends
|
||||
threadGroups []Group // Assuming that a state is part of a 'thread' in the matching process, this array stores the indices of capturing groups in the current thread. As matches are found for this state, its groups will be copied over.
|
||||
}
|
||||
|
||||
// Clones the NFA starting from the given state.
|
||||
|
Reference in New Issue
Block a user