From 437ca2ee578820a2d4d9dd50134009eedb214708 Mon Sep 17 00:00:00 2001 From: Aadhavan Srinivasan Date: Wed, 11 Dec 2024 00:16:24 -0500 Subject: [PATCH] Improved submatch tracking by storing all group indices as a part of the state, which is viewed as a 'thread' --- matching.go | 90 ++++++++++++++++++++++++++++------------------------- nfa.go | 3 +- 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/matching.go b/matching.go index 4f0ae21..9a18987 100644 --- a/matching.go +++ b/matching.go @@ -62,15 +62,21 @@ func (g Group) isValid() bool { // the second ret val is true. // The third ret val is a list of all the group numbers of all the opening parentheses we crossed, // and the fourth is a list of all the closing parentheses we passed -func takeZeroState(states []*State) (rtv []*State, isZero bool, openParenGroups []int, closeParenGroups []int) { +func takeZeroState(states []*State, numGroups int, idx int) (rtv []*State, isZero bool) { for _, state := range states { if len(state.transitions[EPSILON]) > 0 { for _, s := range state.transitions[EPSILON] { + if s.threadGroups == nil { + s.threadGroups = newMatch(numGroups + 1) + } + copy(s.threadGroups, state.threadGroups) if s.groupBegin { - openParenGroups = append(openParenGroups, s.groupNum) + s.threadGroups[s.groupNum].startIdx = idx + // openParenGroups = append(openParenGroups, s.groupNum) } if s.groupEnd { - closeParenGroups = append(closeParenGroups, s.groupNum) + s.threadGroups[s.groupNum].endIdx = idx + // closeParenGroups = append(closeParenGroups, s.groupNum) } } rtv = append(rtv, state.transitions[EPSILON]...) @@ -78,10 +84,10 @@ func takeZeroState(states []*State) (rtv []*State, isZero bool, openParenGroups } for _, state := range rtv { if len(state.transitions[EPSILON]) > 0 { - return rtv, true, openParenGroups, closeParenGroups + return rtv, true } } - return rtv, false, openParenGroups, closeParenGroups + return rtv, false } // zeroMatchPossible returns true if a zero-length match is possible @@ -90,20 +96,16 @@ func takeZeroState(states []*State) (rtv []*State, isZero bool, openParenGroups // so I should probably put it in a function. // It also returns all the capturing groups that both begin and end at the current index. // This is because, by definition, zero-states don't move forward in the string. -func zeroMatchPossible(str []rune, idx int, states ...*State) (bool, []int, []int) { +func zeroMatchPossible(str []rune, idx int, numGroups int, states ...*State) (bool, []int, []int) { allOpenParenGroups := make([]int, 0) allCloseParenGroups := make([]int, 0) - zeroStates, isZero, openParenGroups, closeParenGroups := takeZeroState(states) - allOpenParenGroups = append(allOpenParenGroups, openParenGroups...) - allCloseParenGroups = append(allCloseParenGroups, closeParenGroups...) + zeroStates, isZero := takeZeroState(states, numGroups, idx) tempstates := make([]*State, 0, len(zeroStates)+len(states)) tempstates = append(tempstates, states...) tempstates = append(tempstates, zeroStates...) num_appended := 0 // number of unique states addded to tempstates for isZero == true { - zeroStates, isZero, openParenGroups, closeParenGroups = takeZeroState(tempstates) - allOpenParenGroups = append(allOpenParenGroups, openParenGroups...) - allCloseParenGroups = append(allCloseParenGroups, closeParenGroups...) + zeroStates, isZero = takeZeroState(tempstates, numGroups, idx) tempstates, num_appended = unique_append(tempstates, zeroStates...) if num_appended == 0 { // break if we haven't appended any more unique values break @@ -180,7 +182,7 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( if start.groupBegin { to_return[start.groupNum].startIdx = offset } - if ok, openGrps, closeGrps := zeroMatchPossible(str, offset, start); ok { + if ok, openGrps, closeGrps := zeroMatchPossible(str, offset, numGroups, start); ok { for _, gIdx := range openGrps { to_return[gIdx].startIdx = offset } @@ -227,9 +229,11 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( i++ // Advance to next character (if we aren't at a 0-state, which doesn't match anything), so that we can check for transitions. If we advance at a 0-state, we will never get a chance to match the first character } + start.threadGroups = newMatch(numGroups + 1) // Check if the start state begins a group - if so, add the start index to our list if start.groupBegin { - tempIndices[start.groupNum].startIdx = i + start.threadGroups[start.groupNum].startIdx = i + // tempIndices[start.groupNum].startIdx = i } currentStates = append(currentStates, start) @@ -241,23 +245,11 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( zeroStates := make([]*State, 0) // Keep taking zero-states, until there are no more left to take // Objective: If any of our current states have transitions to 0-states, replace them with the 0-state. Do this until there are no more transitions to 0-states, or there are no more unique 0-states to take. - zeroStates, isZero, openParenGroups, closeParenGroups := takeZeroState(currentStates) - for _, val := range openParenGroups { - tempIndices[val].startIdx = i - } - for _, val := range closeParenGroups { - tempIndices[val].endIdx = i - } + zeroStates, isZero := takeZeroState(currentStates, numGroups, i) tempStates = append(tempStates, zeroStates...) num_appended := 0 for isZero == true { - zeroStates, isZero, openParenGroups, closeParenGroups = takeZeroState(tempStates) - for _, val := range openParenGroups { - tempIndices[val].startIdx = i - } - for _, val := range closeParenGroups { - tempIndices[val].endIdx = i - } + zeroStates, isZero = takeZeroState(tempStates, numGroups, i) tempStates, num_appended = unique_append(tempStates, zeroStates...) if num_appended == 0 { // Break if we haven't appended any more unique values break @@ -271,6 +263,7 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( numStatesMatched := 0 // The number of states which had at least 1 match for this round assertionFailed := false // Whether or not an assertion failed for this round lastStateInList := false // Whether or not a last state was in our list of states + var lastStatePtr *State = nil // Pointer to the last-state, if it was found lastLookaroundInList := false // Whether or not a last state (that is a lookaround) was in our list of states for _, state := range currentStates { matches, numMatches := state.matchesFor(str, i) @@ -278,6 +271,12 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( numStatesMatched++ tempStates = append(tempStates, matches...) foundPath = true + for _, m := range matches { + if m.threadGroups == nil { + m.threadGroups = newMatch(numGroups + 1) + } + copy(m.threadGroups, state.threadGroups) + } } if numMatches < 0 { assertionFailed = true @@ -287,6 +286,7 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( lastLookaroundInList = true } lastStateInList = true + lastStatePtr = state } } @@ -309,14 +309,27 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( return false, []Group{}, i } } + // Check if we can find a state in our list that is: + // a. A last-state + // b. Empty + // c. Doesn't assert anything + for _, s := range currentStates { + if s.isLast && s.isEmpty && s.assert == NONE { + lastStatePtr = s + lastStateInList = true + } + } if lastStateInList { // A last-state was in the list of states. add the matchIndex to our MatchIndex list + for j := 1; j < numGroups+1; j++ { + tempIndices[j] = lastStatePtr.threadGroups[j] + } endIdx = i tempIndices[0] = Group{startIdx, endIdx} } // Check if we can find a zero-length match if foundPath == false { - if ok, _, _ := zeroMatchPossible(str, i, currentStates...); ok { + if ok, _, _ := zeroMatchPossible(str, i, numGroups, currentStates...); ok { if tempIndices[0].isValid() == false { tempIndices[0] = Group{startIdx, startIdx} } @@ -345,23 +358,11 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( // End-of-string reached. Go to any 0-states, until there are no more 0-states to go to. Then check if any of our states are in the end position. // This is the exact same algorithm used inside the loop, so I should probably put it in a function. - zeroStates, isZero, openParenGroups, closeParenGroups := takeZeroState(currentStates) - for _, val := range openParenGroups { - tempIndices[val].startIdx = i - } - for _, val := range closeParenGroups { - tempIndices[val].endIdx = i - } + zeroStates, isZero := takeZeroState(currentStates, numGroups, i) tempStates = append(tempStates, zeroStates...) num_appended := 0 // Number of unique states addded to tempStates for isZero == true { - zeroStates, isZero, openParenGroups, closeParenGroups = takeZeroState(tempStates) - for _, val := range openParenGroups { - tempIndices[val].startIdx = i - } - for _, val := range closeParenGroups { - tempIndices[val].endIdx = i - } + zeroStates, isZero = takeZeroState(tempStates, numGroups, i) tempStates, num_appended = unique_append(tempStates, zeroStates...) if num_appended == 0 { // Break if we haven't appended any more unique values break @@ -376,6 +377,9 @@ func findAllMatchesHelper(start *State, str []rune, offset int, numGroups int) ( // make sure the assertion checks out. if state.isLast && startIdx < len(str) { if state.assert == NONE || state.checkAssertion(str, i) { + for j := 1; j < numGroups+1; j++ { + tempIndices[j] = state.threadGroups[j] + } endIdx = i tempIndices[0] = Group{startIdx, endIdx} } diff --git a/nfa.go b/nfa.go index a7ef67b..4ba6141 100644 --- a/nfa.go +++ b/nfa.go @@ -33,10 +33,11 @@ type State struct { except []rune // Only valid if allChars is true - match all characters _except_ the ones in this block. Useful for inverting character classes. lookaroundRegex string // Only for lookaround states - Contents of the regex that the lookaround state holds lookaroundNFA *State // Holds the NFA of the lookaroundRegex - if it exists - lookaroundNumCaptureGroups int // Number of capturing groups if current node is a lookaround + lookaroundNumCaptureGroups int // Number of capturing groups in lookaround regex if current node is a lookaround groupBegin bool // Whether or not the node starts a capturing group groupEnd bool // Whether or not the node ends a capturing group groupNum int // Which capturing group the node starts / ends + threadGroups []Group // Assuming that a state is part of a 'thread' in the matching process, this array stores the indices of capturing groups in the current thread. As matches are found for this state, its groups will be copied over. } // Clones the NFA starting from the given state.