[K/N and WASM] Implement named capturing group in Regexes #KT-41890

This commit is contained in:
Abduqodiri Qurbonzoda
2022-03-30 17:28:09 +03:00
committed by Space
parent c3f5d03b36
commit 9c4c1ed557
8 changed files with 280 additions and 94 deletions
@@ -66,6 +66,24 @@ public actual enum class RegexOption(override val value: Int, override val mask:
*/
public actual data class MatchGroup(actual val value: String, val range: IntRange)
/**
* Returns a named group with the specified [name].
*
* @return An instance of [MatchGroup] if the group with the specified [name] was matched or `null` otherwise.
* @throws IllegalArgumentException if there is no group with the specified [name] defined in the regex pattern.
* @throws UnsupportedOperationException if this match group collection doesn't support getting match groups by name,
* for example, when it's not supported by the current platform.
*/
@SinceKotlin("1.7")
public operator fun MatchGroupCollection.get(name: String): MatchGroup? {
val namedGroups = this as? MatchNamedGroupCollection
?: throw UnsupportedOperationException("Retrieving groups by name is not supported on this platform.")
return namedGroups[name]
}
/**
* Represents a compiled regular expression.
* Provides functions to match strings in text with a pattern, replace the found occurrences and split text around matches.
@@ -223,17 +241,17 @@ public actual class Regex internal constructor(internal val nativePattern: Patte
/**
* Replaces all occurrences of this regular expression in the specified [input] string with specified [replacement] expression.
*
* The replacement string may contain references to the captured groups during a match. Occurrences of `$index`
* in the replacement string will be substituted with the subsequences corresponding to the captured groups with the specified index.
* The first digit after '$' is always treated as part of group reference. Subsequent digits are incorporated
* The replacement string may contain references to the captured groups during a match. Occurrences of `${name}` or `$index`
* in the replacement string will be substituted with the subsequences corresponding to the captured groups with the specified name or index.
* In case of `$index`, the first digit after '$' is always treated as a part of group reference. Subsequent digits are incorporated
* into `index` only if they would form a valid group reference. Only the digits '0'..'9' are considered as potential components
* of the group reference. Note that indexes of captured groups start from 1, and the group with index 0 is the whole match.
* In case of `${name}`, the `name` can consist of latin letters 'a'..'z' and 'A'..'Z', or digits '0'..'9'. The first character must be
* a letter.
*
* Backslash character '\' can be used to include the succeeding character as a literal in the replacement string, e.g, `\$` or `\\`.
* [Regex.escapeReplacement] can be used if [replacement] have to be treated as a literal string.
*
* Note that named capturing groups are not supported in Kotlin/Native.
*
* @param input the char sequence to find matches of this regular expression in
* @param replacement the expression to replace found matches with
* @return the result of replacing each occurrence of this regular expression in [input] with the result of evaluating the [replacement] expression
@@ -271,17 +289,17 @@ public actual class Regex internal constructor(internal val nativePattern: Patte
/**
* Replaces the first occurrence of this regular expression in the specified [input] string with specified [replacement] expression.
*
* The replacement string may contain references to the captured groups during a match. Occurrences of `$index`
* in the replacement string will be substituted with the subsequences corresponding to the captured groups with the specified index.
* The first digit after '$' is always treated as part of group reference. Subsequent digits are incorporated
* The replacement string may contain references to the captured groups during a match. Occurrences of `${name}` or `$index`
* in the replacement string will be substituted with the subsequences corresponding to the captured groups with the specified name or index.
* In case of `$index`, the first digit after '$' is always treated as a part of group reference. Subsequent digits are incorporated
* into `index` only if they would form a valid group reference. Only the digits '0'..'9' are considered as potential components
* of the group reference. Note that indexes of captured groups start from 1, and the group with index 0 is the whole match.
* In case of `${name}`, the `name` can consist of latin letters 'a'..'z' and 'A'..'Z', or digits '0'..'9'. The first character must be
* a letter.
*
* Backslash character '\' can be used to include the succeeding character as a literal in the replacement string, e.g, `\$` or `\\`.
* [Regex.escapeReplacement] can be used if [replacement] have to be treated as a literal string.
*
* Note that named capturing groups are not supported in Kotlin/Native.
*
* @param input the char sequence to find a match of this regular expression in
* @param replacement the expression to replace the found match with
* @return the result of replacing the first occurrence of this regular expression in [input] with the result of evaluating the [replacement] expression
@@ -388,20 +406,34 @@ private fun substituteGroupRefs(match: MatchResult, replacement: String): String
if (index == replacement.length)
throw IllegalArgumentException("Capturing group index is missing")
if (replacement[index] == '{')
throw IllegalArgumentException("Named capturing group reference currently is not supported")
if (replacement[index] == '{') {
val endIndex = replacement.readGroupName(++index)
val groupName = replacement.substring(index, endIndex)
if (replacement[index] !in '0'..'9')
throw IllegalArgumentException("Invalid capturing group reference")
if (groupName.isEmpty())
throw IllegalArgumentException("Named capturing group reference should have a non-empty name")
if (groupName[0] in '0'..'9')
throw IllegalArgumentException("Named capturing group reference {$groupName} should start with a letter")
val endIndex = replacement.readGroupIndex(index, match.groupValues.size)
val groupIndex = replacement.substring(index, endIndex).toInt()
if (endIndex == replacement.length || replacement[endIndex] != '}')
throw IllegalArgumentException("Named capturing group reference is missing trailing '}'")
if (groupIndex >= match.groupValues.size)
throw IndexOutOfBoundsException("Group with index $groupIndex does not exist")
result.append(match.groups[groupName]?.value ?: "")
index = endIndex + 1 // skip past '}'
} else {
if (replacement[index] !in '0'..'9')
throw IllegalArgumentException("Invalid capturing group reference")
result.append(match.groupValues[groupIndex])
index = endIndex
val groups = match.groups
val endIndex = replacement.readGroupIndex(index, groups.size)
val groupIndex = replacement.substring(index, endIndex).toInt()
if (groupIndex >= groups.size)
throw IndexOutOfBoundsException("Group with index $groupIndex does not exist")
result.append(groups[groupIndex]?.value ?: "")
index = endIndex
}
} else {
result.append(char)
}
@@ -409,6 +441,19 @@ private fun substituteGroupRefs(match: MatchResult, replacement: String): String
return result.toString()
}
private fun String.readGroupName(startIndex: Int): Int {
var index = startIndex
while (index < length) {
val char = this[index]
if (char in 'a'..'z' || char in 'A'..'Z' || char in '0'..'9') {
index++
} else {
break
}
}
return index
}
private fun String.readGroupIndex(startIndex: Int, groupCount: Int): Int {
// at least one digit after '$' is always captured
var index = startIndex + 1
@@ -33,12 +33,14 @@ internal abstract class SpecialToken {
* Returns the type of the token, may return following values:
* TOK_CHARCLASS - token representing character class;
* TOK_QUANTIFIER - token representing quantifier;
* TOK_NAMED_GROUP - token representing named capturing group;
*/
abstract val type: Type
enum class Type {
CHARCLASS,
QUANTIFIER
QUANTIFIER,
NAMED_GROUP
}
}
@@ -333,52 +335,58 @@ internal class Lexer(val patternString: String, flags: Int) {
// Group
lookAhead = CHAR_LEFT_PARENTHESIS
} else {
// Special constructs (non-capturing groups, look ahead/look behind etc).
// Special constructs (non-capturing groups, named capturing groups, look ahead/look behind etc).
nextIndex()
var char = pattern[index]
var isLookBehind = false
do {
if (!isLookBehind) {
when (char) {
// Look ahead or an atomic group.
'!' -> { lookAhead = CHAR_NEG_LOOKAHEAD; nextIndex() }
'=' -> { lookAhead = CHAR_POS_LOOKAHEAD; nextIndex() }
'>' -> { lookAhead = CHAR_ATOMIC_GROUP; nextIndex() }
// Positive / negaitve look behind - need to check the next char.
'<' -> {
nextIndex()
char = pattern[index]
isLookBehind = true
}
// Flags.
else -> {
lookAhead = readFlags()
// We return `res = res or 1 shl 8` from readFlags() if we read (?idmsux-idmsux)
if (lookAhead >= 256) {
// Just flags (no non-capturing group with them). Erase auxiliary bit.
lookAhead = lookAhead and 0xff
flags = lookAhead
lookAhead = lookAhead shl 16
lookAhead = CHAR_FLAGS or lookAhead
} else {
// A non-capturing group with flags: (?<flags>:Foo)
flags = lookAhead
lookAhead = lookAhead shl 16
lookAhead = CHAR_NONCAP_GROUP or lookAhead
}
}
}
} else {
when (char) {
// Look ahead or an atomic group.
'!' -> {
lookAhead = CHAR_NEG_LOOKAHEAD; nextIndex()
}
'=' -> {
lookAhead = CHAR_POS_LOOKAHEAD; nextIndex()
}
'>' -> {
lookAhead = CHAR_ATOMIC_GROUP; nextIndex()
}
// named capturing group or positive / negative look behind - need to check the next char.
'<' -> {
nextIndex()
char = pattern[index]
// Process the second char for look behind construction.
isLookBehind = false
when (char) {
'!' -> { lookAhead = CHAR_NEG_LOOKBEHIND; nextIndex() }
'=' -> { lookAhead = CHAR_POS_LOOKBEHIND; nextIndex() }
else -> throw PatternSyntaxException("Unknown look behind", patternString, curTokenIndex)
'!' -> {
lookAhead = CHAR_NEG_LOOKBEHIND; nextIndex()
}
'=' -> {
lookAhead = CHAR_POS_LOOKBEHIND; nextIndex()
}
else -> {
val name = readGroupName()
lookAhead = CHAR_NAMED_GROUP
lookAheadSpecialToken = NamedGroup(name)
}
}
}
} while (isLookBehind)
// Flags.
else -> {
lookAhead = readFlags()
// We return `res = res or 1 shl 8` from readFlags() if we read (?idmsux-idmsux)
if (lookAhead >= 256) {
// Just flags (no non-capturing group with them). Erase auxiliary bit.
lookAhead = lookAhead and 0xff
flags = lookAhead
lookAhead = lookAhead shl 16
lookAhead = CHAR_FLAGS or lookAhead
} else {
// A non-capturing group with flags: (?<flags>:Foo)
flags = lookAhead
lookAhead = lookAhead shl 16
lookAhead = CHAR_NONCAP_GROUP or lookAhead
}
}
}
}
}
@@ -459,11 +467,21 @@ internal class Lexer(val patternString: String, flags: Int) {
'e' -> lookAhead = '\u001B'.toInt()
// Back references to capturing groups.
// \n
'1', '2', '3', '4', '5', '6', '7', '8', '9' -> {
if (mode == Mode.PATTERN) {
lookAhead = 0x80000000.toInt() or lookAhead // Captured group reference is 0x80...<group number>
}
}
// \k<name>
'k' -> {
if (pattern[nextIndex()] != '<') {
throw PatternSyntaxException("Invalid syntax for named group back reference", patternString, curTokenIndex)
}
val name = readGroupName()
lookAhead = CHAR_NAMED_GROUP_REF
lookAheadSpecialToken = NamedGroup(name)
}
// A literal: octal, hex, or hex unicode.
'0' -> lookAhead = readOctals()
@@ -489,7 +507,7 @@ internal class Lexer(val patternString: String, flags: Int) {
}
}
'C', 'E', 'F', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'T', 'U', 'X', 'Y', 'g', 'i', 'j', 'k', 'l', 'm', 'o', 'q', 'y' ->
'C', 'E', 'F', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'T', 'U', 'X', 'Y', 'g', 'i', 'j', 'l', 'm', 'o', 'q', 'y' ->
throw PatternSyntaxException("Illegal escape sequence", patternString, curTokenIndex)
}
return false
@@ -680,6 +698,24 @@ internal class Lexer(val patternString: String, flags: Int) {
return result
}
private fun readGroupName(): String {
var char = pattern[nextIndex()]
if (char !in 'a'..'z' && char !in 'A'..'Z') {
throw PatternSyntaxException("Capturing group name should start with a letter", patternString, curTokenIndex)
}
val sb = StringBuilder()
do {
sb.append(char)
char = pattern[nextIndex()]
} while (char in 'a'..'z' || char in 'A'..'Z' || char in '0'..'9')
if (char != '>') {
throw PatternSyntaxException("Invalid group name syntax", patternString, curTokenIndex)
}
return sb.toString()
}
companion object {
// Special characters.
val CHAR_DOLLAR = 0xe0000000.toInt() or '$'.toInt()
@@ -692,6 +728,7 @@ internal class Lexer(val patternString: String, flags: Int) {
val CHAR_HYPHEN = 0xe0000000.toInt() or '-'.toInt()
val CHAR_DOT = 0xe0000000.toInt() or '.'.toInt()
val CHAR_LEFT_PARENTHESIS = 0x80000000.toInt() or '('.toInt()
val CHAR_NAMED_GROUP = 0x90000000.toInt() or '('.toInt()
val CHAR_NONCAP_GROUP = 0xc0000000.toInt() or '('.toInt()
val CHAR_POS_LOOKAHEAD = 0xe0000000.toInt() or '('.toInt()
val CHAR_NEG_LOOKAHEAD = 0xf0000000.toInt() or '('.toInt()
@@ -703,6 +740,7 @@ internal class Lexer(val patternString: String, flags: Int) {
val CHAR_WORD_BOUND = 0x80000000.toInt() or 'b'.toInt()
val CHAR_NONWORD_BOUND = 0x80000000.toInt() or 'B'.toInt()
val CHAR_PREVIOUS_MATCH = 0x80000000.toInt() or 'G'.toInt()
val CHAR_NAMED_GROUP_REF = 0x80000000.toInt() or 'k'.toInt()
val CHAR_END_OF_INPUT = 0x80000000.toInt() or 'z'.toInt()
val CHAR_END_OF_LINE = 0x80000000.toInt() or 'Z'.toInt()
val CHAR_LINEBREAK = 0x80000000.toInt() or 'R'.toInt()
@@ -30,7 +30,6 @@ internal class MatchResultImpl
/**
* @param input an input sequence for matching/searching.
* @param regex a [Regex] instance used for matching/searching.
* @param rightBound index in the [input] used as a right bound for matching/searching. Exclusive.
*/
constructor (internal val input: CharSequence,
internal val regex: Regex) : MatchResult {
@@ -92,7 +91,7 @@ constructor (internal val input: CharSequence,
* Groups are indexed from 1 to `groupCount` and group with the index 0 corresponds to the entire match.
*/
// Create one object or several ones?
override val groups: MatchGroupCollection = object: MatchGroupCollection, AbstractCollection<MatchGroup?>() {
override val groups: MatchGroupCollection = object: MatchNamedGroupCollection, AbstractCollection<MatchGroup?>() {
override val size: Int
get() = this@MatchResultImpl.groupCount
@@ -118,6 +117,12 @@ constructor (internal val input: CharSequence,
val value = group(index) ?: return null
return MatchGroup(value, getStart(index) until getEnd(index))
}
override fun get(name: String): MatchGroup? {
val index = nativePattern.groupNameToIndex[name]
?: throw IllegalArgumentException("Capturing group with name {$name} does not exist")
return get(index)
}
}
/**
@@ -0,0 +1,13 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin.text.regex
internal class NamedGroup(val name: String) : SpecialToken() {
override fun toString() = "NamedGroup(name=$name)"
override val type: Type = SpecialToken.Type.NAMED_GROUP
}
@@ -32,9 +32,12 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
/** A lexer instance used to get tokens from the pattern. */
private val lexemes = Lexer(pattern, flags)
/* All back references that may be used in pattern. */
/** All back references that may be used in pattern. */
private val backRefs = arrayOfNulls<FSet>(BACK_REF_NUMBER)
/** Mapping from group name to its index */
val groupNameToIndex = hashMapOf<String, Int>()
/** Is true if back referenced sets replacement by second compilation pass is needed.*/
private var needsBackRefReplacement = false
@@ -129,10 +132,18 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
if (capturingGroupCount < BACK_REF_NUMBER) {
backRefs[capturingGroupCount] = fSet
}
if (ch == Lexer.CHAR_NAMED_GROUP) {
val name = (lexemes.curSpecialToken as NamedGroup).name
groupNameToIndex[name] = capturingGroupCount
}
capturingGroupCount++
}
}
if (last != null) {
lexemes.next()
}
//Process to EOF or ')'
do {
val child: AbstractSet
@@ -466,7 +477,6 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
}
// The terminal is some kind of group: (E). Call processExpression for it.
if (char and 0x8000ffff.toInt() == Lexer.CHAR_LEFT_PARENTHESIS) {
lexemes.next()
var newFlags = flags
if (char and 0xff00ffff.toInt() == Lexer.CHAR_NONCAP_GROUP) {
newFlags = (char shr 16) and flagsBitMask
@@ -569,16 +579,16 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
0x80000000.toInt() or '7'.toInt(),
0x80000000.toInt() or '8'.toInt(),
0x80000000.toInt() or '9'.toInt() -> {
val number = (char and 0x7FFFFFFF) - '0'.toInt()
if (number < capturingGroupCount) { // All is ok - the group exists.
lexemes.next()
term = BackReferenceSet(number, consumersCount++, hasFlag(CASE_INSENSITIVE))
// backRefs[number] is proved to be not null because the group is already created (number < capturingGroupCount)
backRefs[number]!!.isBackReferenced = true
needsBackRefReplacement = true // And process back references in the second pass.
} else {
throw PatternSyntaxException("No such group yet exists at this point in the pattern", pattern, lexemes.curTokenIndex)
}
val groupIndex = (char and 0x7FFFFFFF) - '0'.toInt()
term = createBackReference(groupIndex)
lexemes.next()
}
Lexer.CHAR_NAMED_GROUP_REF -> {
val name = (lexemes.curSpecialToken as NamedGroup).name
val groupIndex = groupNameToIndex[name] ?: -1
term = createBackReference(groupIndex)
lexemes.next()
}
// A special token (\D, \w etc), 'u0000' or the end of the pattern.
@@ -624,6 +634,17 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
return term
}
/** Creates a back reference to the group with specified [groupIndex], or throws if the group doesn't exist yet. */
private fun createBackReference(groupIndex: Int): BackReferenceSet {
if (groupIndex >= 0 && groupIndex < capturingGroupCount) { // All is ok - the group exists.
// backRefs[groupIndex] is proved to be not null because the group is already created (groupIndex < capturingGroupCount)
backRefs[groupIndex]!!.isBackReferenced = true
needsBackRefReplacement = true // And process back references in the second pass.
return BackReferenceSet(groupIndex, consumersCount++, hasFlag(CASE_INSENSITIVE))
} else {
throw PatternSyntaxException("No such group yet exists at this point in the pattern", pattern, lexemes.curTokenIndex)
}
}
/**
* Process [...] ranges
+80 -15
View File
@@ -156,20 +156,19 @@ class RegexTest {
}
@Test fun matchNamedGroups() {
if (!supportsNamedCapturingGroup) return
val regex = "\\b(?<city>[A-Za-z\\s]+),\\s(?<state>[A-Z]{2}):\\s(?<areaCode>[0-9]{3})\\b".toRegex()
val input = "Coordinates: Austin, TX: 123"
regex.find(input)!!.let { match ->
assertEquals(listOf("Austin, TX: 123", "Austin", "TX", "123"), match.groupValues)
val match = regex.find(input)!!
assertEquals(listOf("Austin, TX: 123", "Austin", "TX", "123"), match.groupValues)
if (supportsNamedCapturingGroup) {
val namedGroups = match.groups as MatchNamedGroupCollection
assertEquals(4, namedGroups.size)
assertEquals("Austin", namedGroups["city"]?.value)
assertEquals("TX", namedGroups["state"]?.value)
assertEquals("123", namedGroups["areaCode"]?.value)
}
}
val namedGroups = match.groups as MatchNamedGroupCollection
assertEquals(4, namedGroups.size)
assertEquals("Austin", namedGroups["city"]?.value)
assertEquals("TX", namedGroups["state"]?.value)
assertEquals("123", namedGroups["areaCode"]?.value)
}
@Test fun matchOptionalNamedGroup() {
@@ -208,15 +207,81 @@ class RegexTest {
}
@Test fun matchWithBackReference() {
val regex = "(?<title>\\w+), yes \\k<title>".toRegex()
"(\\w+), yes \\1".toRegex().let { regex ->
val match = regex.find("Do you copy? Sir, yes Sir!")!!
assertEquals("Sir, yes Sir", match.value)
assertEquals("Sir", match.groups[1]?.value)
val match = regex.find("Do you copy? Sir, yes Sir!")!!
assertEquals("Sir, yes Sir", match.value)
assertEquals("Sir", (match.groups as MatchNamedGroupCollection)["title"]?.value)
assertNull(regex.find("Do you copy? Sir, yes I do!"))
}
assertNull(regex.find("Do you copy? Sir, yes I do!"))
// capture the largest valid group index
"(\\w+), yes \\12".toRegex().let { regex ->
val match = regex.find("Do you copy? Sir, yes Sir2")!!
assertEquals("Sir, yes Sir2", match.value)
assertEquals("Sir", match.groups[1]?.value)
}
// back reference to non-existent group
assertNull("a(a)\\21".toRegex().find("aaaa1"))
// back reference to a group with large index
"0(1(2(3(4(5(6(7(8(9(A(B(C))))))))\\11))))".toRegex().let { regex ->
val match = regex.find("0123456789ABCBC")!!
assertEquals("BC", match.groups[11]?.value)
assertEquals("56789ABC", match.groups[5]?.value)
assertEquals("456789ABCBC", match.groups[4]?.value)
}
// back reference to an enclosing group
assertNull("a(a\\1)".toRegex().find("aaaa"))
// back reference to not yet available group
assertNull("a\\1(a)".toRegex().find("aaaa"))
}
@Test fun matchNamedGroupsWithBackReference() {
if (!supportsNamedCapturingGroup) return
"(?<title>\\w+), yes \\k<title>".toRegex().let { regex ->
val match = regex.find("Do you copy? Sir, yes Sir!")!!
assertEquals("Sir, yes Sir", match.value)
assertEquals("Sir", (match.groups as MatchNamedGroupCollection)["title"]?.value)
assertNull(regex.find("Do you copy? Sir, yes I do!"))
}
// back reference to an enclosing group
assertNull("a(?<first>a\\k<first>)".toRegex().find("aaaa"))
// back reference to not yet available group
assertFailsWith<IllegalArgumentException> {
"a\\k<first>(?<first>a)".toRegex()
}
}
@Test fun incompleteNamedGroupDeclaration() {
if (!supportsNamedCapturingGroup) return
assertFailsWith<IllegalArgumentException> {
"(?<".toRegex()
}
assertFailsWith<IllegalArgumentException> {
"(?<)".toRegex()
}
assertFailsWith<IllegalArgumentException> {
"(?<name".toRegex()
}
assertFailsWith<IllegalArgumentException> {
"(?<name)".toRegex()
}
assertFailsWith<IllegalArgumentException> {
"(?<name>".toRegex()
}
assertFailsWith<IllegalArgumentException> {
"(?<>\\w+), yes \\k<>".toRegex()
}
}
// TODO: Test comment mode enabled and group name is separated by space, (before, in the middle, after)
@Test fun matchMultiline() {
val regex = "^[a-z]*$".toRegex(setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE))
val matchedValues = regex.findAll("test\n\nLine").map { it.value }.toList()
+1 -2
View File
@@ -28,7 +28,6 @@ public actual val isFloat32RangeEnforced: Boolean = false
actual val supportsSuppressedExceptions: Boolean get() = true
// TODO: implement named group reference in replacement expression
public actual val supportsNamedCapturingGroup: Boolean get() = false
public actual val supportsNamedCapturingGroup: Boolean get() = true
public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true