From b65c477e68d2e4e3b635d63b7cdc61f41829d472 Mon Sep 17 00:00:00 2001 From: Abduqodiri Qurbonzoda Date: Mon, 19 Jul 2021 00:06:13 +0300 Subject: [PATCH] Regex.splitToSequence, CharSequence.splitToSequence(Regex) #KT-23351 --- .../src/main/kotlin/kotlin/text/Regex.kt | 37 ++++++++++- libraries/stdlib/api/js-v1/kotlin.text.kt | 9 +++ libraries/stdlib/api/js/kotlin.text.kt | 9 +++ libraries/stdlib/common/src/kotlin/TextH.kt | 12 +++- libraries/stdlib/js/src/kotlin/text/regex.kt | 37 ++++++++++- .../stdlib/jvm/src/kotlin/text/StringsJVM.kt | 2 +- .../stdlib/jvm/src/kotlin/text/regex/Regex.kt | 37 ++++++++++- libraries/stdlib/src/kotlin/text/Strings.kt | 22 +++++-- libraries/stdlib/test/text/RegexTest.kt | 61 ++++++++++++++++--- libraries/stdlib/wasm/src/kotlin/Text.kt | 10 ++- .../kotlin-stdlib-runtime-merged.txt | 2 + 11 files changed, 216 insertions(+), 22 deletions(-) diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/text/Regex.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/text/Regex.kt index 0ffc7c2d0d8..0c95bac2554 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/text/Regex.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/text/Regex.kt @@ -305,14 +305,14 @@ public actual class Regex internal constructor(internal val nativePattern: Patte } /** - * Splits the [input] CharSequence around matches of this regular expression. + * Splits the [input] CharSequence to a list of strings around matches of this regular expression. * * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. * Zero by default means no limit is set. */ @Suppress("ACTUAL_FUNCTION_WITH_DEFAULT_ARGUMENTS") actual fun split(input: CharSequence, limit: Int = 0): List { - require(limit >= 0, { "Limit must be non-negative, but was $limit." } ) + requireNonNegativeLimit(limit) var match: MatchResult? = find(input) @@ -334,6 +334,39 @@ public actual class Regex internal constructor(internal val nativePattern: Patte return result } + /** + * Splits the [input] CharSequence to a sequence of strings around matches of this regular expression. + * + * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. + * Zero by default means no limit is set. + */ + @SinceKotlin("1.5") + @ExperimentalStdlibApi + @Suppress("ACTUAL_FUNCTION_WITH_DEFAULT_ARGUMENTS") + public actual fun splitToSequence(input: CharSequence, limit: Int = 0): Sequence { + requireNonNegativeLimit(limit) + + return sequence { + var match = find(input) + if (match == null || limit == 1) { + yield(input.toString()) + return@sequence + } + + var nextStart = 0 + var splitCount = 0 + + do { + val foundMatch = match!! + yield(input.substring(nextStart, foundMatch.range.first)) + nextStart = foundMatch.range.endInclusive + 1 + match = foundMatch.next() + } while (++splitCount != limit - 1 && match != null) + + yield(input.substring(nextStart, input.length)) + } + } + /** * Returns the string representation of this regular expression, namely the [pattern] of this regular expression. * diff --git a/libraries/stdlib/api/js-v1/kotlin.text.kt b/libraries/stdlib/api/js-v1/kotlin.text.kt index 2c14239d628..6ea4dd0b7ad 100644 --- a/libraries/stdlib/api/js-v1/kotlin.text.kt +++ b/libraries/stdlib/api/js-v1/kotlin.text.kt @@ -834,6 +834,11 @@ public fun kotlin.CharSequence.splitToSequence(vararg delimiters: kotlin.String, public fun kotlin.CharSequence.splitToSequence(vararg delimiters: kotlin.Char, ignoreCase: kotlin.Boolean = ..., limit: kotlin.Int = ...): kotlin.sequences.Sequence +@kotlin.SinceKotlin(version = "1.5") +@kotlin.ExperimentalStdlibApi +@kotlin.internal.InlineOnly +public inline fun kotlin.CharSequence.splitToSequence(regex: kotlin.text.Regex, limit: kotlin.Int = ...): kotlin.sequences.Sequence + public fun kotlin.CharSequence.startsWith(char: kotlin.Char, ignoreCase: kotlin.Boolean = ...): kotlin.Boolean public fun kotlin.CharSequence.startsWith(prefix: kotlin.CharSequence, ignoreCase: kotlin.Boolean = ...): kotlin.Boolean @@ -1429,6 +1434,10 @@ public final class Regex { public final fun split(input: kotlin.CharSequence, limit: kotlin.Int = ...): kotlin.collections.List + @kotlin.SinceKotlin(version = "1.5") + @kotlin.ExperimentalStdlibApi + public final fun splitToSequence(input: kotlin.CharSequence, limit: kotlin.Int = ...): kotlin.sequences.Sequence + public open override fun toString(): kotlin.String public companion object of Regex { diff --git a/libraries/stdlib/api/js/kotlin.text.kt b/libraries/stdlib/api/js/kotlin.text.kt index 2c14239d628..6ea4dd0b7ad 100644 --- a/libraries/stdlib/api/js/kotlin.text.kt +++ b/libraries/stdlib/api/js/kotlin.text.kt @@ -834,6 +834,11 @@ public fun kotlin.CharSequence.splitToSequence(vararg delimiters: kotlin.String, public fun kotlin.CharSequence.splitToSequence(vararg delimiters: kotlin.Char, ignoreCase: kotlin.Boolean = ..., limit: kotlin.Int = ...): kotlin.sequences.Sequence +@kotlin.SinceKotlin(version = "1.5") +@kotlin.ExperimentalStdlibApi +@kotlin.internal.InlineOnly +public inline fun kotlin.CharSequence.splitToSequence(regex: kotlin.text.Regex, limit: kotlin.Int = ...): kotlin.sequences.Sequence + public fun kotlin.CharSequence.startsWith(char: kotlin.Char, ignoreCase: kotlin.Boolean = ...): kotlin.Boolean public fun kotlin.CharSequence.startsWith(prefix: kotlin.CharSequence, ignoreCase: kotlin.Boolean = ...): kotlin.Boolean @@ -1429,6 +1434,10 @@ public final class Regex { public final fun split(input: kotlin.CharSequence, limit: kotlin.Int = ...): kotlin.collections.List + @kotlin.SinceKotlin(version = "1.5") + @kotlin.ExperimentalStdlibApi + public final fun splitToSequence(input: kotlin.CharSequence, limit: kotlin.Int = ...): kotlin.sequences.Sequence + public open override fun toString(): kotlin.String public companion object of Regex { diff --git a/libraries/stdlib/common/src/kotlin/TextH.kt b/libraries/stdlib/common/src/kotlin/TextH.kt index bfd27b48e00..d6f14fb477b 100644 --- a/libraries/stdlib/common/src/kotlin/TextH.kt +++ b/libraries/stdlib/common/src/kotlin/TextH.kt @@ -65,13 +65,23 @@ expect class Regex { fun findAll(input: CharSequence, startIndex: Int = 0): Sequence /** - * Splits the [input] CharSequence around matches of this regular expression. + * Splits the [input] CharSequence to a list of strings around matches of this regular expression. * * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. * Zero by default means no limit is set. */ fun split(input: CharSequence, limit: Int = 0): List + /** + * Splits the [input] CharSequence to a sequence of strings around matches of this regular expression. + * + * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. + * Zero by default means no limit is set. + */ + @SinceKotlin("1.5") + @ExperimentalStdlibApi + public fun splitToSequence(input: CharSequence, limit: Int = 0): Sequence + companion object { fun fromLiteral(literal: String): Regex fun escape(literal: String): String diff --git a/libraries/stdlib/js/src/kotlin/text/regex.kt b/libraries/stdlib/js/src/kotlin/text/regex.kt index d1a2e10e7b3..8b3978b14b3 100644 --- a/libraries/stdlib/js/src/kotlin/text/regex.kt +++ b/libraries/stdlib/js/src/kotlin/text/regex.kt @@ -186,14 +186,14 @@ public actual class Regex actual constructor(pattern: String, options: Set { - require(limit >= 0) { "Limit must be non-negative, but was $limit" } + requireNonNegativeLimit(limit) val matches = findAll(input).let { if (limit == 0) it else it.take(limit - 1) } val result = mutableListOf() var lastStart = 0 @@ -206,6 +206,39 @@ public actual class Regex actual constructor(pattern: String, options: Set { + requireNonNegativeLimit(limit) + + return sequence { + var match = find(input) + if (match == null || limit == 1) { + yield(input.toString()) + return@sequence + } + + var nextStart = 0 + var splitCount = 0 + + do { + val foundMatch = match!! + yield(input.substring(nextStart, foundMatch.range.first)) + nextStart = foundMatch.range.endInclusive + 1 + match = foundMatch.next() + } while (++splitCount != limit - 1 && match != null) + + yield(input.substring(nextStart, input.length)) + } + } + /** * Returns the string representation of this regular expression, namely the [pattern] of this regular expression. diff --git a/libraries/stdlib/jvm/src/kotlin/text/StringsJVM.kt b/libraries/stdlib/jvm/src/kotlin/text/StringsJVM.kt index 72cb0b13c5a..173a4a5f84a 100644 --- a/libraries/stdlib/jvm/src/kotlin/text/StringsJVM.kt +++ b/libraries/stdlib/jvm/src/kotlin/text/StringsJVM.kt @@ -387,7 +387,7 @@ public inline fun String.Companion.format(locale: Locale?, format: String, varar * Zero by default means no limit is set. */ public fun CharSequence.split(regex: Pattern, limit: Int = 0): List { - require(limit >= 0, { "Limit must be non-negative, but was $limit." }) + requireNonNegativeLimit(limit) return regex.split(this, if (limit == 0) -1 else limit).asList() } diff --git a/libraries/stdlib/jvm/src/kotlin/text/regex/Regex.kt b/libraries/stdlib/jvm/src/kotlin/text/regex/Regex.kt index 71feb75c494..26cc343296d 100644 --- a/libraries/stdlib/jvm/src/kotlin/text/regex/Regex.kt +++ b/libraries/stdlib/jvm/src/kotlin/text/regex/Regex.kt @@ -198,17 +198,17 @@ internal constructor(private val nativePattern: Pattern) : Serializable { /** - * Splits the [input] CharSequence around matches of this regular expression. + * Splits the [input] CharSequence to a list of strings around matches of this regular expression. * * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. * Zero by default means no limit is set. */ @Suppress("ACTUAL_FUNCTION_WITH_DEFAULT_ARGUMENTS") public actual fun split(input: CharSequence, limit: Int = 0): List { - require(limit >= 0, { "Limit must be non-negative, but was $limit." }) + requireNonNegativeLimit(limit) val matcher = nativePattern.matcher(input) - if (!matcher.find() || limit == 1) return listOf(input.toString()) + if (limit == 1 || !matcher.find()) return listOf(input.toString()) val result = ArrayList(if (limit > 0) limit.coerceAtMost(10) else 10) var lastStart = 0 @@ -225,6 +225,37 @@ internal constructor(private val nativePattern: Pattern) : Serializable { return result } + /** + * Splits the [input] CharSequence to a sequence of strings around matches of this regular expression. + * + * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. + * Zero by default means no limit is set. + */ + @SinceKotlin("1.5") + @ExperimentalStdlibApi + @Suppress("ACTUAL_FUNCTION_WITH_DEFAULT_ARGUMENTS") + public actual fun splitToSequence(input: CharSequence, limit: Int = 0): Sequence { + requireNonNegativeLimit(limit) + + return sequence { + val matcher = nativePattern.matcher(input) + if (limit == 1 || !matcher.find()) { + yield(input.toString()) + return@sequence + } + + var nextStart = 0 + var splitCount = 0 + + do { + yield(input.substring(nextStart, matcher.start())) + nextStart = matcher.end() + } while (++splitCount != limit - 1 && matcher.find()) + + yield(input.substring(nextStart, input.length)) + } + } + /** * Returns the string representation of this regular expression, namely the [pattern] of this regular expression. * diff --git a/libraries/stdlib/src/kotlin/text/Strings.kt b/libraries/stdlib/src/kotlin/text/Strings.kt index da80ca575be..e1387c26bc5 100644 --- a/libraries/stdlib/src/kotlin/text/Strings.kt +++ b/libraries/stdlib/src/kotlin/text/Strings.kt @@ -1237,7 +1237,7 @@ private class DelimitedRangesSequence( * @param limit The maximum number of substrings to return. Zero by default means no limit is set. */ private fun CharSequence.rangesDelimitedBy(delimiters: CharArray, startIndex: Int = 0, ignoreCase: Boolean = false, limit: Int = 0): Sequence { - require(limit >= 0, { "Limit must be non-negative, but was $limit." }) + requireNonNegativeLimit(limit) return DelimitedRangesSequence(this, startIndex, limit, { currentIndex -> indexOfAny(delimiters, currentIndex, ignoreCase = ignoreCase).let { if (it < 0) null else it to 1 } @@ -1260,13 +1260,16 @@ private fun CharSequence.rangesDelimitedBy(delimiters: CharArray, startIndex: In * that matches this string at that position. */ private fun CharSequence.rangesDelimitedBy(delimiters: Array, startIndex: Int = 0, ignoreCase: Boolean = false, limit: Int = 0): Sequence { - require(limit >= 0, { "Limit must be non-negative, but was $limit." } ) + requireNonNegativeLimit(limit) val delimitersList = delimiters.asList() return DelimitedRangesSequence(this, startIndex, limit, { currentIndex -> findAnyOf(delimitersList, currentIndex, ignoreCase = ignoreCase, last = false)?.let { it.first to it.second.length } }) } +internal fun requireNonNegativeLimit(limit: Int) = + require(limit >= 0) { "Limit must be non-negative, but was $limit" } + // split @@ -1340,7 +1343,7 @@ public fun CharSequence.split(vararg delimiters: Char, ignoreCase: Boolean = fal * @param limit The maximum number of substrings to return. */ private fun CharSequence.split(delimiter: String, ignoreCase: Boolean, limit: Int): List { - require(limit >= 0, { "Limit must be non-negative, but was $limit." }) + requireNonNegativeLimit(limit) var currentOffset = 0 var nextIndex = indexOf(delimiter, currentOffset, ignoreCase) @@ -1363,7 +1366,7 @@ private fun CharSequence.split(delimiter: String, ignoreCase: Boolean, limit: In } /** - * Splits this char sequence around matches of the given regular expression. + * Splits this char sequence to a list of strings around matches of the given regular expression. * * @param limit Non-negative value specifying the maximum number of substrings to return. * Zero by default means no limit is set. @@ -1371,6 +1374,17 @@ private fun CharSequence.split(delimiter: String, ignoreCase: Boolean, limit: In @kotlin.internal.InlineOnly public inline fun CharSequence.split(regex: Regex, limit: Int = 0): List = regex.split(this, limit) +/** + * Splits this char sequence to a sequence of strings around matches of the given regular expression. + * + * @param limit Non-negative value specifying the maximum number of substrings to return. + * Zero by default means no limit is set. + */ +@SinceKotlin("1.5") +@ExperimentalStdlibApi +@kotlin.internal.InlineOnly +public inline fun CharSequence.splitToSequence(regex: Regex, limit: Int = 0): Sequence = regex.splitToSequence(this, limit) + /** * Splits this char sequence to a sequence of lines delimited by any of the following character sequences: CRLF, LF or CR. * diff --git a/libraries/stdlib/test/text/RegexTest.kt b/libraries/stdlib/test/text/RegexTest.kt index a217ab8c6f1..66aacbbcd7e 100644 --- a/libraries/stdlib/test/text/RegexTest.kt +++ b/libraries/stdlib/test/text/RegexTest.kt @@ -250,6 +250,18 @@ class RegexTest { assertEquals("/2/3/4/", pattern.replace(input, { it.value.length.toString() })) } + private fun testSplitEquals(expected: List, input: CharSequence, regex: Regex, limit: Int = 0) { + assertEquals(expected, input.split(regex, limit)) + assertEquals(expected, regex.split(input, limit)) + + listOf( + input.splitToSequence(regex, limit), + regex.splitToSequence(input, limit) + ).forEach { sequence -> + assertEquals(expected, sequence.toList()) + assertEquals(expected, sequence.toList()) // assert multiple iterations over the same sequence succeed + } + } @Test fun split() { val input = """ @@ -257,9 +269,9 @@ class RegexTest { split """.trim() - assertEquals(listOf("some", "word", "split"), "\\s+".toRegex().split(input)) + testSplitEquals(listOf("some", "word", "split"), input, "\\s+".toRegex()) - assertEquals(listOf("name", "value=5"), "=".toRegex().split("name=value=5", limit = 2)) + testSplitEquals(listOf("name", "value=5"), "name=value=5", "=".toRegex(), limit = 2) } @@ -268,19 +280,52 @@ class RegexTest { val emptyMatch = "".toRegex() - assertEquals(input.split(""), input.split(emptyMatch)) - assertEquals(input.split("", limit = 3), input.split(emptyMatch, limit = 3)) + testSplitEquals(listOf("", "t", "e", "s", "t", ""), input, emptyMatch) + testSplitEquals(listOf("", "t", "est"), input, emptyMatch, limit = 3) - assertEquals("".split(""), "".split(emptyMatch)) + testSplitEquals("".split(""), "", emptyMatch) val emptyMatchBeforeT = "(?=t)".toRegex() - assertEquals(listOf("", "tes", "t"), input.split(emptyMatchBeforeT)) - assertEquals(listOf("", "test"), input.split(emptyMatchBeforeT, limit = 2)) + testSplitEquals(listOf("", "tes", "t"), input, emptyMatchBeforeT) + testSplitEquals(listOf("", "test"), input, emptyMatchBeforeT, limit = 2) - assertEquals(listOf("", "tee"), "tee".split(emptyMatchBeforeT)) + testSplitEquals(listOf("", "tee"), "tee", emptyMatchBeforeT) } + @Test fun splitByNoMatch() { + val input = "test" + val xMatch = "x".toRegex() + for (limit in 0..2) { + testSplitEquals(listOf(input), input, xMatch, limit) + } + } + + @Test fun splitWithLimitOne() { + val input = "/12/456/7890/" + val regex = "\\d+".toRegex() + + testSplitEquals(listOf(input), input, regex, limit = 1) + } + + @Test fun findAllAndSplitToSequence() { + val input = "a12bc456def7890ghij" + val regex = "\\d+".toRegex() + + val matches = regex.findAll(input).map { it.value }.iterator() + val splits = regex.splitToSequence(input).iterator() + + assertEquals("12", matches.next()) + assertEquals("a", splits.next()) + assertEquals("456", matches.next()) + assertEquals("bc", splits.next()) + assertEquals("def", splits.next()) + assertEquals("ghij", splits.next()) + assertEquals("7890", matches.next()) + + assertFailsWith { matches.next() } + assertFailsWith { splits.next() } + } } diff --git a/libraries/stdlib/wasm/src/kotlin/Text.kt b/libraries/stdlib/wasm/src/kotlin/Text.kt index dbc891de964..dd27761b0a9 100644 --- a/libraries/stdlib/wasm/src/kotlin/Text.kt +++ b/libraries/stdlib/wasm/src/kotlin/Text.kt @@ -40,13 +40,21 @@ actual class Regex { actual fun findAll(input: CharSequence, startIndex: Int): Sequence = TODO("Wasm stdlib: Text") /** - * Splits the [input] CharSequence around matches of this regular expression. + * Splits the [input] CharSequence to a list of strings around matches of this regular expression. * * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. * Zero by default means no limit is set. */ actual fun split(input: CharSequence, limit: Int): List = TODO("Wasm stdlib: Text") + /** + * Splits the [input] CharSequence to a sequence of strings around matches of this regular expression. + * + * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. + * Zero by default means no limit is set. + */ + public actual fun splitToSequence(input: CharSequence, limit: Int): Sequence = TODO("Wasm stdlib: Text") + actual companion object { actual fun fromLiteral(literal: String): Regex = TODO("Wasm stdlib: Text") actual fun escape(literal: String): String = TODO("Wasm stdlib: Text") diff --git a/libraries/tools/binary-compatibility-validator/reference-public-api/kotlin-stdlib-runtime-merged.txt b/libraries/tools/binary-compatibility-validator/reference-public-api/kotlin-stdlib-runtime-merged.txt index a60ed5bf0b0..4b2d2c17403 100644 --- a/libraries/tools/binary-compatibility-validator/reference-public-api/kotlin-stdlib-runtime-merged.txt +++ b/libraries/tools/binary-compatibility-validator/reference-public-api/kotlin-stdlib-runtime-merged.txt @@ -5210,6 +5210,8 @@ public final class kotlin/text/Regex : java/io/Serializable { public final fun replaceFirst (Ljava/lang/CharSequence;Ljava/lang/String;)Ljava/lang/String; public final fun split (Ljava/lang/CharSequence;I)Ljava/util/List; public static synthetic fun split$default (Lkotlin/text/Regex;Ljava/lang/CharSequence;IILjava/lang/Object;)Ljava/util/List; + public final fun splitToSequence (Ljava/lang/CharSequence;I)Lkotlin/sequences/Sequence; + public static synthetic fun splitToSequence$default (Lkotlin/text/Regex;Ljava/lang/CharSequence;IILjava/lang/Object;)Lkotlin/sequences/Sequence; public final fun toPattern ()Ljava/util/regex/Pattern; public fun toString ()Ljava/lang/String; }