azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize 'windowed' and 'chunked' for char sequences, iterables and sequences

Browse Source
This commit is contained in:
Ilya Gorbunov
2017-05-04 23:08:57 +03:00
parent 18c7a01ab5
commit 3f23742298
8 changed files with 236 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.libraries/src/core/generated/_CollectionsJs.kt
+34 -5
View File
@@ -1790,7 +1790,7 @@ public fun <T : Any> List<T?>.requireNoNulls(): List<T> {
@SinceKotlin("1.2")
public fun <T> Iterable<T>.chunked(size: Int): List<List<T>> {
return chunked(size) { it.toList() }
return windowed(size, size)
}
@SinceKotlin("1.2")
@@ -1988,15 +1988,44 @@ public inline fun <T> Collection<T>.plusElement(element: T): List<T> {
@SinceKotlin("1.2")
public fun <T> Iterable<T>.windowed(size: Int, step: Int): List<List<T>> {
return windowed(size, step) { it.toList() }
checkWindowSizeStep(size, step)
if (this is RandomAccess && this is List) {
val thisSize = this.size
val result = ArrayList<List<T>>((thisSize + step - 1) / step)
var index = 0
while (index < thisSize) {
result.add(List(size.coerceAtMost(thisSize - index)) { this[it + index] })
index += step
}
return result
}
val result = ArrayList<List<T>>()
windowedIterator(iterator(), size, step, dropTrailing = false, reuseBuffer = false).forEach {
result.add(it)
}
return result
}
@SinceKotlin("1.2")
public fun <T, R> Iterable<T>.windowed(size: Int, step: Int, transform: (List<T>) -> R): List<R> {
if (this is List) {
return windowIndices(this.size, size, step, dropTrailing = false).asIterable().map { transform(subList(it.start, it.endInclusive + 1)) }
checkWindowSizeStep(size, step)
if (this is RandomAccess && this is List) {
val thisSize = this.size
val result = ArrayList<R>((thisSize + step - 1) / step)
val window = MovingSubList(this)
var index = 0
while (index < thisSize) {
window.move(index, (index + size).coerceAtMost(thisSize))
result.add(transform(window))
index += step
}
return result
}
return windowForwardOnlySequenceImpl(iterator(), size, step, dropTrailing = false).asIterable().map(transform)
val result = ArrayList<R>()
windowedIterator(iterator(), size, step, dropTrailing = false, reuseBuffer = true).forEach {
result.add(transform(it))
}
return result
}
/**
js/js.libraries/src/core/generated/_SequencesJs.kt
+3 -4
View File
@@ -1311,7 +1311,7 @@ public fun <T : Any> Sequence<T?>.requireNoNulls(): Sequence<T> {
@SinceKotlin("1.2")
public fun <T> Sequence<T>.chunked(size: Int): Sequence<List<T>> {
return chunked(size) { it.toList() }
return windowed(size, size)
}
@SinceKotlin("1.2")
@@ -1497,13 +1497,12 @@ public inline fun <T> Sequence<T>.plusElement(element: T): Sequence<T> {
@SinceKotlin("1.2")
public fun <T> Sequence<T>.windowed(size: Int, step: Int): Sequence<List<T>> {
return windowed(size, step) { it.toList() }
return windowedSequence(size, step, dropTrailing = false, reuseBuffer = false)
}
@SinceKotlin("1.2")
public fun <T, R> Sequence<T>.windowed(size: Int, step: Int, transform: (List<T>) -> R): Sequence<R> {
require(size > 0 && step > 0) { "size $size and step $step both must be greater than zero" }
return Sequence { windowForwardOnlySequenceImpl(iterator(), size, step, dropTrailing = false).iterator() }.map(transform)
return windowedSequence(size, step, dropTrailing = false, reuseBuffer = true).map(transform)
}
/**
js/js.libraries/src/core/generated/_StringsJs.kt
+12 -3
View File
@@ -1104,7 +1104,7 @@ public inline fun CharSequence.sumByDouble(selector: (Char) -> Double): Double {
@SinceKotlin("1.2")
public fun CharSequence.chunked(size: Int): List<String> {
return chunked(size) { it.toString() }
return windowed(size, size)
}
@SinceKotlin("1.2")
@@ -1181,7 +1181,15 @@ public fun CharSequence.windowed(size: Int, step: Int): List<String> {
@SinceKotlin("1.2")
public fun <R> CharSequence.windowed(size: Int, step: Int, transform: (CharSequence) -> R): List<R> {
return windowIndices(this.length, size, step, dropTrailing = false).asIterable().map { transform(subSequence(it)) }
checkWindowSizeStep(size, step)
val thisSize = this.length
val result = ArrayList<R>((thisSize + step - 1) / step)
var index = 0
while (index < thisSize) {
result.add(transform(subSequence(index, (index + size).coerceAtMost(thisSize))))
index += step
}
return result
}
@SinceKotlin("1.2")
@@ -1191,7 +1199,8 @@ public fun CharSequence.windowedSequence(size: Int, step: Int): Sequence<String>
@SinceKotlin("1.2")
public fun <R> CharSequence.windowedSequence(size: Int, step: Int, transform: (CharSequence) -> R): Sequence<R> {
return windowIndices(this.length, size, step, dropTrailing = false).map { transform(subSequence(it)) }
checkWindowSizeStep(size, step)
return (indices step step).asSequence().map { index -> transform(subSequence(index, (index + size).coerceAtMost(length))) }
}
/**
libraries/stdlib/src/generated/_Collections.kt
+34 -5
View File
@@ -1800,7 +1800,7 @@ public fun <T : Any> List<T?>.requireNoNulls(): List<T> {
@SinceKotlin("1.2")
public fun <T> Iterable<T>.chunked(size: Int): List<List<T>> {
return chunked(size) { it.toList() }
return windowed(size, size)
}
@SinceKotlin("1.2")
@@ -1998,15 +1998,44 @@ public inline fun <T> Collection<T>.plusElement(element: T): List<T> {
@SinceKotlin("1.2")
public fun <T> Iterable<T>.windowed(size: Int, step: Int): List<List<T>> {
return windowed(size, step) { it.toList() }
checkWindowSizeStep(size, step)
if (this is RandomAccess && this is List) {
val thisSize = this.size
val result = ArrayList<List<T>>((thisSize + step - 1) / step)
var index = 0
while (index < thisSize) {
result.add(List(size.coerceAtMost(thisSize - index)) { this[it + index] })
index += step
}
return result
}
val result = ArrayList<List<T>>()
windowedIterator(iterator(), size, step, dropTrailing = false, reuseBuffer = false).forEach {
result.add(it)
}
return result
}
@SinceKotlin("1.2")
public fun <T, R> Iterable<T>.windowed(size: Int, step: Int, transform: (List<T>) -> R): List<R> {
if (this is List) {
return windowIndices(this.size, size, step, dropTrailing = false).asIterable().map { transform(subList(it.start, it.endInclusive + 1)) }
checkWindowSizeStep(size, step)
if (this is RandomAccess && this is List) {
val thisSize = this.size
val result = ArrayList<R>((thisSize + step - 1) / step)
val window = MovingSubList(this)
var index = 0
while (index < thisSize) {
window.move(index, (index + size).coerceAtMost(thisSize))
result.add(transform(window))
index += step
}
return result
}
return windowForwardOnlySequenceImpl(iterator(), size, step, dropTrailing = false).asIterable().map(transform)
val result = ArrayList<R>()
windowedIterator(iterator(), size, step, dropTrailing = false, reuseBuffer = true).forEach {
result.add(transform(it))
}
return result
}
/**
libraries/stdlib/src/generated/_Sequences.kt
+3 -4
View File
@@ -1333,7 +1333,7 @@ public fun <T : Any> Sequence<T?>.requireNoNulls(): Sequence<T> {
@SinceKotlin("1.2")
public fun <T> Sequence<T>.chunked(size: Int): Sequence<List<T>> {
return chunked(size) { it.toList() }
return windowed(size, size)
}
@SinceKotlin("1.2")
@@ -1519,13 +1519,12 @@ public inline fun <T> Sequence<T>.plusElement(element: T): Sequence<T> {
@SinceKotlin("1.2")
public fun <T> Sequence<T>.windowed(size: Int, step: Int): Sequence<List<T>> {
return windowed(size, step) { it.toList() }
return windowedSequence(size, step, dropTrailing = false, reuseBuffer = false)
}
@SinceKotlin("1.2")
public fun <T, R> Sequence<T>.windowed(size: Int, step: Int, transform: (List<T>) -> R): Sequence<R> {
require(size > 0 && step > 0) { "size $size and step $step both must be greater than zero" }
return Sequence { windowForwardOnlySequenceImpl(iterator(), size, step, dropTrailing = false).iterator() }.map(transform)
return windowedSequence(size, step, dropTrailing = false, reuseBuffer = true).map(transform)
}
/**
libraries/stdlib/src/generated/_Strings.kt
+12 -3
View File
@@ -1112,7 +1112,7 @@ public inline fun CharSequence.sumByDouble(selector: (Char) -> Double): Double {
@SinceKotlin("1.2")
public fun CharSequence.chunked(size: Int): List<String> {
return chunked(size) { it.toString() }
return windowed(size, size)
}
@SinceKotlin("1.2")
@@ -1189,7 +1189,15 @@ public fun CharSequence.windowed(size: Int, step: Int): List<String> {
@SinceKotlin("1.2")
public fun <R> CharSequence.windowed(size: Int, step: Int, transform: (CharSequence) -> R): List<R> {
return windowIndices(this.length, size, step, dropTrailing = false).asIterable().map { transform(subSequence(it)) }
checkWindowSizeStep(size, step)
val thisSize = this.length
val result = ArrayList<R>((thisSize + step - 1) / step)
var index = 0
while (index < thisSize) {
result.add(transform(subSequence(index, (index + size).coerceAtMost(thisSize))))
index += step
}
return result
}
@SinceKotlin("1.2")
@@ -1199,7 +1207,8 @@ public fun CharSequence.windowedSequence(size: Int, step: Int): Sequence<String>
@SinceKotlin("1.2")
public fun <R> CharSequence.windowedSequence(size: Int, step: Int, transform: (CharSequence) -> R): Sequence<R> {
return windowIndices(this.length, size, step, dropTrailing = false).map { transform(subSequence(it)) }
checkWindowSizeStep(size, step)
return (indices step step).asSequence().map { index -> transform(subSequence(index, (index + size).coerceAtMost(length))) }
}
/**
libraries/stdlib/src/kotlin/collections/SlidingWindow.kt
+76 -143
View File
@@ -16,119 +16,90 @@
package kotlin.collections
import kotlin.coroutines.experimental.buildIterator
internal fun windowIndices(sourceSize: Int, size: Int, step: Int, dropTrailing: Boolean): Sequence<IntRange> {
require(size > 0 && step > 0) { "size $size and step $step both must be greater than zero" }
if (sourceSize == 0 || (size > sourceSize && dropTrailing)) {
return emptySequence()
}
if (size == 0) {
return when {
step > 0 -> (0 .. sourceSize - 1 step step)
else -> (sourceSize - 1 downTo 0 step -step)
}.asSequence().map { it .. it - 1 } // empty ranges with valid start
}
var currentIndex = when {
step > 0 -> 0
else -> sourceSize - size
}
return generateSequence {
val startIndex = currentIndex
val endExclusive = currentIndex + size
when {
startIndex >= sourceSize -> null
endExclusive > sourceSize && dropTrailing -> null
startIndex < 0 && dropTrailing -> null
step < 0 && endExclusive <= 0 -> null
else -> {
currentIndex += step
startIndex.coerceAtLeast(0) .. endExclusive.coerceAtMost(sourceSize) - 1
}
}
internal fun checkWindowSizeStep(size: Int, step: Int) {
require(size > 0 && step > 0) {
if (size != step)
"Both size $size and step $step must be greater than zero."
else
"size $size must be greater than zero."
}
}
internal fun <T> windowForwardOnlySequenceImpl(iterator: Iterator<T>, size: Int, step: Int, dropTrailing: Boolean): Sequence<List<T>> {
require(size > 0 && step > 0) { "size $size and step $step both must be greater than zero" }
return if (step >= size) {
windowForwardWithGap(iterator, size, step, dropTrailing)
} else {
windowForwardWithOverlap(iterator, size, step, dropTrailing)
}
internal fun <T> Sequence<T>.windowedSequence(size: Int, step: Int, dropTrailing: Boolean, reuseBuffer: Boolean): Sequence<List<T>> {
checkWindowSizeStep(size, step)
return Sequence { windowedIterator(iterator(), size, step, dropTrailing, reuseBuffer) }
}
private fun <T> windowForwardWithGap(iterator: Iterator<T>, size: Int, step: Int, dropTrailing: Boolean): Sequence<List<T>> {
require(step >= size)
var first = true
val gap = step - size
fun skipGap() {
for (skip in 1..gap) {
if (!iterator.hasNext()) {
break
internal fun <T> windowedIterator(iterator: Iterator<T>, size: Int, step: Int, dropTrailing: Boolean, reuseBuffer: Boolean): Iterator<List<T>> {
if (!iterator.hasNext()) return EmptyIterator
return buildIterator<List<T>> {
val gap = step - size
if (gap >= 0) {
var buffer = ArrayList<T>(size)
var skip = 0
for (e in iterator) {
if (skip > 0) { skip -= 1; continue }
buffer.add(e)
if (buffer.size == size) {
yield(buffer)
if (reuseBuffer) buffer.clear() else buffer = ArrayList(size)
skip = gap
}
}
if (buffer.isNotEmpty()) {
if (!dropTrailing || buffer.size == size) yield(buffer)
}
iterator.next()
}
}
return generateSequence {
if (first) {
first = false
} else {
skipGap()
}
val buffer = ArrayList<T>(size)
for (i in 1..size) {
if (!iterator.hasNext()) {
break
val buffer = RingBuffer<T>(size)
for (e in iterator) {
buffer.add(e)
if (buffer.isFull()) {
yield(if (reuseBuffer) buffer else ArrayList(buffer))
buffer.removeFirst(step)
}
}
if (dropTrailing) {
if (buffer.size == size) yield(buffer)
} else {
while (buffer.size > step) {
yield(if (reuseBuffer) buffer else ArrayList(buffer))
buffer.removeFirst(step)
}
if (buffer.isNotEmpty()) yield(buffer)
}
buffer.add(iterator.next())
}
when {
buffer.isEmpty() && !iterator.hasNext() -> null
buffer.size < size && dropTrailing -> null
else -> buffer
}
}
}
private fun <T> windowForwardWithOverlap(iterator: Iterator<T>, size: Int, step: Int, dropTrailing: Boolean): Sequence<List<T>> {
require(step < size)
internal class MovingSubList<out E>(private val list: List<E>) : AbstractList<E>(), RandomAccess {
private var fromIndex: Int = 0
private var _size: Int = 0
val buffer = RingBuffer<T>(size)
return generateSequence {
if (!buffer.isEmpty()) {
buffer.removeFirst(minOf(step, buffer.size))
}
while (!buffer.isFull() && iterator.hasNext()) {
buffer.add(iterator.next())
}
@Suppress("UNCHECKED_CAST")
when {
buffer.isEmpty() && !iterator.hasNext() -> null
!buffer.isFull() && dropTrailing -> null
else -> buffer.toArray().asList() as List<T>
}
fun move(fromIndex: Int, toIndex: Int) {
checkRangeIndexes(fromIndex, toIndex, list.size)
this.fromIndex = fromIndex
this._size = toIndex - fromIndex
}
override fun get(index: Int): E {
checkElementIndex(index, _size)
return list[fromIndex + index]
}
override val size: Int get() = _size
}
/**
* Provides ring buffer implementation.
*
* Buffer overflow is not allowed so [add] doesn't overwrite tail but raises an exception while [offer] returns `false`
* If it is going to be public API perhaps this behaviour could be customizable
*/
internal class RingBuffer<T>(val capacity: Int): Iterable<T> {
private class RingBuffer<T>(val capacity: Int): AbstractList<T>(), RandomAccess {
init {
require(capacity >= 0) { "ring buffer capacity should not be negative but it is $capacity" }
}
@@ -136,31 +107,32 @@ internal class RingBuffer<T>(val capacity: Int): Iterable<T> {
private val buffer = arrayOfNulls<Any?>(capacity)
private var writePosition = 0
var size: Int = 0
override var size: Int = 0
private set
fun isEmpty() = size == 0
override fun get(index: Int): T {
checkElementIndex(index, size)
return getAtUnsafe(writePosition.backward(size - index))
}
fun isFull() = size == capacity
override fun iterator(): Iterator<T> = when {
isEmpty() -> EmptyIterator
else -> object : AbstractIterator<T>() {
override fun iterator(): Iterator<T> = object : AbstractIterator<T>() {
private var count = size
private var idx = writePosition.backward(count)
private var index = writePosition.backward(size)
override fun computeNext() {
if (count == 0) {
done()
} else {
setNext(getAtUnsafe(idx))
idx = idx.forward()
setNext(getAtUnsafe(index))
index = index.forward(1)
count--
}
}
}
}
fun toArray(): Array<out Any?> {
override fun toArray(): Array<Any?> {
val size = this.size
val result = arrayOfNulls<Any?>(size)
var widx = 0
@@ -187,7 +159,7 @@ internal class RingBuffer<T>(val capacity: Int): Iterable<T> {
*/
fun add(element: T) {
if (!offer(element)) {
throw IllegalStateException("ring buffer is full")
throw IllegalStateException("Ring buffer is full.")
}
}
@@ -200,29 +172,11 @@ internal class RingBuffer<T>(val capacity: Int): Iterable<T> {
}
buffer[writePosition] = element
writePosition = writePosition.forward()
writePosition = writePosition.forward(1)
size++
return true
}
/**
* Takes first element from the buffer or fails with [NoSuchElementException] if the buffer is empty
*/
fun get(): T {
if (isEmpty()) {
throw NoSuchElementException("ring buffer is empty")
}
val readPosition = writePosition.backward(size)
val result = getAtUnsafe(readPosition)
buffer[readPosition] = null
size--
return result
}
/**
* Removes [n] first elements from the buffer or fails with [IllegalArgumentException] if not enough elements in the buffer to remove
*/
@@ -234,9 +188,6 @@ internal class RingBuffer<T>(val capacity: Int): Iterable<T> {
val start = writePosition.backward(size)
val end = start.forward(n - 1)
for (i in start .. end) {
buffer[i] = null
}
if (start > end) {
buffer.fill(null, start, capacity)
buffer.fill(null, 0, end + 1)
@@ -248,33 +199,15 @@ internal class RingBuffer<T>(val capacity: Int): Iterable<T> {
}
}
/**
* Removes all elements from the buffer
*/
fun clear() {
size = 0
buffer.fill(null)
}
@Suppress("NOTHING_TO_INLINE", "UNCHECKED_CAST")
private inline fun getAtUnsafe(idx: Int): T = buffer[idx] as T
@Suppress("NOTHING_TO_INLINE")
private inline fun Int.forward(n: Int = 1): Int {
require(n >= 0)
require(n <= capacity)
val result = this + n
return if (result >= capacity) result - capacity else result
}
private inline fun Int.forward(n: Int): Int = (this + n) % capacity
@Suppress("NOTHING_TO_INLINE")
private inline fun Int.backward(n: Int = 1): Int {
require(n >= 0)
require(n <= capacity)
return if (this < n) (this - n + capacity) else this - n
}
private inline fun Int.backward(n: Int): Int = ((this - n) % capacity + capacity) % capacity
// TODO: replace with Array.fill from stdlib when available in common
private fun <T> Array<T>.fill(element: T, fromIndex: Int = 0, toIndex: Int = size): Unit {
libraries/tools/kotlin-stdlib-gen/src/templates/Generators.kt
+62 -12
View File
@@ -568,25 +568,46 @@ fun generators(): List<GenericFunction> {
body {
"""
if (this is List) {
return windowIndices(this.size, size, step, dropTrailing = false).asIterable().map { transform(subList(it.start, it.endInclusive + 1)) }
checkWindowSizeStep(size, step)
if (this is RandomAccess && this is List) {
val thisSize = this.size
val result = ArrayList<R>((thisSize + step - 1) / step)
val window = MovingSubList(this)
var index = 0
while (index < thisSize) {
window.move(index, (index + size).coerceAtMost(thisSize))
result.add(transform(window))
index += step
}
return result
}
return windowForwardOnlySequenceImpl(iterator(), size, step, dropTrailing = false).asIterable().map(transform)
val result = ArrayList<R>()
windowedIterator(iterator(), size, step, dropTrailing = false, reuseBuffer = true).forEach {
result.add(transform(it))
}
return result
"""
}
customSignature(CharSequences) { "windowed(size: Int, step: Int, transform: (CharSequence) -> R)" }
body(CharSequences) {
"""
return windowIndices(this.length, size, step, dropTrailing = false).asIterable().map { transform(subSequence(it)) }
checkWindowSizeStep(size, step)
val thisSize = this.length
val result = ArrayList<R>((thisSize + step - 1) / step)
var index = 0
while (index < thisSize) {
result.add(transform(subSequence(index, (index + size).coerceAtMost(thisSize))))
index += step
}
return result
"""
}
returns(Sequences) { "Sequence<R>" }
body(Sequences) {
"""
require(size > 0 && step > 0) { "size ${"$"}size and step ${"$"}step both must be greater than zero" }
return Sequence { windowForwardOnlySequenceImpl(iterator(), size, step, dropTrailing = false).iterator() }.map(transform)
return windowedSequence(size, step, dropTrailing = false, reuseBuffer = true).map(transform)
"""
}
}
@@ -598,8 +619,33 @@ fun generators(): List<GenericFunction> {
returns(Sequences) { "Sequence<List<T>>" }
returns(CharSequences) { "List<String>" }
body { "return windowed(size, step) { it.toList() }" }
body {
"""
checkWindowSizeStep(size, step)
if (this is RandomAccess && this is List) {
val thisSize = this.size
val result = ArrayList<List<T>>((thisSize + step - 1) / step)
var index = 0
while (index < thisSize) {
result.add(List(size.coerceAtMost(thisSize - index)) { this[it + index] })
index += step
}
return result
}
val result = ArrayList<List<T>>()
windowedIterator(iterator(), size, step, dropTrailing = false, reuseBuffer = false).forEach {
result.add(it)
}
return result
"""
}
body(CharSequences) { "return windowed(size, step) { it.toString() }" }
body(Sequences) {
"""
return windowedSequence(size, step, dropTrailing = false, reuseBuffer = false)
"""
}
}
templates add f("windowedSequence(size: Int, step: Int, transform: (CharSequence) -> R)") {
@@ -608,9 +654,10 @@ fun generators(): List<GenericFunction> {
typeParam("R")
returns { "Sequence<R> "}
body(CharSequences) {
body {
"""
return windowIndices(this.length, size, step, dropTrailing = false).map { transform(subSequence(it)) }
checkWindowSizeStep(size, step)
return (indices step step).asSequence().map { index -> transform(subSequence(index, (index + size).coerceAtMost(length))) }
"""
}
}
@@ -643,8 +690,7 @@ fun generators(): List<GenericFunction> {
returns(Sequences) { "Sequence<List<T>>" }
returns(CharSequences) { "List<String>" }
body { "return chunked(size) { it.toList() }" }
body(CharSequences) { "return chunked(size) { it.toString() }" }
body { "return windowed(size, size)" }
}
templates add f("chunkedSequence(size: Int, transform: (CharSequence) -> R)") {
@@ -653,7 +699,11 @@ fun generators(): List<GenericFunction> {
typeParam("R")
returns { "Sequence<R> "}
body { "return windowedSequence(size, size, transform)" }
body {
"""
return windowedSequence(size, size, transform)
"""
}
}
templates add f("chunkedSequence(size: Int)") {