Provide distinctBy(keySelector) method for collections and sequences.

Implement distinct() and toMutableSet() for sequences.
Breaking change: distinct() now returns List instead of Set.

#KT-5834 Fixed
#KT-6063 Fixed

libraries/stdlib/src/generated/_Sets.kt

@@ -11,73 +11,269 @@ import java.util.*
 import java.util.Collections // TODO: it's temporary while we have java.util.Collections in js
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun <T> Array<out T>.distinct(): Set<T> {
-    return this.toMutableSet()
+public fun <T> Array<out T>.distinct(): List<T> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun BooleanArray.distinct(): Set<Boolean> {
-    return this.toMutableSet()
+public fun BooleanArray.distinct(): List<Boolean> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun ByteArray.distinct(): Set<Byte> {
-    return this.toMutableSet()
+public fun ByteArray.distinct(): List<Byte> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun CharArray.distinct(): Set<Char> {
-    return this.toMutableSet()
+public fun CharArray.distinct(): List<Char> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun DoubleArray.distinct(): Set<Double> {
-    return this.toMutableSet()
+public fun DoubleArray.distinct(): List<Double> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun FloatArray.distinct(): Set<Float> {
-    return this.toMutableSet()
+public fun FloatArray.distinct(): List<Float> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun IntArray.distinct(): Set<Int> {
-    return this.toMutableSet()
+public fun IntArray.distinct(): List<Int> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun LongArray.distinct(): Set<Long> {
-    return this.toMutableSet()
+public fun LongArray.distinct(): List<Long> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun ShortArray.distinct(): Set<Short> {
-    return this.toMutableSet()
+public fun ShortArray.distinct(): List<Short> {
+    return this.toMutableSet().toList()
 }
 
 /**
- * Returns a set containing all distinct elements from the given collection.
+ * Returns a list containing only distinct elements from the given collection.
+ * The elements in the resulting list are in the same order as they were in the source collection.
  */
-public fun <T> Iterable<T>.distinct(): Set<T> {
-    return this.toMutableSet()
+public fun <T> Iterable<T>.distinct(): List<T> {
+    return this.toMutableSet().toList()
+}
+
+/**
+ * Returns a sequence containing only distinct elements from the given sequence.
+ * The elements in the resulting sequence are in the same order as they were in the source sequence.
+ */
+public fun <T> Sequence<T>.distinct(): Sequence<T> {
+    return this.distinctBy { it }
+}
+
+
+deprecated("Migrate to using Sequence<T> and respective functions")
+/**
+ * Returns a stream containing only distinct elements from the given stream.
+ * The elements in the resulting stream are in the same order as they were in the source stream.
+ */
+public fun <T> Stream<T>.distinct(): Stream<T> {
+    return this.distinctBy { it }
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <T, K> Array<out T>.distinctBy(keySelector: (T) -> K): List<T> {
+    val set = HashSet<K>()
+    val list = ArrayList<T>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> BooleanArray.distinctBy(keySelector: (Boolean) -> K): List<Boolean> {
+    val set = HashSet<K>()
+    val list = ArrayList<Boolean>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> ByteArray.distinctBy(keySelector: (Byte) -> K): List<Byte> {
+    val set = HashSet<K>()
+    val list = ArrayList<Byte>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> CharArray.distinctBy(keySelector: (Char) -> K): List<Char> {
+    val set = HashSet<K>()
+    val list = ArrayList<Char>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> DoubleArray.distinctBy(keySelector: (Double) -> K): List<Double> {
+    val set = HashSet<K>()
+    val list = ArrayList<Double>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> FloatArray.distinctBy(keySelector: (Float) -> K): List<Float> {
+    val set = HashSet<K>()
+    val list = ArrayList<Float>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> IntArray.distinctBy(keySelector: (Int) -> K): List<Int> {
+    val set = HashSet<K>()
+    val list = ArrayList<Int>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> LongArray.distinctBy(keySelector: (Long) -> K): List<Long> {
+    val set = HashSet<K>()
+    val list = ArrayList<Long>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <K> ShortArray.distinctBy(keySelector: (Short) -> K): List<Short> {
+    val set = HashSet<K>()
+    val list = ArrayList<Short>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a list containing only distinct elements from the given collection according to the [keySelector].
+ * The elements in the resulting list are in the same order as they were in the source collection.
+ */
+public inline fun <T, K> Iterable<T>.distinctBy(keySelector: (T) -> K): List<T> {
+    val set = HashSet<K>()
+    val list = ArrayList<T>()
+    for (e in this) {
+        val key = keySelector(e)
+        if (set.add(key))
+            list.add(e)
+    }
+    return list
+}
+
+/**
+ * Returns a sequence containing only distinct elements from the given sequence according to the [keySelector].
+ * The elements in the resulting sequence are in the same order as they were in the source sequence.
+ */
+public fun <T, K> Sequence<T>.distinctBy(keySelector: (T) -> K): Sequence<T> {
+    return DistinctSequence(this, keySelector)
+}
+
+
+deprecated("Migrate to using Sequence<T> and respective functions")
+/**
+ * Returns a stream containing only distinct elements from the given stream according to the [keySelector].
+ * The elements in the resulting stream are in the same order as they were in the source stream.
+ */
+public fun <T, K> Stream<T>.distinctBy(keySelector: (T) -> K): Stream<T> {
+    return DistinctStream(this, keySelector)
 }
 
 /**
@@ -351,6 +547,26 @@ public fun <T> Iterable<T>.toMutableSet(): MutableSet<T> {
     }
 }
 
+/**
+ * Returns a mutable set containing all distinct elements from the given sequence.
+ */
+public fun <T> Sequence<T>.toMutableSet(): MutableSet<T> {
+    val set = LinkedHashSet<T>()
+    for (item in this) set.add(item)
+    return set
+}
+
+
+deprecated("Migrate to using Sequence<T> and respective functions")
+/**
+ * Returns a mutable set containing all distinct elements from the given stream.
+ */
+public fun <T> Stream<T>.toMutableSet(): MutableSet<T> {
+    val set = LinkedHashSet<T>()
+    for (item in this) set.add(item)
+    return set
+}
+
 /**
  * Returns a set containing all distinct elements from both collections.
  */

libraries/stdlib/src/kotlin/collections/Sequence.kt

@@ -1,7 +1,7 @@
 package kotlin
 
-import java.util.Enumeration
-import java.util.NoSuchElementException
+import java.util.*
+import kotlin.support.AbstractIterator
 
 deprecated("Use Sequence<T> instead.")
 public interface Stream<out T> {
@@ -481,6 +481,32 @@ public class DropWhileSequence<T>(private val sequence: Sequence<T>,
     }
 }
 
+deprecated("Use DistinctSequence instead and remove with streams.")
+private class DistinctStream<T, K>(private val source: Stream<T>, private val keySelector : (T) -> K) : Stream<T> by DistinctSequence<T, K>(source.toSequence(), keySelector)
+
+private class DistinctSequence<T, K>(private val source : Sequence<T>, private val keySelector : (T) -> K) : Sequence<T> {
+    override fun iterator(): Iterator<T> = DistinctIterator(source.iterator(), keySelector)
+}
+
+private class DistinctIterator<T, K>(private val source : Iterator<T>, private val keySelector : (T) -> K) : AbstractIterator<T>() {
+    private val observed = HashSet<K>()
+
+    override fun computeNext() {
+        while (source.hasNext()) {
+            val next = source.next()
+            val key = keySelector(next)
+
+            if (observed.add(key)) {
+                setNext(next)
+                return
+            }
+        }
+
+        done()
+    }
+}
+
+
 /**
  * A sequence which repeatedly calls the specified [producer] function and returns its return values, until
  * `null` is returned from [producer].

libraries/stdlib/test/collections/SequenceTest.kt

@@ -262,6 +262,16 @@ public class SequenceTest {
         assertEquals(listOf(0, 1, 0, 1, 2), data.flatten().toList())
     }
 
+    test fun distinct() {
+        val sequence = fibonacci().dropWhile { it < 10 }.take(20)
+        assertEquals(listOf(1, 2, 3, 0), sequence.map { it % 4 }.distinct().toList())
+    }
+
+    test fun distinctBy() {
+        val sequence = fibonacci().dropWhile { it < 10 }.take(20)
+        assertEquals(listOf(13, 34, 55, 144), sequence.distinctBy { it % 4 }.toList())
+    }
+
 
     /*
     test fun pairIterator() {

libraries/stdlib/test/collections/SetOperationsTest.kt

@@ -5,8 +5,13 @@ import org.junit.Test as test
 
 class SetOperationsTest {
     test fun distinct() {
-        assertEquals(listOf(1, 3, 5), listOf(1, 3, 3, 1, 5, 1, 3).distinct().toList())
-        assertTrue(listOf<Int>().distinct().none())
+        assertEquals(listOf(1, 3, 5), listOf(1, 3, 3, 1, 5, 1, 3).distinct())
+        assertTrue(listOf<Int>().distinct().isEmpty())
+    }
+
+    test fun distinctBy() {
+        assertEquals(listOf("some", "cat", "do"), arrayOf("some", "case", "cat", "do", "dog", "it").distinctBy { it.length() })
+        assertTrue(charArrayOf().distinctBy { it }.isEmpty())
     }
 
     test fun union() {

libraries/tools/kotlin-stdlib-gen/src/templates/Sets.kt

@@ -6,7 +6,7 @@ fun sets(): List<GenericFunction> {
     val templates = arrayListOf<GenericFunction>()
 
     templates add f("toMutableSet()") {
-        exclude(Strings, Sequences)
+        exclude(Strings)
         doc { "Returns a mutable set containing all distinct elements from the given collection." }
         returns("MutableSet<T>")
         body {
@@ -24,18 +24,66 @@ fun sets(): List<GenericFunction> {
             return set
             """
         }
+        doc(Sequences) { "Returns a mutable set containing all distinct elements from the given sequence." }
+        body(Sequences) {
+            """
+            val set = LinkedHashSet<T>()
+            for (item in this) set.add(item)
+            return set
+            """
+        }
     }
 
     templates add f("distinct()") {
-        exclude(Strings, Sequences)
-        doc { "Returns a set containing all distinct elements from the given collection." }
+        exclude(Strings)
+        val collectionDoc = """
+            Returns a list containing only distinct elements from the given collection.
 
-        returns("Set<T>")
+            The elements in the resulting list are in the same order as they were in the source collection.
+            """
+        doc { collectionDoc }
+
+        returns("List<T>")
+        body { "return this.toMutableSet().toList()" }
+        doc(Sequences) { collectionDoc.replace("list", "sequence").replace("collection", "sequence") }
+        returns(Sequences) { "Sequence<T>" }
+        body(Sequences) { "return this.distinctBy { it }" }
+    }
+
+    templates add f("distinctBy(keySelector: (T) -> K)") {
+        exclude(Strings)
+        val collectionDoc = """
+            Returns a list containing only distinct elements from the given collection according to the [keySelector].
+
+            The elements in the resulting list are in the same order as they were in the source collection.
+            """
+        doc { collectionDoc }
+
+        inline(true)
+        typeParam("K")
+        returns("List<T>")
         body {
             """
-            return this.toMutableSet()
+            val set = HashSet<K>()
+            val list = ArrayList<T>()
+            for (e in this) {
+                val key = keySelector(e)
+                if (set.add(key))
+                    list.add(e)
+            }
+            return list
             """
         }
+
+        inline(false, Sequences)
+        doc(Sequences) { collectionDoc.replace("list", "sequence").replace("collection", "sequence") }
+        returns(Sequences) { "Sequence<T>" }
+        body(Sequences) {
+            """
+            return DistinctSequence(this, keySelector)
+            """
+        }
+
     }
 
     templates add f("union(other: Iterable<T>)") {