[Commonizer] Implement associative commonization

^KT-47301 Verification Pending
2021-06-11 15:19:35 +02:00
parent 5fdbcb3dd1
commit 42f60d981f
58 changed files with 957 additions and 1091 deletions
@@ -26,7 +26,7 @@ public class CliCommonizer(private val executor: Executor) : Commonizer {
        konanHome: File,
        inputLibraries: Set<File>,
        dependencyLibraries: Set<CommonizerDependency>,
-        outputCommonizerTarget: SharedCommonizerTarget,
+        outputTargets: Set<SharedCommonizerTarget>,
        outputDirectory: File,
        logLevel: CommonizerLogLevel
    ) {
@@ -35,7 +35,7 @@ public class CliCommonizer(private val executor: Executor) : Commonizer {
            add("native-klib-commonize")
            add("-distribution-path"); add(konanHome.absolutePath)
            add("-input-libraries"); add(inputLibraries.joinToString(";") { it.absolutePath })
-            add("-output-commonizer-target"); add(outputCommonizerTarget.identityString)
+            add("-output-targets"); add(outputTargets.joinToString(";") { it.identityString })
            add("-output-path"); add(outputDirectory.absolutePath)
            if (dependencyLibraries.isNotEmpty()) {
                add("-dependency-libraries"); add(dependencyLibraries.joinToString(";"))
@@ -44,6 +44,23 @@ public class CliCommonizer(private val executor: Executor) : Commonizer {
        }
        executor(arguments)
    }
+
+    override fun commonizeNativeDistribution(
+        konanHome: File,
+        outputDirectory: File,
+        outputTargets: Set<SharedCommonizerTarget>,
+        logLevel: CommonizerLogLevel
+    ) {
+        val arguments = mutableListOf<String>().apply {
+            add("native-dist-commonize")
+            add("-distribution-path"); add(konanHome.absolutePath)
+            add("-output-path"); add(outputDirectory.absolutePath)
+            add("-output-targets"); add(outputTargets.joinToString(";") { it.identityString })
+            add("-log-level"); add(logLevel.name.lowercase())
+        }
+
+        executor(arguments)
+    }
 }

 private class CommonizerClassLoaderExecutor(private val commonizerClassLoader: ClassLoader) : CliCommonizer.Executor {
@@ -9,13 +9,22 @@ import java.io.File
 import java.io.Serializable

 public interface Commonizer : Serializable {
+
    @Throws(Throwable::class)
    public fun commonizeLibraries(
        konanHome: File,
        inputLibraries: Set<File>,
        dependencyLibraries: Set<CommonizerDependency>,
-        outputCommonizerTarget: SharedCommonizerTarget,
+        outputTargets: Set<SharedCommonizerTarget>,
        outputDirectory: File,
        logLevel: CommonizerLogLevel = CommonizerLogLevel.Quiet
    )
+
+    @Throws(Throwable::class)
+    public fun commonizeNativeDistribution(
+        konanHome: File,
+        outputDirectory: File,
+        outputTargets: Set<SharedCommonizerTarget>,
+        logLevel: CommonizerLogLevel = CommonizerLogLevel.Quiet
+    )
 }
@@ -7,14 +7,13 @@

 package org.jetbrains.kotlin.commonizer

-import org.jetbrains.kotlin.commonizer.util.transitiveClosure
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import java.io.Serializable

 // N.B. TargetPlatform/SimplePlatform are non exhaustive enough to address both target platforms such as
 // JVM, JS and concrete Kotlin/Native targets, e.g. macos_x64, ios_x64, linux_x64.
 public sealed class CommonizerTarget : Serializable {
-    final override fun toString(): String = prettyName
+    final override fun toString(): String = identityString
 }

 public data class LeafCommonizerTarget public constructor(val name: String) : CommonizerTarget() {
@@ -25,16 +24,10 @@ public data class LeafCommonizerTarget public constructor(val name: String) : Co
    public val konanTarget: KonanTarget get() = konanTargetOrNull ?: error("Unknown KonanTarget: $name")
 }

-public data class SharedCommonizerTarget(val targets: Set<CommonizerTarget>) : CommonizerTarget() {
-    public constructor(vararg targets: CommonizerTarget) : this(targets.toSet())
+public data class SharedCommonizerTarget(val targets: Set<LeafCommonizerTarget>) : CommonizerTarget() {
+    public constructor(vararg targets: LeafCommonizerTarget) : this(targets.toSet())
    public constructor(vararg targets: KonanTarget) : this(targets.toSet())
    public constructor(targets: Iterable<KonanTarget>) : this(targets.map(::LeafCommonizerTarget).toSet())
-
-    public companion object {
-        public fun ifNotEmpty(targets: Set<CommonizerTarget>): SharedCommonizerTarget? {
-            return if (targets.isNotEmpty()) SharedCommonizerTarget(targets) else null
-        }
-    }
 }

 public fun CommonizerTarget(konanTargets: Iterable<KonanTarget>): CommonizerTarget {
@@ -56,8 +49,11 @@ public fun CommonizerTarget(konanTarget: KonanTarget, vararg konanTargets: Konan
    return SharedCommonizerTarget(targets.map(::LeafCommonizerTarget).toSet())
 }

-public fun CommonizerTarget(commonizerTarget: CommonizerTarget, vararg commonizerTargets: CommonizerTarget): SharedCommonizerTarget {
-    val targets = mutableListOf<CommonizerTarget>().apply {
+public fun CommonizerTarget(
+    commonizerTarget: LeafCommonizerTarget,
+    vararg commonizerTargets: LeafCommonizerTarget
+): SharedCommonizerTarget {
+    val targets = mutableListOf<LeafCommonizerTarget>().apply {
        add(commonizerTarget)
        addAll(commonizerTargets)
    }
@@ -78,22 +74,6 @@ private val SharedCommonizerTarget.identityString: String
        )
    }

-public val CommonizerTarget.prettyName: String
-    get() = when (this) {
-        is LeafCommonizerTarget -> "[$name]"
-        is SharedCommonizerTarget -> prettyName(null)
-    }
-
-public fun SharedCommonizerTarget.prettyName(highlightedChild: CommonizerTarget?): String {
-    return targets
-        .sortedWith(compareBy<CommonizerTarget> { it.level }.thenBy { it.identityString }).joinToString(", ", "[", "]") { child ->
-            when (child) {
-                is LeafCommonizerTarget -> child.name
-                is SharedCommonizerTarget -> child.prettyName(highlightedChild)
-            } + if (child == highlightedChild) "(*)" else ""
-        }
-}
-
 public val CommonizerTarget.konanTargets: Set<KonanTarget>
    get() {
        return when (this) {
@@ -102,6 +82,9 @@ public val CommonizerTarget.konanTargets: Set<KonanTarget>
        }
    }

+public val Iterable<CommonizerTarget>.konanTargets: Set<KonanTarget> get() = flatMapTo(mutableSetOf()) { it.konanTargets }
+
+// REMOVE
 public val CommonizerTarget.level: Int
    get() {
        return when (this) {
@@ -110,30 +93,25 @@ public val CommonizerTarget.level: Int
        }
    }

-public fun CommonizerTarget.withAllAncestors(): Set<CommonizerTarget> {
-    return setOf(this) + transitiveClosure(this) {
-        when (this) {
-            is SharedCommonizerTarget -> targets
-            is LeafCommonizerTarget -> emptyList()
-        }
-    }
-}

 public fun CommonizerTarget.allLeaves(): Set<LeafCommonizerTarget> {
-    return withAllAncestors().filterIsInstance<LeafCommonizerTarget>().toSet()
-}
-
-public infix fun CommonizerTarget.isAncestorOf(other: CommonizerTarget): Boolean {
-    if (this is SharedCommonizerTarget) {
-        return targets.any { it == other } || targets.any { it.isAncestorOf(other) }
+    return when (this) {
+        is LeafCommonizerTarget -> setOf(this)
+        is SharedCommonizerTarget -> this.targets
    }
-    return false
 }

-public infix fun CommonizerTarget.isEqualOrAncestorOf(other: CommonizerTarget): Boolean {
-    return this == other || this.isAncestorOf(other)
+public fun Iterable<CommonizerTarget>.allLeaves(): Set<LeafCommonizerTarget> {
+    return flatMapTo(mutableSetOf()) { target -> target.allLeaves() }
 }

-public infix fun CommonizerTarget.isDescendentOf(other: CommonizerTarget): Boolean {
-    return other.isAncestorOf(this)
+public fun CommonizerTarget.withAllLeaves(): Set<CommonizerTarget> {
+    return when (this) {
+        is LeafCommonizerTarget -> setOf(this)
+        is SharedCommonizerTarget -> setOf(this) + targets
+    }
 }
+
+public fun Iterable<CommonizerTarget>.withAllLeaves(): Set<CommonizerTarget> {
+    return flatMapTo(mutableSetOf()) { it.withAllLeaves() }
+}
@@ -5,30 +5,14 @@

 package org.jetbrains.kotlin.commonizer

-import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR
-import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR
 import java.io.File
 import java.security.MessageDigest
 import java.util.*

-
-public fun interface CommonizerOutputLayout {
-    public fun getTargetDirectory(root: File, target: CommonizerTarget): File
-}
-
-public object NativeDistributionCommonizerOutputLayout : CommonizerOutputLayout {
-    override fun getTargetDirectory(root: File, target: CommonizerTarget): File {
-        return when (target) {
-            is LeafCommonizerTarget -> root.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(target.name)
-            is SharedCommonizerTarget -> root.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
-        }
-    }
-}
-
-public object HierarchicalCommonizerOutputLayout : CommonizerOutputLayout {
+public object CommonizerOutputFileLayout {
    internal const val maxFileNameLength = 150

-    override fun getTargetDirectory(root: File, target: CommonizerTarget): File {
+    public fun getCommonizedDirectory(root: File, target: CommonizerTarget): File {
        return root.resolve(target.fileName)
    }

@@ -42,11 +26,18 @@ public object HierarchicalCommonizerOutputLayout : CommonizerOutputLayout {
            }
        }

+    public val Set<CommonizerTarget>.fileName: String
+        get() = this.joinToString(";") { it.identityString }.base64Hash
+
+
    private val CommonizerTarget.identityStringHash: String
+        get() = identityString.base64Hash
+
+    private val String.base64Hash: String
        get() {
            val sha = MessageDigest.getInstance("SHA-1")
            val base64 = Base64.getUrlEncoder()
-            return base64.encode(sha.digest(identityString.encodeToByteArray())).decodeToString()
+            return base64.encode(sha.digest(this.encodeToByteArray())).decodeToString()
        }
 }

@@ -192,8 +192,9 @@ private sealed class IdentityStringSyntaxNode {
 private fun buildCommonizerTarget(node: IdentityStringSyntaxNode): CommonizerTarget {
    return when (node) {
        is LeafTargetSyntaxNode -> LeafCommonizerTarget(node.token.value)
+        // Previous nested ((a, b), c) notation is still valid and will be flattened to (a, b, c)
        is SharedTargetSyntaxNode -> SharedCommonizerTarget(
-            node.children.map { child -> buildCommonizerTarget(child) }.toSet()
+            node.children.flatMap { child -> buildCommonizerTarget(child).allLeaves() }.toSet()
        )
    }
 }