From a7b3f26cef77a3d5c0b118bb5ff2343384ef24dd Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Thu, 21 Nov 2019 18:28:47 +0300 Subject: [PATCH] [native-gradle-plugin] Supported klib cache (building and using) --- .../kotlin-gradle-plugin/build.gradle.kts | 1 + .../compilerRunner/KotlinNativeToolRunner.kt | 3 + .../kotlin/gradle/plugin/KotlinProperties.kt | 6 + .../targets/native/tasks/KotlinNativeTasks.kt | 203 ++++++++++++++++++ 4 files changed, 213 insertions(+) diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index c7fd4cbd8bc..14172c33e85 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { compile(kotlinStdlib()) compile(project(":kotlin-native:kotlin-native-utils")) + compile(project(":kotlin-util-klib")) compileOnly(project(":kotlin-reflect-api")) compileOnly(project(":kotlin-android-extensions")) compileOnly(project(":kotlin-build-common")) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeToolRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeToolRunner.kt index 013be74f5b0..5a573c169b8 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeToolRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeToolRunner.kt @@ -51,6 +51,9 @@ internal val Project.konanVersion: CompilerVersion internal val Project.disableKonanDaemon: Boolean get() = PropertiesProvider(this).nativeDisableCompilerDaemon == true +internal val Project.disableKonanCache: Boolean + get() = PropertiesProvider(this).nativeDisableCompilerCache == true + internal interface KonanToolRunner : Named { val mainClass: String val classpath: FileCollection diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt index 48b2a5f89e0..e093d6c5b47 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt @@ -146,6 +146,12 @@ internal class PropertiesProvider private constructor(private val project: Proje val nativeDisableCompilerDaemon: Boolean? get() = booleanProperty("kotlin.native.disableCompilerDaemon") + /** + * Forbids dependencies precompilation to dynamic/static libraries. + */ + val nativeDisableCompilerCache: Boolean? + get() = booleanProperty("kotlin.native.disableCompilerCache") + /** * Generate kotlin/js external declarations from all .d.ts files found in npm modules */ diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt index 40925dde83c..0c3f1a065f1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt @@ -15,6 +15,7 @@ import org.gradle.api.file.FileTree import org.gradle.api.provider.Provider import org.gradle.api.tasks.* import org.gradle.api.tasks.compile.AbstractCompile +import org.jetbrains.kotlin.compilerRunner.* import org.jetbrains.kotlin.compilerRunner.KonanCompilerRunner import org.jetbrains.kotlin.compilerRunner.KonanInteropRunner import org.jetbrains.kotlin.compilerRunner.konanHome @@ -28,7 +29,13 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.* import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.CompilerOutputKind.* import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.impl.createKotlinLibrary +import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.library.unresolvedDependencies import java.io.File +import java.nio.charset.StandardCharsets +import java.security.MessageDigest // TODO: It's just temporary tasks used while KN isn't integrated with Big Kotlin compilation infrastructure. // region Useful extensions @@ -418,6 +425,10 @@ open class KotlinNativeLink : AbstractKotlinNativeCompile = mutableListOf().apply { addAll(super.buildCompilerArgs()) + addAll(CacheBuilder(project, binary).buildCompilerArgs()) + addKey("-tr", processTests) addArgIfNotNull("-entry", entryPoint) when (embedBitcode) { @@ -605,6 +618,196 @@ open class KotlinNativeLink : AbstractKotlinNativeCompile { + val allDependencies = mutableSetOf() + + fun traverseAllDependencies(dependency: ResolvedDependency) { + if (dependency in allDependencies) + return + allDependencies.add(dependency) + dependency.children.forEach { traverseAllDependencies(it) } + } + + dependency.children.forEach { traverseAllDependencies(it) } + return allDependencies + } + + private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') } + + private fun computeDependenciesHash(dependency: ResolvedDependency): String { + val allArtifactsPaths = + (dependency.moduleArtifacts + getAllDependencies(dependency).flatMap { it.moduleArtifacts }) + .map { it.file.absolutePath } + .distinct() + .sortedBy { it } + .joinToString("|") { it } + val digest = MessageDigest.getInstance("SHA-256") + val hash = digest.digest(allArtifactsPaths.toByteArray(StandardCharsets.UTF_8)) + return hash.toHexString() + } + + private fun getCacheDirectory(dependency: ResolvedDependency): File { + val moduleCacheDirectory = File(rootCacheDirectory, dependency.moduleName) + val versionCacheDirectory = File(moduleCacheDirectory, dependency.moduleVersion) + return File(versionCacheDirectory, computeDependenciesHash(dependency)) + } + + private fun needCache(libraryPath: String) = + libraryPath.startsWith(project.gradle.gradleUserHomeDir.absolutePath) && libraryPath.endsWith(".klib") + + private fun ensureDependencyPrecached(dependency: ResolvedDependency, visitedDependencies: MutableSet) { + if (dependency in visitedDependencies) + return + visitedDependencies += dependency + dependency.children.forEach { ensureDependencyPrecached(it, visitedDependencies) } + + val artifactsToAddToCache = dependency.moduleArtifacts.filter { needCache(it.file.absolutePath) } + if (artifactsToAddToCache.isEmpty()) return + + val dependenciesCacheDirectories = getAllDependencies(dependency) + .map { childDependency -> + val hasKlibs = childDependency.moduleArtifacts.any { it.file.absolutePath.endsWith(".klib") } + val cacheDirectory = getCacheDirectory(childDependency) + // We can only compile klib to cache if all of its dependencies are also cached. + if (hasKlibs && !cacheDirectory.exists()) + return + cacheDirectory + } + .filter { it.exists() } + val cacheDirectory = getCacheDirectory(dependency) + cacheDirectory.mkdirs() + + val artifactsLibraries = artifactsToAddToCache + .map { createKotlinLibrary(org.jetbrains.kotlin.konan.file.File(it.file.absolutePath)) } + .associateBy { it.uniqueName } + + // Top sort artifacts. + val sortedLibraries = mutableListOf() + val visitedLibraries = mutableSetOf() + + fun dfs(library: KotlinLibrary) { + visitedLibraries += library + library.unresolvedDependencies + .map { artifactsLibraries[it.path] } + .forEach { + if (it != null && it !in visitedLibraries) + dfs(it) + } + sortedLibraries += library + } + + for (library in artifactsLibraries.values) + if (library !in visitedLibraries) + dfs(library) + + for (library in sortedLibraries) { + project.logger.info("Compiling ${library.uniqueName} to cache") + val args = mutableListOf( + "-p", "dynamic_cache", + "-target", target + ) + if (debuggable) + args += "-g" + args += "-Xadd-cache=${library.libraryFile.absolutePath}" + args += "-Xcache-directory=${cacheDirectory.absolutePath}" + args += "-Xcache-directory=${rootCacheDirectory.absolutePath}" + + dependenciesCacheDirectories.forEach { + args += "-Xcache-directory=${it.absolutePath}" + } + getAllDependencies(dependency) + .flatMap { it.moduleArtifacts } + .map { it.file } + .filterExternalKlibs(project) + .forEach { + args += "-l" + args += it.absolutePath + } + library.unresolvedDependencies + .mapNotNull { artifactsLibraries[it.path] } + .forEach { + args += "-l" + args += it.libraryFile.absolutePath + } + KonanCompilerRunner(project).run(args) + } + } + + private fun ensureCompilerProvidedLibPrecached(platformLibName: String, platformLibs: Map, visitedLibs: MutableSet) { + if (platformLibName in visitedLibs) + return + visitedLibs += platformLibName + val platformLib = platformLibs[platformLibName] ?: error("$platformLibName is not found in platform libs") + if (File(rootCacheDirectory, System.mapLibraryName("$platformLibName-cache")).exists()) + return + for (dependency in createKotlinLibrary(org.jetbrains.kotlin.konan.file.File(platformLib.absolutePath)).unresolvedDependencies) + ensureCompilerProvidedLibPrecached(dependency.path, platformLibs, visitedLibs) + project.logger.info("Compiling $platformLibName (${visitedLibs.size}/${platformLibs.size}) to cache") + val args = mutableListOf( + "-p", "dynamic_cache", + "-target", target + ) + if (debuggable) + args += "-g" + args += "-Xadd-cache=${platformLib.absolutePath}" + args += "-Xcache-directory=${rootCacheDirectory.absolutePath}" + KonanCompilerRunner(project).run(args) + } + + private fun ensureCompilerProvidedLibsPrecached() { + val platformLibs = libraries.filter { it.providedByCompiler(project) }.associateBy { it.name } + val visitedLibs = mutableSetOf() + for (platformLibName in platformLibs.keys) + ensureCompilerProvidedLibPrecached(platformLibName, platformLibs, visitedLibs) + } + + fun buildCompilerArgs(): List = mutableListOf().apply { + if (!disableKonanCache && !optimized && compilation.konanTarget.family.isAppleFamily) { + rootCacheDirectory.mkdirs() + ensureCompilerProvidedLibsPrecached() + add("-Xcache-directory=${rootCacheDirectory.absolutePath}") + val visitedDependencies = mutableSetOf() + val allCacheDirectories = mutableSetOf() + val compileDependencyConfiguration = project.configurations.getByName(compilation.compileDependencyConfigurationName) + for (root in compileDependencyConfiguration.resolvedConfiguration.firstLevelModuleDependencies) { + ensureDependencyPrecached(root, visitedDependencies) + for (dependency in listOf(root) + getAllDependencies(root)) { + val cacheDirectory = getCacheDirectory(dependency) + if (cacheDirectory.exists()) + allCacheDirectories += cacheDirectory.absolutePath + } + } + for (cacheDirectory in allCacheDirectories) + add("-Xcache-directory=$cacheDirectory") + } + } +} + open class CInteropProcess : DefaultTask() { @Internal