[Gradle, K/N] Make compile and link tasks cacheable

Currently build cache isn't supported for frameworks as symlinks in build output is used. Symlinks are transformed into regular files by Gradle build cache and that breaks incremental build. Cinterop process task is not cacheable at the moment as it depends on OS-environment
#KT-39564 Fixed
This commit is contained in:
Alexander.Likhachev
2020-10-01 16:25:29 +07:00
parent a9b53adc50
commit a0f4898009
16 changed files with 278 additions and 49 deletions
@@ -36,6 +36,8 @@ abstract class BaseGradleIT {
open val defaultGradleVersion: GradleVersionRequired
get() = GradleVersionRequired.None
val isTeamCityRun = System.getenv("TEAMCITY_VERSION") != null
@Before
fun setUp() {
// Aapt2 from Android Gradle Plugin 3.2 and below does not handle long paths on Windows.
@@ -198,6 +200,7 @@ abstract class BaseGradleIT {
val jsIrBackend: Boolean? = null,
val androidHome: File? = null,
val javaHome: File? = null,
val gradleUserHome: File? = null,
val androidGradlePluginVersion: AGPVersion? = null,
val forceOutputToStdout: Boolean = false,
val debug: Boolean = false,
@@ -510,6 +513,16 @@ abstract class BaseGradleIT {
assertTasksExecuted(tasks.toList())
}
fun CompiledProject.assertTasksRetrievedFromCache(tasks: Iterable<String>) {
for (task in tasks) {
assertContains("$task FROM-CACHE")
}
}
fun CompiledProject.assertTasksRetrievedFromCache(vararg tasks: String) {
assertTasksRetrievedFromCache(tasks.toList())
}
fun CompiledProject.assertTasksUpToDate(tasks: Iterable<String>) {
for (task in tasks) {
assertContains("$task UP-TO-DATE")
@@ -828,6 +841,10 @@ Finished executing task ':$taskName'|
options.javaHome?.let {
put("JAVA_HOME", it.canonicalPath)
}
options.gradleUserHome?.let {
put("GRADLE_USER_HOME", it.canonicalPath)
}
}
private fun String.normalize() = this.lineSequence().joinToString(SYSTEM_LINE_SEPARATOR)
@@ -97,7 +97,8 @@ class KlibBasedMppIT : BaseGradleIT() {
"published-producer-commonMain.klib",
"published-dependency-$hostSpecificSourceSet.klib",
"published-dependency-commonMain.klib"
)
),
isNative = true
)
}
@@ -291,11 +291,13 @@ private var testBuildRunId = 0
fun BaseGradleIT.Project.checkTaskCompileClasspath(
taskPath: String,
checkModulesInClasspath: List<String> = emptyList(),
checkModulesNotInClasspath: List<String> = emptyList()
checkModulesNotInClasspath: List<String> = emptyList(),
isNative: Boolean = false
) {
val subproject = taskPath.substringBeforeLast(":").takeIf { it.isNotEmpty() && it != taskPath }
val taskName = taskPath.removePrefix(subproject.orEmpty())
val expression = """(tasks.getByName("$taskName") as AbstractCompile).classpath.toList()"""
val taskClass = if (isNative) "org.jetbrains.kotlin.gradle.tasks.AbstractKotlinNativeCompile<*, *>" else "AbstractCompile"
val expression = """(tasks.getByName("$taskName") as $taskClass).classpath.toList()"""
checkPrintedItems(subproject, expression, checkModulesInClasspath, checkModulesNotInClasspath)
}
@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.gradle.internals.DISABLED_NATIVE_TARGETS_REPORTER_WA
import org.jetbrains.kotlin.gradle.internals.NO_NATIVE_STDLIB_PROPERTY_WARNING
import org.jetbrains.kotlin.gradle.internals.NO_NATIVE_STDLIB_WARNING
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeOutputKind
import org.jetbrains.kotlin.gradle.prepareLocalBuildCache
import org.jetbrains.kotlin.gradle.transformProjectWithPluginsDsl
import org.jetbrains.kotlin.gradle.util.isWindows
import org.jetbrains.kotlin.gradle.util.modify
@@ -25,6 +26,7 @@ import org.junit.Assume
import org.junit.Ignore
import org.junit.Test
import java.io.File
import java.nio.file.Files
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
@@ -86,6 +88,8 @@ class GeneralNativeIT : BaseGradleIT() {
val nativeHostTargetName = MPPNativeTargets.current
private val buildCacheEnabledOptions = super.defaultBuildOptions().copy(withBuildCache = true)
private fun Project.targetClassesDir(targetName: String, sourceSetName: String = "main") =
classesDir(sourceSet = "$targetName/$sourceSetName")
@@ -138,6 +142,8 @@ class GeneralNativeIT : BaseGradleIT() {
@Test
fun testCanProduceNativeLibraries() = with(transformNativeTestProjectWithPluginDsl("libraries", directoryPrefix = "native-binaries")) {
prepareLocalBuildCache()
val baseName = "native_library"
val sharedPrefix = CompilerOutputKind.DYNAMIC.prefix(HostManager.host)
@@ -174,10 +180,16 @@ class GeneralNativeIT : BaseGradleIT() {
val klibTask = ":compileKotlinHost"
// Building
build(":assemble") {
// Building to local build cache
build("assemble", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksExecuted(linkTasks + klibTask)
}
// Retrieving from build cache
build(":clean", ":assemble", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksRetrievedFromCache(linkTasks + klibTask)
sharedPaths.forEach { assertFileExists(it) }
staticPaths.forEach { assertFileExists(it) }
@@ -830,4 +842,92 @@ class GeneralNativeIT : BaseGradleIT() {
}
}
@Test
fun testNativeDependenciesBuildCache() {
val libProject = transformNativeTestProjectWithPluginDsl("build-cache-lib", directoryPrefix = "native-build-cache")
val appProject = transformNativeTestProjectWithPluginDsl("build-cache-app", directoryPrefix = "native-build-cache")
val appProjectAltPath = Project("build-cache-app2", directoryPrefix = "native-build-cache")
val libLocalRepoUri = libProject.projectDir.resolve("repo").toURI()
with(libProject) {
prepareLocalBuildCache()
val compileTask = ":compileKotlinHost"
build("assemble", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksExecuted(compileTask)
}
build("clean", "publish", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksRetrievedFromCache(compileTask)
}
}
val compileAppTasks = listOf(
":compileKotlinHost",
":lib-module:compileKotlinHost",
)
with(appProject) {
prepareLocalBuildCache()
val klibPrefix = CompilerOutputKind.LIBRARY.prefix(HostManager.host)
val klibSuffix = CompilerOutputKind.LIBRARY.suffix(HostManager.host)
val klibPaths = listOf(
"${targetClassesDir("host")}${klibPrefix}app$klibSuffix",
"lib-module/${targetClassesDir("host")}${klibPrefix}lib-module$klibSuffix",
)
gradleBuildScript().appendText("\nrepositories { maven { setUrl(\"$libLocalRepoUri\") } }")
build("assemble", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksExecuted(compileAppTasks)
klibPaths.forEach { assertFileExists(it) }
}
build("clean", "assemble", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksRetrievedFromCache(compileAppTasks)
klibPaths.forEach { assertFileExists(it) }
}
assertTrue(projectDir.resolve(klibPaths[0]).delete())
build("clean", ":lib-module:assemble") {
assertSuccessful()
assertTasksExecuted(compileAppTasks[1])
}
build("assemble", options = buildCacheEnabledOptions) {
assertSuccessful()
assertTasksRetrievedFromCache(compileAppTasks[0])
assertTasksUpToDate(compileAppTasks[1])
klibPaths.forEach { assertFileExists(it) }
}
build("clean") {
assertSuccessful()
}
}
Files.move(appProject.projectDir.toPath(), appProjectAltPath.projectDir.toPath())
// It's very time-consuming check so on Mac TC agent we test only source relocation
val alternateBuildEnvOptions = if (isTeamCityRun && HostManager.hostIsMac) {
buildCacheEnabledOptions
} else {
val alternateGradleHome = File(appProject.projectDir.parentFile, "gradleUserHome")
buildCacheEnabledOptions.copy(gradleUserHome = alternateGradleHome)
}
with(appProjectAltPath) {
build("assemble", options = alternateBuildEnvOptions) {
assertSuccessful()
assertTasksRetrievedFromCache(compileAppTasks)
}
}
}
}
@@ -0,0 +1,21 @@
plugins {
id("org.jetbrains.kotlin.multiplatform").version("<pluginMarkerVersion>")
}
repositories {
mavenLocal()
jcenter()
}
kotlin {
<SingleNativeTarget>("host")
sourceSets {
val hostMain by getting {
dependencies {
implementation("com.example:build-cache-lib:1.0")
api(project(":lib-module"))
}
}
}
}
@@ -0,0 +1,12 @@
plugins {
id("org.jetbrains.kotlin.multiplatform")
}
repositories {
mavenLocal()
jcenter()
}
kotlin {
<SingleNativeTarget>("host")
}
@@ -0,0 +1,6 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
fun foo() = "some text"
@@ -0,0 +1,10 @@
pluginManagement {
repositories {
mavenLocal()
jcenter()
gradlePluginPortal()
}
}
include ":lib-module"
rootProject.name = "app"
@@ -0,0 +1,6 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
fun printNumber() = println("${getNumber()} ${foo()}")
@@ -0,0 +1,22 @@
group = "com.example"
version = "1.0"
plugins {
id("org.jetbrains.kotlin.multiplatform").version("<pluginMarkerVersion>")
id("maven-publish")
}
repositories {
mavenLocal()
jcenter()
}
kotlin {
<SingleNativeTarget>("host")
}
publishing {
repositories {
maven(url = "file://${projectDir.absolutePath.replace('\\', '/')}/repo")
}
}
@@ -0,0 +1,7 @@
pluginManagement {
repositories {
mavenLocal()
jcenter()
gradlePluginPortal()
}
}
@@ -0,0 +1,6 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
fun getNumber() = 42
@@ -23,10 +23,7 @@ import org.gradle.api.plugins.MavenPluginConvention
import org.gradle.api.provider.Provider
import org.gradle.api.publish.PublishingExtension
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.api.tasks.CompileClasspathNormalizer
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskProvider
import org.gradle.api.tasks.Upload
import org.gradle.api.tasks.*
import org.gradle.api.tasks.compile.AbstractCompile
import org.gradle.jvm.tasks.Jar
import org.gradle.tooling.provider.model.ToolingModelBuilderRegistry
@@ -61,7 +58,7 @@ val KOTLIN_DSL_NAME = "kotlin"
val KOTLIN_JS_DSL_NAME = "kotlin2js"
val KOTLIN_OPTIONS_DSL_NAME = "kotlinOptions"
abstract class KotlinCompilationProcessor<out T : AbstractCompile>(
abstract class KotlinCompilationProcessor<out T : SourceTask>(
open val kotlinCompilation: KotlinCompilation<*>
) {
abstract val kotlinTask: TaskProvider<out T>
@@ -17,9 +17,8 @@ import org.gradle.api.publish.PublicationContainer
import org.gradle.api.publish.PublishingExtension
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.api.publish.maven.internal.publication.MavenPublicationInternal
import org.gradle.api.publish.maven.tasks.AbstractPublishToMaven
import org.gradle.api.tasks.SourceTask
import org.gradle.api.tasks.TaskProvider
import org.gradle.api.tasks.compile.AbstractCompile
import org.gradle.jvm.tasks.Jar
import org.gradle.util.ConfigureUtil
import org.gradle.util.GradleVersion
@@ -153,7 +152,7 @@ class KotlinMultiplatformPlugin(
(sourceSet.languageSettings as? DefaultLanguageSettingsBuilder)?.run {
compilerPluginOptionsTask = lazy {
val associatedCompilation = primaryCompilationsBySourceSet[sourceSet] ?: metadataCompilation
project.tasks.getByName(associatedCompilation.compileKotlinTaskName) as AbstractCompile
project.tasks.getByName(associatedCompilation.compileKotlinTaskName) as SourceTask
}
}
}
@@ -8,8 +8,7 @@ package org.jetbrains.kotlin.gradle.plugin.sources
import org.gradle.api.InvalidUserDataException
import org.gradle.api.file.FileCollection
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.TaskProvider
import org.gradle.api.tasks.compile.AbstractCompile
import org.gradle.api.tasks.SourceTask
import org.jetbrains.kotlin.config.ApiVersion
import org.jetbrains.kotlin.config.LanguageFeature
import org.jetbrains.kotlin.config.LanguageVersion
@@ -70,7 +69,7 @@ internal class DefaultLanguageSettingsBuilder : LanguageSettingsBuilder {
/* A Kotlin task that is responsible for code analysis of the owner of this language settings builder. */
@Transient // not needed during Gradle Instant Execution
var compilerPluginOptionsTask: Lazy<AbstractCompile?> = lazyOf(null)
var compilerPluginOptionsTask: Lazy<SourceTask?> = lazyOf(null)
val compilerPluginArguments: List<String>?
get() {
@@ -18,7 +18,6 @@ import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.*
import org.gradle.api.tasks.compile.AbstractCompile
import org.jetbrains.kotlin.compilerRunner.*
import org.jetbrains.kotlin.gradle.dsl.KotlinCommonOptions
import org.jetbrains.kotlin.gradle.dsl.KotlinCommonToolOptions
@@ -113,13 +112,7 @@ private fun Collection<File>.filterKlibsPassedToCompiler(project: Project) = fil
}
// endregion
abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : AbstractKotlinNativeCompilation> : AbstractCompile() {
init {
sourceCompatibility = "1.6"
targetCompatibility = "1.6"
}
abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : AbstractKotlinNativeCompilation> : SourceTask() {
@get:Internal
abstract val compilation: Provider<K>
@@ -137,7 +130,14 @@ abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : Abst
abstract val baseName: String
// Inputs and outputs
@get:InputFiles
@InputFiles
@SkipWhenEmpty
@PathSensitive(PathSensitivity.RELATIVE)
override fun getSource(): FileTree {
return super.getSource()
}
@get:Classpath
val libraries: FileCollection by project.provider {
// Avoid resolving these dependencies during task graph construction when we can't build the target:
if (compilation.get().konanTarget.enabledOnCurrentHost)
@@ -145,10 +145,9 @@ abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : Abst
else project.files()
}
override fun getClasspath(): FileCollection = libraries
override fun setClasspath(configuration: FileCollection?) {
throw UnsupportedOperationException("Setting classpath directly is unsupported.")
}
@get:Internal
val classpath: FileCollection
get() = libraries
@get:Input
val target: String by project.provider { compilation.get().konanTarget.name }
@@ -173,15 +172,21 @@ abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : Abst
// endregion.
@get:Input
val enableEndorsedLibs by
project.provider { compilation.map { it.enableEndorsedLibs }.get() }
val enableEndorsedLibs: Boolean by project.provider { compilation.map { it.enableEndorsedLibs }.get() }
@get:Input
val kotlinNativeVersion: String
@Input get() = project.konanVersion.toString()
get() = project.konanVersion.toString()
private val destinationDirectory = getProject().getObjects().directoryProperty()
@get:Internal
open var destinationDir: File
get() = destinationDirectory.get().asFile
set(value) = destinationDirectory.set(value)
// OutputFile is located under the destinationDir, so there is no need to register it as a separate output.
@Internal
val outputFile: Provider<File> = project.provider {
open val outputFile: Provider<File> = project.provider {
val konanTarget = compilation.get().konanTarget
val prefix = outputKind.prefix(konanTarget)
@@ -203,20 +208,23 @@ abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : Abst
@Internal
val compilerPluginOptions = CompilerPluginOptions()
@get:Input
val compilerPluginCommandLine
@Input get() = compilerPluginOptions.arguments
get() = compilerPluginOptions.arguments
@Optional
@InputFiles
@Classpath
var compilerPluginClasspath: FileCollection? = null
// Used by IDE via reflection.
@get:Internal
val serializedCompilerArguments: List<String>
@Internal get() = buildCommonArgs()
get() = buildCommonArgs()
// Used by IDE via reflection.
@get:Internal
val defaultSerializedCompilerArguments: List<String>
@Internal get() = buildCommonArgs(true)
get() = buildCommonArgs(true)
// Args used by both the compiler and IDEA.
protected open fun buildCommonArgs(defaultsOnly: Boolean = false): List<String> = mutableListOf<String>().apply {
@@ -319,6 +327,7 @@ abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : Abst
/**
* A task producing a klibrary from a compilation.
*/
@CacheableTask
open class KotlinNativeCompile : AbstractKotlinNativeCompile<KotlinCommonOptions, AbstractKotlinNativeCompilation>(),
KotlinCompile<KotlinCommonOptions> {
@Internal
@@ -347,6 +356,10 @@ open class KotlinNativeCompile : AbstractKotlinNativeCompile<KotlinCommonOptions
project.klibModuleName(baseName)
}
@get:OutputFile
override val outputFile: Provider<File>
get() = super.outputFile
@get:Input
val shortModuleName: String by project.provider { baseName }
@@ -445,6 +458,7 @@ open class KotlinNativeCompile : AbstractKotlinNativeCompile<KotlinCommonOptions
/**
* A task producing a final binary from a compilation.
*/
@CacheableTask
open class KotlinNativeLink : AbstractKotlinNativeCompile<KotlinCommonToolOptions, KotlinNativeCompilation>() {
@get:Internal
@@ -453,6 +467,9 @@ open class KotlinNativeLink : AbstractKotlinNativeCompile<KotlinCommonToolOption
init {
dependsOn(project.provider { compilation.get().compileKotlinTask })
// Frameworks actively uses symlinks.
// Gradle build cache transforms symlinks into regular files https://guides.gradle.org/using-build-cache/#symbolic_links
outputs.cacheIf { outputKind != FRAMEWORK }
}
@Internal
@@ -469,14 +486,10 @@ open class KotlinNativeLink : AbstractKotlinNativeCompile<KotlinCommonToolOption
override fun getSource(): FileTree =
project.files(intermediateLibrary.get()).asFileTree
@OutputDirectory
override fun getDestinationDir(): File {
return binary.outputDirectory
}
override fun setDestinationDir(destinationDir: File) {
binary.outputDirectory = destinationDir
}
@get:OutputDirectory
override var destinationDir: File
get() = binary.outputDirectory
set(value) { binary.outputDirectory = value }
override val outputKind: CompilerOutputKind
@Input get() = binary.outputKind.compilerOutputKind
@@ -531,6 +544,7 @@ open class KotlinNativeLink : AbstractKotlinNativeCompile<KotlinCommonToolOption
@Input get() = binary is TestExecutable
@get:InputFiles
@get:Classpath
val exportLibraries: FileCollection
get() = binary.let {
if (it is AbstractNativeLibrary) {
@@ -913,8 +927,14 @@ open class CInteropProcess : DefaultTask() {
val outputFileProvider: Provider<File> =
project.provider { destinationDir.get().resolve(outputFileName) }
@OutputDirectory
val outputDirectoryProvider: Provider<File> =
project.provider { destinationDir.get().resolve("$outputFileName-build") }
@get:InputFile
@get:PathSensitive(PathSensitivity.RELATIVE)
val defFile: File
@InputFile get() = settings.defFileProperty.get()
get() = settings.defFileProperty.get()
val packageName: String?
@Optional @Input get() = settings.packageName
@@ -925,8 +945,10 @@ open class CInteropProcess : DefaultTask() {
val linkerOpts: List<String>
@Input get() = settings.linkerOpts
@get:InputFiles
@get:PathSensitive(PathSensitivity.RELATIVE)
val headers: FileCollection
@InputFiles get() = settings.headers
get() = settings.headers
val allHeadersDirs: Set<File>
@Input get() = settings.includeDirs.allHeadersDirs.files
@@ -934,8 +956,10 @@ open class CInteropProcess : DefaultTask() {
val headerFilterDirs: Set<File>
@Input get() = settings.includeDirs.headerFilterDirs.files
@get:InputFiles
@get:PathSensitive(PathSensitivity.RELATIVE)
val libraries: FileCollection
@InputFiles get() = settings.dependencyFiles.filterOutPublishableInteropLibs(project)
get() = settings.dependencyFiles.filterOutPublishableInteropLibs(project)
val extraOpts: List<String>
@Input get() = settings.extraOpts