azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove all project iteration for projectIsolation support

Browse Source 
KT-62131 Fixed
This commit is contained in:
Nataliya.Valtman
2023-10-02 18:40:49 +02:00
committed by Space Team
parent 21c46a6ddb
commit b9090e12b8
8 changed files with 152 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/FusStatisticsIT.kt
+20
View File
@@ -166,6 +166,26 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
}
}
@DisplayName("fus metric for multiproject")
@GradleTest
@GradleTestVersions(
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
)
fun testFusStatisticsForMultiproject(gradleVersion: GradleVersion) {
project(
"incrementalMultiproject", gradleVersion,
) {
//after KT-58768 and KT-62616 are done it will be possible to check build metrics also
build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") {
assertFileContains(
fusStatisticsPath,
"CONFIGURATION_IMPLEMENTATION_COUNT=2",
"NUMBER_OF_SUBPROJECTS=2",
)
}
}
}
@DisplayName("general fields with configuration cache")
@GradleTest
@GradleTestVersions(
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/mpp/MppDiagnosticsIt.kt
+1 -4
View File
@@ -123,10 +123,7 @@ class MppDiagnosticsIt : KGPBaseTest() {
fun testSuppressGradlePluginFatals(gradleVersion: GradleVersion) {
project("suppressGradlePluginFatals", gradleVersion) {
buildAndFail("assemble") {
// Gradle 8.0+ for some reason renders exception twice in the build log
val testDataSuffixIfAny = if (gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_0)) "gradle-6.8.3" else null
assertEqualsToFile(expectedOutputFile(testDataSuffixIfAny), extractProjectsAndTheirDiagnostics())
assertEqualsToFile(expectedOutputFile(), extractProjectsAndTheirDiagnostics())
}
}
}
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/suppressGradlePluginFatals/expectedOutput-gradle-6.8.3.txt
-6
View File
@@ -1,6 +0,0 @@
> Configure project :
[CircularDependsOnEdges | FATAL] Circular dependsOn hierarchy found in the Kotlin source sets: intermediate -> jvmMain -> intermediate
#diagnostic-end
org.gradle.api.InvalidUserCodeException: [CircularDependsOnEdges | FATAL] Circular dependsOn hierarchy found in the Kotlin source sets: intermediate -> jvmMain -> intermediate
#diagnostic-end
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/suppressGradlePluginFatals/expectedOutput.txt
-6
View File
@@ -4,9 +4,3 @@
org.gradle.api.InvalidUserCodeException: [CircularDependsOnEdges | FATAL] Circular dependsOn hierarchy found in the Kotlin source sets: intermediate -> jvmMain -> intermediate
#diagnostic-end
[CircularDependsOnEdges | FATAL] Circular dependsOn hierarchy found in the Kotlin source sets: intermediate -> jvmMain -> intermediate
#diagnostic-end
org.gradle.api.InvalidUserCodeException: [CircularDependsOnEdges | FATAL] Circular dependsOn hierarchy found in the Kotlin source sets: intermediate -> jvmMain -> intermediate
#diagnostic-end
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildFlowService.kt
+16 -6
View File
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.plugin.statistics
import org.gradle.api.Project
import org.gradle.api.logging.Logging
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
@@ -38,7 +39,7 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
}
interface Parameters : BuildServiceParameters {
val configurationMetrics: Property<MetricContainer>
val configurationMetrics: ListProperty<MetricContainer>
val fusStatisticsAvailable: Property<Boolean>
}
@@ -58,9 +59,15 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
project: Project,
): Provider<BuildFlowService> {
val isProjectIsolationEnabled = project.isProjectIsolationEnabled
project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
@Suppress("UNCHECKED_CAST")
return it.service as Provider<BuildFlowService>
return (it.service as Provider<BuildFlowService>).also {
it.get().parameters.configurationMetrics.add(project.provider {
KotlinBuildStatsService.getInstance()?.collectProjectConfigurationMetrics(project, isProjectIsolationEnabled)
})
}
}
val fusStatisticsAvailable = fusStatisticsAvailable(project)
@@ -70,7 +77,6 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
//Workaround for known issues for Gradle 8+: https://github.com/gradle/gradle/issues/24887:
// when this OperationCompletionListener is called services can be already closed for Gradle 8,
// so there is a change that no VariantImplementationFactory will be found
val isProjectIsolationEnabled = project.isProjectIsolationEnabled
return project.gradle.sharedServices.registerIfAbsent(serviceName, BuildFlowService::class.java) { spec ->
if (fusStatisticsAvailable) {
KotlinBuildStatsService.applyIfInitialised {
@@ -78,8 +84,12 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
}
}
spec.parameters.configurationMetrics.set(project.provider {
KotlinBuildStatsService.getInstance()?.collectStartMetrics(project, isProjectIsolationEnabled, buildReportOutputs)
spec.parameters.configurationMetrics.add(project.provider {
KotlinBuildStatsService.getInstance()?.collectGeneralConfigurationMetrics(project, isProjectIsolationEnabled, buildReportOutputs)
})
spec.parameters.configurationMetrics.add(project.provider {
KotlinBuildStatsService.getInstance()?.collectProjectConfigurationMetrics(project, isProjectIsolationEnabled)
})
spec.parameters.fusStatisticsAvailable.set(fusStatisticsAvailable)
}.also { buildService ->
@@ -114,7 +124,7 @@ internal abstract class BuildFlowService : BuildService<BuildFlowService.Paramet
internal fun recordBuildFinished(action: String?, buildFailed: Boolean) {
KotlinBuildStatsService.applyIfInitialised {
it.recordBuildFinish(action, buildFailed, parameters.configurationMetrics.orElse(MetricContainer()).get())
it.recordBuildFinish(action, buildFailed, parameters.configurationMetrics.orElse(emptyList()).get())
}
}
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatHandler.kt
+99 -85
View File
@@ -73,120 +73,134 @@ class KotlinBuildStatHandler {
sessionLogger: BuildSessionLogger,
action: String?,
buildFailed: Boolean,
configurationMetrics: MetricContainer,
configurationMetrics: List<MetricContainer>,
) {
runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") {
configurationMetrics.report(sessionLogger)
configurationMetrics.forEach { it.report(sessionLogger) }
sessionLogger.finishBuildSession(action, buildFailed)
}
}
internal fun collectConfigurationTimeMetrics(
internal fun collectGeneralConfigurationTimeMetrics(
project: Project,
sessionLogger: BuildSessionLogger,
isProjectIsolationEnabled: Boolean,
buildReportOutputs: List<BuildReportType>,
): MetricContainer {
val gradle = project.gradle
val configurationTimeMetrics = MetricContainer()
configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
buildReportOutputs.forEach {
when (it) {
BuildReportType.BUILD_SCAN -> configurationTimeMetrics.put(BooleanMetrics.BUILD_SCAN_BUILD_REPORT, true)
BuildReportType.FILE -> configurationTimeMetrics.put(BooleanMetrics.FILE_BUILD_REPORT, true)
BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true)
BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true)
BuildReportType.TRY_K2_CONSOLE -> {}//ignore
val statisticOverhead = measureTimeMillis {
buildReportOutputs.forEach {
when (it) {
BuildReportType.BUILD_SCAN -> configurationTimeMetrics.put(BooleanMetrics.BUILD_SCAN_BUILD_REPORT, true)
BuildReportType.FILE -> configurationTimeMetrics.put(BooleanMetrics.FILE_BUILD_REPORT, true)
BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true)
BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true)
BuildReportType.TRY_K2_CONSOLE -> {}//ignore
}
}
val gradle = project.gradle
configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
if (!isProjectIsolationEnabled) {
gradle.taskGraph.whenReady { taskExecutionGraph ->
val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct()
configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install"))
}
}
}
sessionLogger.report(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
return configurationTimeMetrics
}
internal fun collectProjectConfigurationTimeMetrics(
project: Project,
sessionLogger: BuildSessionLogger,
isProjectIsolationEnabled: Boolean,
): MetricContainer {
val configurationTimeMetrics = MetricContainer()
if (isProjectIsolationEnabled) { //support project isolation - KT-58768
return configurationTimeMetrics
}
gradle.taskGraph.whenReady { taskExecutionGraph ->
val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct()
configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install"))
}// constants are saved in IDEA plugin and could not be accessed directly
fun buildSrcExists(project: Project) = File(project.projectDir, "buildSrc").exists()
configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, buildSrcExists(gradle.rootProject))
val statisticOverhead = measureTimeMillis {
gradle.allprojects { project ->
collectAppliedPluginsStatistics(project, configurationTimeMetrics)
collectAppliedPluginsStatistics(project, configurationTimeMetrics)
val configurations = project.configurations.asMap.values
for (configuration in configurations) {
try {
val configurationName = configuration.name
val dependencies = configuration.dependencies
val configurations = project.configurations.asMap.values
for (configuration in configurations) {
try {
val configurationName = configuration.name
val dependencies = configuration.dependencies
when (configurationName) {
"KoverEngineConfig" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true)
}
"kapt" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true)
for (dependency in dependencies) {
when (dependency.group) {
"com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true)
"com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true)
}
when (configurationName) {
"KoverEngineConfig" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true)
}
"kapt" -> {
configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true)
for (dependency in dependencies) {
when (dependency.group) {
"com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true)
"com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true)
}
}
API -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
IMPLEMENTATION -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_ONLY_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_ONLY_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
}
} catch (e: Throwable) {
// log?
API -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
IMPLEMENTATION -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
COMPILE_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_ONLY_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
RUNTIME_ONLY -> {
configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_ONLY_COUNT,1)
reportLibrariesVersions(configurationTimeMetrics, dependencies)
}
}
} catch (e: Throwable) {
// log?
}
val taskNames = project.tasks.names.toList()
}
configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1)
configurationTimeMetrics.put(
BooleanMetrics.KOTLIN_KTS_USED,
project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false
)
configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong())
configurationTimeMetrics.put(
NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS,
taskNames.count { name ->
try {
project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider")
} catch (_: Exception) {
true
}
}.toLong()
)
configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1)
if (project.name == "buildSrc") {
configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1)
configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true)
}
val taskNames = project.tasks.names.toList()
configurationTimeMetrics.put(
BooleanMetrics.KOTLIN_KTS_USED,
project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false
)
configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong())
configurationTimeMetrics.put(
NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS,
taskNames.count { name ->
try {
project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider")
} catch (_: Exception) {
true
}
}.toLong()
)
if (project.name == "buildSrc") {
configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1)
configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true)
}
}
sessionLogger.report(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt
+15 -7
View File
@@ -103,7 +103,8 @@ internal abstract class KotlinBuildStatsService internal constructor() : IStatis
kotlinBuildStatsServicesRegistry = it
}
val defaultServiceName = KotlinBuildStatsServicesRegistry.getBeanName(KotlinBuildStatsServicesRegistry.DEFAULT_SERVICE_QUALIFIER)
val defaultServiceName =
KotlinBuildStatsServicesRegistry.getBeanName(KotlinBuildStatsServicesRegistry.DEFAULT_SERVICE_QUALIFIER)
val instance = kotlinBuildStatsServicesRegistry?.getDefaultService()
if (instance != null) {
registry.logger.debug("$defaultServiceName is already instantiated. Current instance is $instance")
@@ -191,12 +192,17 @@ internal abstract class KotlinBuildStatsService internal constructor() : IStatis
/**
* Collects metrics at the end of a build
*/
open fun recordBuildFinish(action: String?, buildFailed: Boolean, configurationTimeMetrics: MetricContainer) {}
open fun recordBuildFinish(action: String?, buildFailed: Boolean, configurationTimeMetrics: List<MetricContainer>) {}
/**
* Collect project general and configuration metrics at the start of a build
* Collect project's configuration metrics
*/
open fun collectStartMetrics(project: Project, isProjectIsolationEnabled: Boolean, buildReportOutputs: List<BuildReportType>): MetricContainer = MetricContainer()
open fun collectProjectConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean): MetricContainer = MetricContainer()
/**
* Collect general configuration metrics
*/
open fun collectGeneralConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean, buildReportOutputs: List<BuildReportType>): MetricContainer = MetricContainer()
open fun recordProjectsEvaluated(gradle: Gradle) {}
}
@@ -308,12 +314,14 @@ internal class DefaultKotlinBuildStatsService internal constructor(
report(StringMetrics.valueOf(name), value, subprojectName, weight)
//only one jmx bean service should report global metrics
override fun recordBuildFinish(action: String?, buildFailed: Boolean, configurationTimeMetrics: MetricContainer) {
override fun recordBuildFinish(action: String?, buildFailed: Boolean, configurationTimeMetrics: List<MetricContainer>) {
KotlinBuildStatHandler().reportGlobalMetrics(sessionLogger)
KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, configurationTimeMetrics)
}
override fun collectStartMetrics(project: Project, isProjectIsolationEnabled: Boolean, buildReportOutputs: List<BuildReportType>) =
KotlinBuildStatHandler().collectConfigurationTimeMetrics(project, sessionLogger, isProjectIsolationEnabled, buildReportOutputs)
override fun collectProjectConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean) =
KotlinBuildStatHandler().collectProjectConfigurationTimeMetrics(project, sessionLogger, isProjectIsolationEnabled)
override fun collectGeneralConfigurationMetrics(project: Project, isProjectIsolationEnabled: Boolean, buildReportOutputs: List<BuildReportType>) =
KotlinBuildStatHandler().collectGeneralConfigurationTimeMetrics(project, sessionLogger, isProjectIsolationEnabled, buildReportOutputs)
}
libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/metrics/NumericalMetrics.kt
+1
View File
@@ -56,6 +56,7 @@ enum class NumericalMetrics(val type: NumberOverridePolicy, val anonymization: N
ANALYSIS_LINES_PER_SECOND(AVERAGE, SAFE),
CODE_GENERATION_LINES_PER_SECOND(AVERAGE, SAFE),
//only Kotlin subprojects are counted
NUMBER_OF_SUBPROJECTS(SUM, RANDOM_10_PERCENT),
STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD(SUM, RANDOM_10_PERCENT),