From 791762d7eabf669e6ea45c07cfa404a524132527 Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Sat, 15 Jan 2022 17:11:32 +0300 Subject: [PATCH] Update kotlin report properties -add kotlin.build.report.output to add file/http/build_scan reports -all reports verbose by default -verbose reports includes compiler metrics -fix "custom value exceed length limit" --- .../report/metrics/BuildPerformanceMetric.kt | 2 + .../report/metrics/BuildPerformanceMetrics.kt | 2 +- .../jetbrains/kotlin/gradle/BaseGradleIT.kt | 6 +++ .../kotlin/gradle/ConfigurationCacheIT.kt | 3 +- .../kotlin/gradle/KotlinGradlePluginIT.kt | 48 +++++++++++-------- .../kotlin/gradle/testbase/BuildOptions.kt | 8 ++-- .../GradleCompilationResults.kt | 2 + .../GradleKotlinCompilerWork.kt | 4 +- .../plugin/KotlinGradleBuildServices.kt | 36 ++++++++++---- .../kotlin/gradle/plugin/KotlinProperties.kt | 36 +++++++++++++- ...ticSearch.kt => ReportStatisticsByHttp.kt} | 20 +++----- .../statistics/ReportStatisticsToBuildScan.kt | 28 +++++++++-- .../report/BuildMetricsReporterService.kt | 5 +- .../kotlin/gradle/report/BuildReportType.kt | 18 +++++++ .../report/KotlinBuildReporterListener.kt | 33 ------------- .../report/PlainTextBuildReportWriter.kt | 5 +- .../kotlin/gradle/report/ReportingSettings.kt | 26 ++++++++-- .../kotlin/gradle/report/configureReporing.kt | 45 ++++++++++------- .../gradle/tasks/KotlinCompileCommon.kt | 1 + .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 8 ++++ 20 files changed, 224 insertions(+), 112 deletions(-) rename libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/{ReportStatisticsToElasticSearch.kt => ReportStatisticsByHttp.kt} (72%) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportType.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/KotlinBuildReporterListener.kt diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index 3f72a048ac4..5db3c6992e0 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -14,6 +14,8 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size"), SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size"), + COMPILE_ITERATION(parent = null, "Total compiler iteration"), + // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)"), SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot"), diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt index af93cb04477..677bf43bf40 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetrics.kt @@ -21,7 +21,7 @@ class BuildPerformanceMetrics : Serializable { } } - fun add(metric: BuildPerformanceMetric, value: Long) { + fun add(metric: BuildPerformanceMetric, value: Long = 1) { myBuildMetrics[metric] = myBuildMetrics.getOrDefault(metric, 0) + value } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt index 0af40117ac6..d72a594ce5d 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt @@ -14,6 +14,7 @@ import org.intellij.lang.annotations.Language import org.jetbrains.kotlin.gradle.model.ModelContainer import org.jetbrains.kotlin.gradle.model.ModelFetcherBuildAction import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType +import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.testbase.applyAndroidTestFixes import org.jetbrains.kotlin.gradle.testbase.enableCacheRedirector import org.jetbrains.kotlin.gradle.testbase.extractJavaCompiledSources @@ -267,6 +268,7 @@ abstract class BaseGradleIT { val abiSnapshot: Boolean = false, val hierarchicalMPPStructureSupport: Boolean? = null, val enableCompatibilityMetadataVariant: Boolean? = null, + val withReports: List = emptyList(), ) enum class ConfigurationCacheProblems { @@ -944,6 +946,10 @@ Finished executing task ':$taskName'| add("-Pkotlin.mpp.enableCompatibilityMetadataVariant=${options.enableCompatibilityMetadataVariant}") } + if (options.withReports.isNotEmpty()) { + add("-Pkotlin.build.report.output=${options.withReports.joinToString { it.name }}") + } + add("-Dorg.gradle.unsafe.configuration-cache=${options.configurationCache}") add("-Dorg.gradle.unsafe.configuration-cache-problems=${options.configurationCacheProblems.name.toLowerCase()}") diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt index 15b36b18b30..d7c7edc48a0 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ConfigurationCacheIT.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.gradle import org.gradle.util.GradleVersion +import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.targets.js.dukat.ExternalsOutputFormat import org.jetbrains.kotlin.gradle.testbase.* import org.junit.jupiter.api.DisplayName @@ -176,7 +177,7 @@ class ConfigurationCacheIT : AbstractConfigurationCacheIT() { @GradleTest fun testBuildReportSmokeTestForConfigurationCache(gradleVersion: GradleVersion) { project("simpleProject", gradleVersion) { - val buildOptions = defaultBuildOptions.copy(buildReport = true) + val buildOptions = defaultBuildOptions.copy(buildReport = listOf(BuildReportType.FILE)) build("assemble", buildOptions = buildOptions) { assertOutputContains("Kotlin build report is written to") } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt index 22a42c04ccf..b80094cb950 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinGradlePluginIT.kt @@ -22,6 +22,7 @@ import org.gradle.util.GradleVersion import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData import org.jetbrains.kotlin.gradle.plugin.MULTIPLE_KOTLIN_PLUGINS_LOADED_WARNING import org.jetbrains.kotlin.gradle.plugin.MULTIPLE_KOTLIN_PLUGINS_SPECIFIC_PROJECTS_WARNING +import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.scripting.internal.ScriptingGradleSubplugin import org.jetbrains.kotlin.gradle.tasks.USING_JVM_INCREMENTAL_COMPILATION_MESSAGE import org.jetbrains.kotlin.gradle.testbase.TestVersions @@ -38,6 +39,8 @@ import kotlin.test.* class KotlinGradleIT : BaseGradleIT() { + private fun defaultOptionsWithReports() = defaultBuildOptions().copy(withReports = listOf(BuildReportType.FILE)) + @Test fun testCrossCompile() { val project = Project("kotlinJavaProject") @@ -908,21 +911,30 @@ class KotlinGradleIT : BaseGradleIT() { } @Test - fun testBuildReportSmokeTest() = with(Project("simpleProject")) { - build("assemble", "-Pkotlin.build.report.enable=true") { + fun testBuildReportSmokeTest() = with(Project("simpleProject"), ) { + build("assemble", options = defaultOptionsWithReports()) { assertSuccessful() assertContains("Kotlin build report is written to") } - build("clean", "assemble", "-Pkotlin.build.report.enable=true") { + build("clean", "assemble", options = defaultOptionsWithReports()) { assertSuccessful() assertContains("Kotlin build report is written to") } } + @Test + fun testBuildReportOutputProperty() = with(Project("simpleProject"), ) { + build("assemble", "-Pkotlin.build.report.output=file,invalid") { + assertFailed() + assertContains("Unknown output type:") + } + + } + @Test fun testBuildMetricsSmokeTest() = with(Project("simpleProject")) { - build("assemble", "-Pkotlin.build.report.enable=true", "-Pkotlin.build.report.verbose=true") { + build("assemble", options = defaultOptionsWithReports()) { assertSuccessful() assertContains("Kotlin build report is written to") } @@ -941,25 +953,23 @@ class KotlinGradleIT : BaseGradleIT() { assertTrue { report.contains("SNAPSHOT_SIZE:") } assertTrue { report.contains("Build attributes:") } assertTrue { report.contains("REBUILD_REASON:") } + assertTrue { report.contains("COMPILE_ITERATION:") } } @Test - fun testCompilerBuildMetricsSmokeTest() { - val buildOptions = defaultBuildOptions().copy(customEnvironmentVariables = mapOf("KOTLIN_REPORT_PERF" to "true")) - - return with(Project("simpleProject")) { - build("assemble", "-Pkotlin.build.report.enable=true", "-Pkotlin.build.report.verbose=true", options = buildOptions) { - assertSuccessful() - assertContains("Kotlin build report is written to") - } - val reportFolder = projectDir.resolve("build/reports/kotlin-build") - val reports = reportFolder.listFiles() - assertNotNull(reports) - assertEquals(1, reports.size) - val report = reports[0].readText() - assertTrue { report.contains("CODE_ANALYSIS:") } - assertTrue { report.contains("CODE_GENERATION:") } + fun testCompilerBuildMetricsSmokeTest() = with(Project("simpleProject")) { + build("assemble", options = defaultOptionsWithReports()) { + assertSuccessful() + assertContains("Kotlin build report is written to") } + val reportFolder = projectDir.resolve("build/reports/kotlin-build") + val reports = reportFolder.listFiles() + assertNotNull(reports) + assertEquals(1, reports.size) + val report = reports[0].readText() + assertTrue { report.contains("CODE_ANALYSIS:") } + assertTrue { report.contains("CODE_GENERATION:") } + assertTrue { report.contains("COMPILER_INITIALIZATION:") } } @Test diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt index 1c16b08f26a..c66b363604b 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt @@ -10,6 +10,7 @@ import org.gradle.api.logging.configuration.WarningMode import org.gradle.util.GradleVersion import org.jetbrains.kotlin.gradle.BaseGradleIT import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType +import org.jetbrains.kotlin.gradle.report.BuildReportType data class BuildOptions( val logLevel: LogLevel = LogLevel.INFO, @@ -26,7 +27,7 @@ data class BuildOptions( val kaptOptions: KaptOptions? = null, val androidVersion: String? = null, val jsOptions: JsOptions? = null, - val buildReport: Boolean = false, + val buildReport: List = emptyList(), ) { data class KaptOptions( val verbose: Boolean = false, @@ -112,9 +113,8 @@ data class BuildOptions( } arguments.add("-Ptest_fixes_version=${TestVersions.Kotlin.CURRENT}") - if (buildReport) { - arguments.add("-Pkotlin.build.report.enable=true") - arguments.add("-Pkotlin.build.report.verbose=true") + if (buildReport.isNotEmpty()) { + arguments.add("-Pkotlin.build.report.output=${buildReport.joinToString()}") } return arguments.toList() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt index 710b625d8b6..b7fc42d8b4f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilationResults.kt @@ -2,6 +2,7 @@ package org.jetbrains.kotlin.compilerRunner import org.jetbrains.kotlin.build.report.metrics.BuildMetrics import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl +import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.daemon.common.CompilationResultCategory import org.jetbrains.kotlin.daemon.common.CompilationResults import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface @@ -39,6 +40,7 @@ internal class GradleCompilationResults( val sourceFiles = compileIterationResult.sourceFiles if (sourceFiles.any()) { log.kotlinDebug { "compile iteration: ${sourceFiles.pathsAsStringRelativeTo(projectRootFile)}" } + buildMetrics.buildPerformanceMetrics.add(BuildPerformanceMetric.COMPILE_ITERATION) } val exitCode = compileIterationResult.exitCode log.kotlinDebug { "compiler exit code: $exitCode" } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 99141549193..63dcc3dfe06 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -103,7 +103,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( private val kotlinScriptExtensions = config.kotlinScriptExtensions private val allWarningsAsErrors = config.allWarningsAsErrors private val buildDir = config.projectFiles.buildDir - private val metrics = if (reportingSettings.reportMetrics) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter + private val metrics = if (reportingSettings.buildReportOutputs.isNotEmpty()) BuildMetricsReporterImpl() else DoNothingBuildMetricsReporter private var icLogLines: List = emptyList() private val daemonJvmArgs = config.daemonJvmArgs private val compilerExecutionStrategy = config.compilerExecutionStrategy @@ -386,7 +386,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( BuildReportMode.SIMPLE -> CompilationResultCategory.BUILD_REPORT_LINES BuildReportMode.VERBOSE -> CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES }?.let { requestedCompilationResults.add(it) } - if (reportingSettings.reportMetrics) { + if (reportingSettings.buildReportOutputs.isNotEmpty()) { requestedCompilationResults.add(CompilationResultCategory.BUILD_METRICS) } return requestedCompilationResults diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt index 7f00640bef9..5beada285d4 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt @@ -17,7 +17,10 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatListener import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToBuildScan -import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsToElasticSearch +import org.jetbrains.kotlin.gradle.plugin.statistics.ReportStatisticsByHttp +import org.jetbrains.kotlin.gradle.report.BuildReportType +import org.jetbrains.kotlin.gradle.report.ReportingSettings +import org.jetbrains.kotlin.gradle.report.reportingSettings import java.io.File internal abstract class KotlinGradleBuildServices : BuildService, AutoCloseable { @@ -54,20 +57,33 @@ internal abstract class KotlinGradleBuildServices : BuildService service.parameters.rootDir = project.rootProject.rootDir service.parameters.buildDir = project.rootProject.buildDir - addListeners(project) + + val reportingSettings = reportingSettings(project.rootProject) + addListeners(project, reportingSettings) } - fun addListeners(project: Project) { + fun addListeners(project: Project, reportingSettings: ReportingSettings) { + val listeners = project.rootProject.objects.listProperty(ReportStatistics::class.java) + .value(listOf()) + + reportingSettings.httpReportSettings?.let { + listeners.add( + ReportStatisticsByHttp(reportingSettings.httpReportSettings) + ) + } + project.rootProject.extensions.findByName("buildScan") ?.also { - val listeners = project.rootProject.objects.listProperty(ReportStatistics::class.java) - .value(listOf(ReportStatisticsToElasticSearch)) - listeners.add(ReportStatisticsToBuildScan(it as BuildScanExtension)) - val statListener = KotlinBuildStatListener(project.rootProject.name, listeners.get()) - val listenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project) - - listenerRegistryHolder.listenerRegistry.onTaskCompletion(project.provider { statListener }) + if (reportingSettings.buildReportOutputs.contains(BuildReportType.BUILD_SCAN)) { + listeners.add(ReportStatisticsToBuildScan(it as BuildScanExtension)) + } } + + if (listeners.get().isNotEmpty()) { + val listenerRegistryHolder = BuildEventsListenerRegistryHolder.getInstance(project) + val statListener = KotlinBuildStatListener(project.rootProject.name, listeners.get()) + listenerRegistryHolder.listenerRegistry.onTaskCompletion(project.provider { statListener }) + } } private val multipleProjectsHolder = KotlinPluginInMultipleProjectsHolder( diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt index af358a0f4ca..48f58656b93 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt @@ -93,8 +93,41 @@ internal class PropertiesProvider private constructor(private val project: Proje val singleBuildMetricsFile: File? get() = property("kotlin.internal.single.build.metrics.file")?.let { File(it) } + @Deprecated(message = "Please use kotlin.build.report.output instead ") val buildReportEnabled: Boolean - get() = booleanProperty("kotlin.build.report.enable") ?: false + get() { + val propValue = booleanProperty("kotlin.build.report.enable")?.also { + SingleWarningPerBuild.show( + project, + """ + 'kotlin.build.report.enable' property does nothing since 1.7.0 release + and scheduled to be removed in Kotlin 1.8.0 release! + Please use 'kotlin.build.report.output' instead + """.trimIndent() + ) + } + return propValue ?: false + } + + val buildReportOutputs: List + get() = property("kotlin.build.report.output")?.split(",") ?: emptyList() + + val buildReportLabel: String? + get() = property("kotlin.build.report.label") + + val buildReportFileOutputDir: File? + get() = property("kotlin.build.report.file.output_dir")?.let { File(it) } + + val buildReportHttpUrlProperty = "kotlin.build.report.http.url" + + val buildReportHttpUrl: String? + get() = property(buildReportHttpUrlProperty) + + val buildReportHttpUser: String? + get() = property("kotlin.build.report.http.user") + + val buildReportHttpPassword: String? + get() = property("kotlin.build.report.http.password") val buildReportMetrics: Boolean get() = booleanProperty("kotlin.build.report.metrics") ?: false @@ -102,6 +135,7 @@ internal class PropertiesProvider private constructor(private val project: Proje val buildReportVerbose: Boolean get() = booleanProperty("kotlin.build.report.verbose") ?: false + @Deprecated("Please use \"kotlin.build.report.file.output_dir\" property instead") val buildReportDir: File? get() = property("kotlin.build.report.dir")?.let { File(it) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToElasticSearch.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt similarity index 72% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToElasticSearch.kt rename to libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt index e6a6ce7d33d..89f6c16de09 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToElasticSearch.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsByHttp.kt @@ -11,32 +11,26 @@ import org.jetbrains.kotlin.cli.common.CompilerSystemProperties import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.gradle.plugin.stat.ReportStatistics import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatData +import org.jetbrains.kotlin.gradle.report.HttpReportSettings import java.net.HttpURLConnection import java.net.URL import java.util.* import kotlin.system.measureTimeMillis -object ReportStatisticsToElasticSearch : ReportStatistics { - val url by lazy { CompilerSystemProperties.KOTLIN_STAT_ENDPOINT_PROPERTY.value } - val user by lazy { CompilerSystemProperties.KOTLIN_STAT_USER_PROPERTY.value } - val enable: Boolean by lazy { CompilerSystemProperties.KOTLIN_STAT_ENABLED_PROPERTY.value?.toBooleanLenient() ?: false } +class ReportStatisticsByHttp( + private val httpProperties: HttpReportSettings +) : ReportStatistics { - //TODO Do not store password as string - val password by lazy { CompilerSystemProperties.KOTLIN_STAT_PASSWORD_PROPERTY.value } private val log = Logging.getLogger(this.javaClass) override fun report(data: CompileStatData) { val elapsedTime = measureTimeMillis { - if (!enable) { - return; - } - - val connection = URL(url).openConnection() as HttpURLConnection + val connection = URL(httpProperties.url).openConnection() as HttpURLConnection try { - if (user != null && password != null) { + if (httpProperties.user != null && httpProperties.password != null) { val auth = Base64.getEncoder() - .encode("$user:$password".toByteArray()).toString(Charsets.UTF_8) + .encode("${httpProperties.user}:${httpProperties.password}".toByteArray()).toString(Charsets.UTF_8) connection.addRequestProperty("Authorization", "Basic $auth") } connection.addRequestProperty("Content-Type", "application/json") diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt index 9b618bf630e..897f1ad9f78 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/ReportStatisticsToBuildScan.kt @@ -19,6 +19,7 @@ class ReportStatisticsToBuildScan( const val kbSize = 1024 const val mbSize = kbSize * kbSize const val gbSize = kbSize * mbSize + const val lengthLimit = 20 } private val tags = LinkedHashSet() @@ -26,7 +27,8 @@ class ReportStatisticsToBuildScan( override fun report(data: CompileStatData) { val elapsedTime = measureTimeMillis { - buildScan.value(data.taskName, readableString(data)) + + readableString(data).forEach { buildScan.value(data.taskName, it) } data.tags .filter { !tags.contains(it) } @@ -38,7 +40,7 @@ class ReportStatisticsToBuildScan( log.debug("Report statistic to build scan takes $elapsedTime ms") } - private fun readableString(data: CompileStatData): String { + private fun readableString(data: CompileStatData): List { val nonIncrementalReasons = data.nonIncrementalAttributes.filterValues { it > 0 }.keys val readableString = StringBuilder() if (nonIncrementalReasons.isEmpty()) { @@ -51,9 +53,29 @@ class ReportStatisticsToBuildScan( val timeData = data.timeData.map { (key, value) -> "${key.readableString}: ${value}ms"} //sometimes it is better to have separate variable to be able debug val perfData = data.perfData.map { (key, value) -> "${key.readableString}: ${readableFileLength(value)}"} timeData.union(perfData).joinTo(readableString, ",", "Performance: [", "]") - return readableString.toString() + + return splitStringIfNeed(readableString.toString(), lengthLimit) } + private fun splitStringIfNeed(str: String, lengthLimit: Int): List { + val splattedString = ArrayList() + var tempStr = str + while (tempStr.length > lengthLimit) { + val subSequence = tempStr.substring(lengthLimit) + var index = subSequence.lastIndexOf(';') + if (index == -1) { + index = subSequence.lastIndexOf(',') + if (index == -1) { + index = lengthLimit + } + } + splattedString.add(tempStr.substring(index)) + tempStr = tempStr.substring(index) + + } + splattedString.add(tempStr) + return splattedString + } private fun readableFileLength(length: Long): String = when { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt index 548e3e8bf57..f63ca50d30a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/BuildMetricsReporterService.kt @@ -108,11 +108,10 @@ abstract class BuildMetricsReporterService : BuildService() - internal var allTasksTimeNs: Long = 0L - - override fun onFinish(event: FinishEvent?) { - val descriptor = event?.descriptor - if (descriptor is TaskOperationDescriptor) { - val taskPath = descriptor.taskPath - val executionTime = event.result.endTime - event.result.startTime - allTasksTimeNs += executionTime - kotlinTaskTimeNs[taskPath] = executionTime - } - } - - override fun close() { - KotlinBuildReporterHandler().buildFinished(gradle, perfReportFile, kotlinTaskTimeNs, allTasksTimeNs) - } -} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt index 84702fb6e89..e3db050c65b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt @@ -20,13 +20,12 @@ import kotlin.collections.HashSet import kotlin.math.max internal class PlainTextBuildReportWriterDataProcessor( - val reportingSettings: ReportingSettings, - val reportDir: File, //from reportingSettings.buildReportDir to avoid null check + val reportingSettings: FileReportSettings, val rootProjectName: String ) : BuildExecutionDataProcessor, Serializable { override fun process(build: BuildExecutionData, log: Logger) { val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time) - val reportFile = reportDir.resolve("${rootProjectName}-build-$ts.txt") + val reportFile = reportingSettings.buildReportDir.resolve("${rootProjectName}-build-$ts.txt") PlainTextBuildReportWriter( outputFile = reportFile, diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt index 7e289797a92..7308b68d4cd 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt @@ -10,10 +10,30 @@ import java.io.Serializable data class ReportingSettings( val metricsOutputFile: File? = null, - val buildReportDir: File? = null, - val reportMetrics: Boolean = false, + val buildReportOutputs: List = emptyList(), + val buildReportMode: BuildReportMode = BuildReportMode.NONE, + val fileReportSettings: FileReportSettings? = null, + val httpReportSettings: HttpReportSettings? = null +) : Serializable { + companion object { + const val serialVersionUID: Long = 1 + } +} + +data class FileReportSettings( + val metricsOutputFile: File? = null, + val buildReportDir: File, val includeMetricsInReport: Boolean = false, - val buildReportMode: BuildReportMode = BuildReportMode.NONE +) : Serializable { + companion object { + const val serialVersionUID: Long = 0 + } +} + +data class HttpReportSettings( + val url: String, + val password: String?, + val user: String? ) : Serializable { companion object { const val serialVersionUID: Long = 0 diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt index 108e15c18e6..221aee048d6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt @@ -6,31 +6,44 @@ package org.jetbrains.kotlin.gradle.report import org.gradle.api.Project -import org.gradle.api.invocation.Gradle -import org.gradle.api.logging.Logger import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider -import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor -import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile -import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable -import java.text.SimpleDateFormat -import java.util.* internal fun reportingSettings(rootProject: Project): ReportingSettings { val properties = PropertiesProvider(rootProject) + val buildReportOutputTypes = properties.buildReportOutputs.map { + BuildReportType.values().firstOrNull { brt -> brt.name == it.trim().toUpperCase() } + ?: throw IllegalStateException("Unknown output type: $it") + } val buildReportMode = when { - !properties.buildReportEnabled -> BuildReportMode.NONE - properties.buildReportVerbose -> BuildReportMode.VERBOSE - else -> BuildReportMode.SIMPLE + buildReportOutputTypes.isEmpty() -> BuildReportMode.NONE + else -> BuildReportMode.VERBOSE } + val fileReportSettings = if (buildReportOutputTypes.contains(BuildReportType.FILE)) { + val buildReportDir = properties.buildReportFileOutputDir ?: rootProject.buildDir.resolve("reports/kotlin-build") + val includeMetricsInReport = properties.buildReportMetrics || buildReportMode == BuildReportMode.VERBOSE + FileReportSettings(buildReportDir = buildReportDir, includeMetricsInReport = includeMetricsInReport) + } else { + null + } + + val httpReportSettings = if (buildReportOutputTypes.contains(BuildReportType.HTTP)) { + val url = properties.buildReportHttpUrl + ?: throw IllegalStateException("Can't configure http report: '${properties.buildReportHttpUrlProperty}' property is mandatory") + val password = properties.buildReportHttpPassword + val user = properties.buildReportHttpUser + HttpReportSettings(url, password, user) + } else { + null + } val metricsOutputFile = properties.singleBuildMetricsFile - val buildReportDir = properties.buildReportDir ?: rootProject.buildDir.resolve("reports/kotlin-build") - val includeMetricsInReport = properties.buildReportMetrics || buildReportMode == BuildReportMode.VERBOSE return ReportingSettings( metricsOutputFile = metricsOutputFile, - buildReportDir = buildReportDir, - reportMetrics = metricsOutputFile != null || includeMetricsInReport, - includeMetricsInReport = includeMetricsInReport, - buildReportMode = buildReportMode + buildReportMode = buildReportMode, + fileReportSettings = fileReportSettings, + httpReportSettings = httpReportSettings, + buildReportOutputs = buildReportOutputTypes ) } + + diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt index cb438fbf31f..f387501845d 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinCompileCommon.kt @@ -39,6 +39,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinCompilationData import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinMetadataCompilationData import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.refinesClosure import org.jetbrains.kotlin.gradle.plugin.sources.resolveAllDependsOnSourceSets +import org.jetbrains.kotlin.gradle.report.BuildReportMode import org.jetbrains.kotlin.gradle.utils.propertyWithConvention import java.io.File import javax.inject.Inject diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index 64744eaa988..511d9b054ff 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -52,6 +52,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.* import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinCompilationData import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.report.BuildMetricsReporterService +import org.jetbrains.kotlin.gradle.report.BuildReportMode import org.jetbrains.kotlin.gradle.report.ReportingSettings import org.jetbrains.kotlin.gradle.targets.js.ir.isProduceUnzippedKlib import org.jetbrains.kotlin.gradle.utils.* @@ -478,6 +479,9 @@ abstract class AbstractKotlinCompile : AbstractKotl args, compilerArgumentsConfigurationFlags(defaultsOnly, ignoreClasspathResolutionErrors) ) + if (reportingSettings().buildReportMode == BuildReportMode.VERBOSE) { + args.reportPerf = true + } } } @@ -722,6 +726,10 @@ abstract class KotlinCompile @Inject constructor( if (state.executing) { defaultKotlinJavaToolchain.get().updateJvmTarget(this, args) } + + if (reportingSettings().buildReportMode == BuildReportMode.VERBOSE) { + args.reportPerf = true + } } @get:Internal