Add JSON output type for build reports

#KT-65792 Fixed
2024-01-24 11:47:36 +01:00
parent f493df42a9
commit 5885514c3d
21 changed files with 515 additions and 330 deletions
@@ -5,9 +5,16 @@

 package org.jetbrains.kotlin.gradle

+import com.google.gson.*
+import com.google.gson.stream.JsonReader
 import org.gradle.api.logging.LogLevel
 import org.gradle.util.GradleVersion
+import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
+import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
+import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
+import org.jetbrains.kotlin.build.report.statistics.StatTag
 import org.jetbrains.kotlin.build.report.statistics.formatSize
+import org.jetbrains.kotlin.buildtools.api.SourcesChanges
 import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData
 import org.jetbrains.kotlin.gradle.report.BuildReportType
 import org.jetbrains.kotlin.gradle.testbase.*
@@ -17,10 +24,14 @@ import java.nio.file.Path
 import kotlin.io.path.*
 import kotlin.test.assertTrue
 import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
+import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
+import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
 import org.jetbrains.kotlin.gradle.testbase.TestVersions.ThirdPartyDependencies.GRADLE_ENTERPRISE_PLUGIN_VERSION
+import java.lang.reflect.Type
 import java.nio.file.Files
 import kotlin.streams.asSequence
 import kotlin.test.assertEquals
+import kotlin.test.assertNotNull

@DisplayName("Build reports")
@JvmGradlePluginTests
@@ -507,4 +518,79 @@ class BuildReportsIT : KGPBaseTest() {
        }
    }

+    @DisplayName("json validation")
+    @GradleTestVersions(
+        additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
+    )
+    @GradleTest
+    fun testJsonBuildMetricsFileValidation(gradleVersion: GradleVersion) {
+        project("simpleProject", gradleVersion) {
+            buildAndFail(
+                "compileKotlin", "-Pkotlin.build.report.output=JSON",
+            ) {
+                assertOutputContains("Can't configure json report: 'kotlin.build.report.json.directory' property is mandatory")
+            }
+        }
+    }
+
+    @DisplayName("json report")
+    @GradleTestVersions(
+        additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
+    )
+    @GradleTest
+    fun testJsonBuildReport(gradleVersion: GradleVersion) {
+        project("simpleProject", gradleVersion) {
+            build(
+                "compileKotlin",
+                "-Pkotlin.build.report.output=JSON",
+                "-Pkotlin.build.report.json.directory=${projectPath.resolve("report").pathString}"
+            ) {
+                //TODO: KT-66071 update deserialization
+                val gsonBuilder = GsonBuilder()
+                    .registerTypeAdapter(BuildOperationRecord::class.java, object : JsonDeserializer<BuildOperationRecord> {
+                        override fun deserialize(
+                            json: JsonElement?,
+                            typeOfT: Type?,
+                            context: JsonDeserializationContext?,
+                        ): BuildOperationRecord? {
+                            //workaround to read both TaskRecord and TransformRecord
+                            return context?.deserialize(json, BuildOperationRecordImpl::class.java)
+                        }
+                    }).registerTypeAdapter(SourcesChanges::class.java, object : JsonDeserializer<SourcesChanges> {
+                        override fun deserialize(
+                            json: JsonElement?,
+                            typeOfT: Type?,
+                            context: JsonDeserializationContext,
+                        ): SourcesChanges? {
+                            return null //ignore source changes right now
+                        }
+                    })
+
+                val jsonReport = projectPath.getSingleFileInDir("report")
+                val buildExecutionData = jsonReport.bufferedReader().use {
+                    gsonBuilder.create().fromJson(JsonReader(it), BuildExecutionData::class.java) as BuildExecutionData
+                }
+                val buildOperationRecords = buildExecutionData.buildOperationRecord.first { it.path == ":compileKotlin" } as BuildOperationRecordImpl
+                assertEquals(KotlinVersion.DEFAULT, buildOperationRecords.kotlinLanguageVersion)
+            }
+        }
+    }
+
 }
+
+data class BuildOperationRecordImpl(
+    override val path: String,
+    override val classFqName: String,
+    override val isFromKotlinPlugin: Boolean,
+    override val startTimeMs: Long, // Measured by System.currentTimeMillis(),
+    override val totalTimeMs: Long,
+    override val buildMetrics: BuildMetrics<GradleBuildTime, GradleBuildPerformanceMetric>,
+    override val didWork: Boolean,
+    override val skipMessage: String?,
+    override val icLogLines: List<String>,
+    //taskRecords
+    val kotlinLanguageVersion: KotlinVersion?,
+    val changedFiles: SourcesChanges? = null,
+    val compilerArguments: Array<String> = emptyArray(),
+    val statTags: Set<StatTag> = emptySet(),
+): BuildOperationRecord
@@ -110,6 +110,9 @@ internal class PropertiesProvider private constructor(private val project: Proje
    val buildReportMetrics: Boolean
        get() = booleanProperty("kotlin.build.report.metrics") ?: false

+    val buildReportJsonDir: File?
+        get() = property(PropertyNames.KOTLIN_BUILD_REPORT_JSON_DIR).orNull?.let { File(it) }
+
    val buildReportVerbose: Boolean
        get() = booleanProperty("kotlin.build.report.verbose") ?: false

@@ -643,6 +646,7 @@ internal class PropertiesProvider private constructor(private val project: Proje
        val KOTLIN_JS_KARMA_BROWSERS = property("kotlin.js.browser.karma.browsers")
        val KOTLIN_BUILD_REPORT_SINGLE_FILE = property("kotlin.build.report.single_file")
        val KOTLIN_BUILD_REPORT_HTTP_URL = property("kotlin.build.report.http.url")
+        val KOTLIN_BUILD_REPORT_JSON_DIR = property("kotlin.build.report.json.directory")
        val KOTLIN_OPTIONS_SUPPRESS_FREEARGS_MODIFICATION_WARNING = property("kotlin.options.suppressFreeCompilerArgsModificationWarning")
        val KOTLIN_NATIVE_USE_XCODE_MESSAGE_STYLE = property("kotlin.native.useXcodeMessageStyle")
        val KOTLIN_INCREMENTAL_USE_CLASSPATH_SNAPSHOT = property("kotlin.incremental.useClasspathSnapshot")
@@ -7,13 +7,11 @@ package org.jetbrains.kotlin.gradle.plugin.statistics

 import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
 import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
-import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
-import org.jetbrains.kotlin.buildtools.api.KotlinLogger
+import org.jetbrains.kotlin.build.report.statistics.file.ReadableFileReportService
 import java.io.File

 class GradleFileReportService(
    buildReportDir: File,
    projectName: String,
    printMetrics: Boolean,
-    logger: KotlinLogger,
-) : FileReportService<GradleBuildTime, GradleBuildPerformanceMetric>(buildReportDir, projectName, printMetrics, logger) {}
+) : ReadableFileReportService<GradleBuildTime, GradleBuildPerformanceMetric>(buildReportDir, projectName, printMetrics)
@@ -39,6 +39,7 @@ internal fun collectGeneralConfigurationTimeMetrics(
                BuildReportType.HTTP -> configurationTimeMetrics.put(BooleanMetrics.HTTP_BUILD_REPORT, true)
                BuildReportType.SINGLE_FILE -> configurationTimeMetrics.put(BooleanMetrics.SINGLE_FILE_BUILD_REPORT, true)
                BuildReportType.TRY_NEXT_CONSOLE -> {}//ignore
+                BuildReportType.JSON -> configurationTimeMetrics.put(BooleanMetrics.JSON_BUILD_REPORT, true)
            }
        }
        configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
@@ -13,9 +13,10 @@ enum class BuildReportType : Serializable {
    BUILD_SCAN,
    SINGLE_FILE,
    TRY_NEXT_CONSOLE,
+    JSON,
    ;

    companion object {
-        const val serialVersionUID: Long = 2L
+        const val serialVersionUID: Long = 3L
    }
 }
@@ -9,11 +9,8 @@ import org.gradle.api.Project
 import org.gradle.api.logging.Logging
 import org.gradle.tooling.events.task.TaskFinishEvent
 import org.jetbrains.kotlin.build.report.metrics.ValueType
-import org.jetbrains.kotlin.build.report.statistics.HttpReportService
-import org.jetbrains.kotlin.build.report.statistics.formatSize
-import org.jetbrains.kotlin.build.report.statistics.BuildFinishStatisticsData
-import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters
-import org.jetbrains.kotlin.build.report.statistics.StatTag
+import org.jetbrains.kotlin.build.report.statistics.*
+import org.jetbrains.kotlin.build.report.statistics.file.ReadableFileReportData
 import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
 import org.jetbrains.kotlin.gradle.plugin.statistics.GradleFileReportService
 import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
@@ -69,11 +66,13 @@ class BuildReportsService {
                it.buildReportDir,
                parameters.projectName,
                it.includeMetricsInReport,
-                loggerAdapter
            ).process(
-                transformOperationRecordsToCompileStatisticsData(buildOperationRecords, parameters, onlyKotlinTask = false),
-                parameters.startParameters,
-                failureMessages.filter { it.isNotEmpty() },
+                ReadableFileReportData(
+                    transformOperationRecordsToCompileStatisticsData(buildOperationRecords, parameters, onlyKotlinTask = false),
+                    parameters.startParameters,
+                    failureMessages.filter { it.isNotEmpty() },
+                ),
+                loggerAdapter
            )
        }

@@ -85,6 +84,10 @@ class BuildReportsService {
            reportTryNextToConsole(buildData)
        }

+        reportingSettings.jsonOutputDir?.also {
+            JsonReportService(it, parameters.projectName).process(buildData, loggerAdapter)
+        }
+
        //It's expected that bad internet connection can cause a significant delay for big project
        executorService.shutdown()
    }
@@ -18,6 +18,7 @@ data class ReportingSettings(
    val httpReportSettings: HttpReportSettings? = null,
    val buildScanReportSettings: BuildScanSettings? = null,
    val singleOutputFile: File? = null,
+    val jsonOutputDir: File? = null,
    val experimentalTryNextConsoleOutput: Boolean = false,
    val includeCompilerArguments: Boolean = false,
 ) : Serializable {
@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_SINGLE_FILE
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_HTTP_URL
+import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_JSON_DIR
 import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled
 import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly

@@ -78,6 +79,11 @@ internal fun reportingSettings(project: Project): ReportingSettings {
            ?: throw IllegalStateException("Can't configure single file report: '$KOTLIN_BUILD_REPORT_SINGLE_FILE' property is mandatory")
    } else null

+    val jsonReportDir = if (buildReportOutputTypes.contains(BuildReportType.JSON)) {
+        properties.buildReportJsonDir
+            ?: throw IllegalStateException("Can't configure json report: '$KOTLIN_BUILD_REPORT_JSON_DIR' property is mandatory")
+    } else null
+
    //temporary solution. support old property
    @Suppress("DEPRECATION")
    val oldSingleBuildMetric = properties.singleBuildMetricsFile?.also { buildReportOutputTypes.add(BuildReportType.SINGLE_FILE) }
@@ -90,6 +96,7 @@ internal fun reportingSettings(project: Project): ReportingSettings {
        buildScanReportSettings = buildScanSettings,
        buildReportOutputs = buildReportOutputTypes,
        singleOutputFile = singleOutputFile ?: oldSingleBuildMetric,
+        jsonOutputDir = jsonReportDir,
        includeCompilerArguments = properties.buildReportIncludeCompilerArguments,
        experimentalTryNextConsoleOutput = experimentalTryNextEnabled
    )
@@ -10,14 +10,13 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
 import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
 import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters

-class BuildExecutionData(
+data class BuildExecutionData(
    val startParameters: BuildStartParameters,
    val failureMessages: List<String?>,
-    val buildOperationRecord: Collection<BuildOperationRecord>
+    val buildOperationRecord: Collection<BuildOperationRecord>,
 ) {
-    val aggregatedMetrics by lazy {
-        BuildMetrics<GradleBuildTime, GradleBuildPerformanceMetric>().also { acc ->
-            buildOperationRecord.forEach { acc.addAll(it.buildMetrics) }
-        }
+    val aggregatedMetrics = BuildMetrics<GradleBuildTime, GradleBuildPerformanceMetric>().also { acc ->
+        buildOperationRecord.forEach { acc.addAll(it.buildMetrics) }
    }
+
 }
@@ -52,6 +52,7 @@ enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: Bo
    BUILD_SCAN_BUILD_REPORT(OR, SAFE),
    HTTP_BUILD_REPORT(OR, SAFE),
    SINGLE_FILE_BUILD_REPORT(OR, SAFE),
+    JSON_BUILD_REPORT(OR, SAFE),

    //Dokka features
    ENABLED_DOKKA(OR, SAFE),
@@ -80,6 +81,6 @@ enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: Bo
    COCOAPODS_PLUGIN_ENABLED(OR, SAFE);

    companion object {
-        const val VERSION = 3
+        const val VERSION = 4
    }
 }
@@ -23,7 +23,7 @@ private const val STRING_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/Str
 private const val NUMERICAL_METRICS_RELATIVE_PATH = "$SOURCE_CODE_RELATIVE_PATH/NumericalMetrics.kt"

 private val STRING_METRICS_EXPECTED_VERSION_AND_HASH = Pair(1, "90347332db2ce54b51e7daa64595371e")
-private val BOOLEAN_METRICS_EXPECTED_VERSION_AND_HASH = Pair(3, "3c8c4ca636adee168e99862244a22520")
+private val BOOLEAN_METRICS_EXPECTED_VERSION_AND_HASH = Pair(4, "56284d1f90da498710ecf3207c781dd7")
 private val NUMERICAL_METRICS_EXPECTED_VERSION_AND_HASH = Pair(2, "d8c1a1f4fb7227fbe8247320bf3370ca")
 private val SOURCE_FOLDER_EXPECTED_VERSION_AND_HASH =
    Pair(