Collecting performance information in Gradle plugin supported

The build performance information is collected during the build
when the Kotlin Gradle plugin is applied. Only hashed values are
saved if they may contain sensitive information. Numeric metrics
obtained on the basis of user's project are saved with random seed.
Persisted information is saved in gradleUserHomeDir.

#KT-33404 Fixed

libraries/tools/kotlin-gradle-plugin/build.gradle.kts

@@ -44,6 +44,8 @@ dependencies {
     compileOnly(project(":kotlin-annotation-processing"))
     compileOnly(project(":kotlin-annotation-processing-gradle"))
     compileOnly(project(":kotlin-scripting-compiler"))
+    compileOnly(project(":kotlin-gradle-statistics"))
+    embedded(project(":kotlin-gradle-statistics"))
 
     compile("com.google.code.gson:gson:${rootProject.extra["versions.jar.gson"]}")
     compile("de.undercouch:gradle-download-task:4.0.2")

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt

@@ -29,6 +29,7 @@ import org.jetbrains.kotlin.gradle.logging.kotlinDebug
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinUsages
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory
+import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
 import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin
 import org.jetbrains.kotlin.gradle.targets.js.npm.addNpmDependencyExtension
 import org.jetbrains.kotlin.gradle.tasks.KOTLIN_COMPILER_EMBEDDABLE
@@ -36,6 +37,9 @@ import org.jetbrains.kotlin.gradle.tasks.KOTLIN_MODULE_GROUP
 import org.jetbrains.kotlin.gradle.testing.internal.KotlinTestsRegistry
 import org.jetbrains.kotlin.gradle.utils.checkGradleCompatibility
 import org.jetbrains.kotlin.gradle.utils.loadPropertyFromResources
+import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
+import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
+import org.jetbrains.kotlin.statistics.metrics.StringMetrics
 import javax.inject.Inject
 import kotlin.reflect.KClass
 
@@ -51,6 +55,8 @@ abstract class KotlinBasePluginWrapper(
         DefaultKotlinSourceSetFactory(project, fileResolver)
 
     override fun apply(project: Project) {
+        val statisticsReporter = KotlinBuildStatsService.getOrCreateInstance(project.gradle)
+
         checkGradleCompatibility()
 
         project.configurations.maybeCreate(COMPILER_CLASSPATH_CONFIGURATION_NAME).defaultDependencies {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/DaemonReusageCounter.kt.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.statistics
+
+import java.lang.management.ManagementFactory
+import java.util.concurrent.atomic.AtomicLong
+import javax.management.MBeanServer
+import javax.management.ObjectName
+import javax.management.StandardMBean
+
+interface IDaemonReuseCounterMXBean {
+
+    fun getOrdinal(): Long
+
+    fun incrementAndGetOrdinal(): Long
+
+}
+
+/**
+ * This class is used for counting builds involving Kotlin withing one Gradle daemon
+ * One instance is kept available via JMX bean after build completion. If other builds are executed with other version
+ * of Kotlin plugin, they are still able to reuse this build counter vis JXM interface
+ */
+class DaemonReuseCounter private constructor() : IDaemonReuseCounterMXBean {
+    private val count = AtomicLong()
+
+    override fun getOrdinal(): Long {
+        return count.get()
+    }
+
+    override fun incrementAndGetOrdinal(): Long {
+        return count.incrementAndGet()
+    }
+
+    companion object {
+        // Do not rename this bean otherwise compatibility with the older Kotlin Gradle Plugins would be lost
+        const val JMX_BEAN_NAME = "org.jetbrains.kotlin.gradle.plugin.statistics:type=BuildCounter"
+
+        private fun ensureRegistered(beanName: ObjectName, mbs: MBeanServer) {
+            if (!mbs.isRegistered(beanName)) {
+                mbs.registerMBean(StandardMBean(DaemonReuseCounter(), IDaemonReuseCounterMXBean::class.java), beanName)
+            }
+        }
+
+        fun incrementAndGetOrdinal(): Long {
+            val beanName = ObjectName(JMX_BEAN_NAME)
+            val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
+            ensureRegistered(beanName, mbs)
+            return mbs.invoke(beanName, "incrementAndGetOrdinal", emptyArray(), emptyArray<String>()) as? Long ?: 0
+        }
+
+        fun getOrdinal(): Long {
+            val beanName = ObjectName(KotlinBuildStatsService.JMX_BEAN_NAME)
+            val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
+            ensureRegistered(beanName, mbs)
+            return mbs.invoke(beanName, "getOrdinal", emptyArray(), emptyArray<String>()) as? Long ?: 0
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt

@@ -0,0 +1,296 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.statistics
+
+import org.gradle.BuildAdapter
+import org.gradle.BuildResult
+import org.gradle.api.invocation.Gradle
+import org.gradle.api.logging.Logging
+import org.gradle.initialization.BuildCompletionListener
+import org.gradle.initialization.BuildRequestMetaData
+import org.gradle.invocation.DefaultGradle
+import org.jetbrains.kotlin.statistics.BuildSessionLogger
+import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME
+import org.jetbrains.kotlin.statistics.metrics.*
+import java.io.File
+import java.lang.management.ManagementFactory
+import javax.management.MBeanServer
+import javax.management.ObjectName
+import javax.management.StandardMBean
+
+/**
+ * Interface for populating statistics collection method via JXM interface
+ * JMX could be used for reporting both from other JVMs, other versions
+ * of Kotlin Plugin and other classloaders
+ */
+interface KotlinBuildStatsMXBean {
+
+    fun reportBoolean(name: String, value: Boolean, subprojectName: String?)
+
+    fun reportNumber(name: String, value: Long, subprojectName: String?)
+
+    fun reportString(name: String, value: String, subprojectName: String?)
+
+}
+
+
+internal abstract class KotlinBuildStatsService internal constructor() : BuildAdapter(), IStatisticsValuesConsumer,
+    BuildCompletionListener {
+
+    companion object {
+
+        // Do not rename this bean otherwise compatibility with the older Kotlin Gradle Plugins would be lost
+        const val JMX_BEAN_NAME = "org.jetbrains.kotlin.gradle.plugin.statistics:type=StatsService"
+
+        // Property name for disabling saving statistical information
+        const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile"
+
+        // default state
+        const val DEFAULT_STATISTICS_STATE = true
+
+        // "emergency file" collecting statistics is disabled it the file exists
+        const val DISABLE_STATISTICS_FILE_NAME = "${STATISTICS_FOLDER_NAME}/.disable"
+
+
+        /**
+         * Method for getting IStatisticsValuesConsumer for reporting some statistics
+         * Could be invoked during any build phase after applying first Kotlin plugin and
+         * until build is completed
+         */
+        @JvmStatic
+        @Synchronized
+        fun getInstance(): IStatisticsValuesConsumer? {
+            if (statisticsIsEnabled != true) {
+                return null
+            }
+            return instance
+        }
+
+        /**
+         * Method for creating new instance of IStatisticsValuesConsumer
+         * It could be invoked only when applying Kotlin gradle plugin.
+         * When executed, this method checks, whether it is already executed in current build.
+         * If it was not executed, the new instance of IStatisticsValuesConsumer is created
+         *
+         * If it was already executed in the same classpath (i.e. with the same version of Kotlin plugin),
+         * the previously returned instance is returned.
+         *
+         * If it was already executed in the other classpath, a JXM implementation is returned.
+         *
+         * All the created instances are registered as build listeners
+         */
+        @JvmStatic
+        @Synchronized
+        internal fun getOrCreateInstance(gradle: Gradle): IStatisticsValuesConsumer? {
+            return runSafe("${KotlinBuildStatsService::class.java}.getOrCreateInstance") {
+                statisticsIsEnabled = statisticsIsEnabled ?: checkStatisticsEnabled(gradle)
+                if (statisticsIsEnabled != true) {
+                    null
+                } else {
+                    val log = getLogger()
+
+                    if (instance != null) {
+                        log.debug("${KotlinBuildStatsService::class.java} is already instantiated. Current instance is $instance")
+                    } else {
+                        val beanName = ObjectName(JMX_BEAN_NAME)
+                        val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
+                        if (mbs.isRegistered(beanName)) {
+                            log.debug(
+                                "${KotlinBuildStatsService::class.java} is already instantiated in another classpath. Creating JMX-wrapper"
+                            )
+                            instance = JMXKotlinBuildStatsService(mbs, beanName)
+                        } else {
+                            val newInstance = DefaultKotlinBuildStatsService(gradle, beanName)
+                            instance = newInstance
+                            log.debug("Instantiated ${KotlinBuildStatsService::class.java}: new instance $instance")
+                            mbs.registerMBean(StandardMBean(newInstance, KotlinBuildStatsMXBean::class.java), beanName)
+                        }
+                    }
+                    gradle.addBuildListener(instance)
+                    instance
+                }
+            }
+        }
+
+        @JvmStatic
+        internal fun getLogger() = Logging.getLogger(KotlinBuildStatsService::class.java)
+
+        internal var instance: KotlinBuildStatsService? = null
+
+        private var statisticsIsEnabled: Boolean? = null
+
+        private fun checkStatisticsEnabled(gradle: Gradle): Boolean {
+            return if (File(gradle.gradleUserHomeDir, DISABLE_STATISTICS_FILE_NAME).exists()) {
+                false
+            } else {
+                gradle.rootProject.properties[ENABLE_STATISTICS_PROPERTY_NAME]?.toString()?.toBoolean() ?: DEFAULT_STATISTICS_STATE
+            }
+        }
+
+        private fun logException(description: String, e: Throwable) {
+            getLogger().info(description)
+            getLogger().debug(e.message, e)
+        }
+
+        internal fun <T> runSafe(methodName: String, action: () -> T?): T? {
+            return try {
+                getLogger().debug("Executing [$methodName]")
+                action.invoke()
+            } catch (e: Throwable) {
+                logException("Could not execute [$methodName]", e)
+                null
+            }
+        }
+    }
+
+
+}
+
+internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) :
+    KotlinBuildStatsService() {
+
+    override fun buildFinished(result: BuildResult) {
+    }
+
+    private fun callJmx(method: String, type: String, metricName: String, value: Any, subprojectName: String?) {
+        mbs.invoke(
+            beanName,
+            method,
+            arrayOf(metricName, value, subprojectName),
+            arrayOf("java.lang.String", type, "java.lang.String")
+        )
+    }
+
+    override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) {
+        runSafe("report metric ${metric.name}") {
+            callJmx("reportBoolean", "boolean", metric.name, value, subprojectName)
+        }
+    }
+
+    override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) {
+        runSafe("report metric ${metric.name}") {
+            callJmx("reportNumber", "long", metric.name, value, subprojectName)
+        }
+    }
+
+    override fun report(metric: StringMetrics, value: String, subprojectName: String?) {
+        runSafe("report metric ${metric.name}") {
+            callJmx("reportString", "java.lang.String", metric.name, value, subprojectName)
+        }
+    }
+
+    override fun completed() {
+        instance = null
+    }
+
+}
+
+internal class DefaultKotlinBuildStatsService internal constructor(
+    gradle: Gradle,
+    val beanName: ObjectName
+) : KotlinBuildStatsService(), KotlinBuildStatsMXBean {
+
+    private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir)
+
+    private fun gradleBuildStartTime(gradle: Gradle): Long? {
+        return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime
+    }
+
+    private fun reportGlobalMetrics(gradle: Gradle) {
+        System.getProperty("os.name")?.also {
+            sessionLogger.report(StringMetrics.OS_TYPE, gradle.gradleVersion)
+        }
+        sessionLogger.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong())
+        sessionLogger.report(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
+        sessionLogger.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null)
+
+
+        gradle.allprojects { project ->
+            for (configuration in project.configurations) {
+                val configurationName = configuration.name
+                val dependencies = configuration.dependencies
+
+                when (configurationName) {
+                    "kapt" -> {
+                        sessionLogger.report(BooleanMetrics.ENABLED_KAPT, true)
+                        dependencies.forEach { dependency ->
+                            when (dependency.group) {
+                                "com.google.dagger" -> sessionLogger.report(BooleanMetrics.ENABLED_DAGGER, true)
+                                "com.android.databinding" -> sessionLogger.report(BooleanMetrics.ENABLED_DATABINDING, true)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    override fun projectsEvaluated(gradle: Gradle) {
+        runSafe("${DefaultKotlinBuildStatsService::class.java}.projectEvaluated") {
+            if (!sessionLogger.isBuildSessionStarted()) {
+                sessionLogger.startBuildSession(
+                    DaemonReuseCounter.incrementAndGetOrdinal(),
+                    gradleBuildStartTime(gradle)
+                )
+                reportGlobalMetrics(gradle)
+            }
+        }
+    }
+
+    @Synchronized
+    override fun buildFinished(result: BuildResult) {
+        runSafe("${DefaultKotlinBuildStatsService::class.java}.buildFinished") {
+            sessionLogger.finishBuildSession(result.action, result.failure)
+        }
+    }
+
+
+    @Synchronized
+    override fun completed() {
+        runSafe("${DefaultKotlinBuildStatsService::class.java}.completed") {
+            try {
+                sessionLogger.unlockJournalFile()
+            } finally {
+                val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
+                if (mbs.isRegistered(beanName)) {
+                    mbs.unregisterMBean(beanName)
+                }
+                instance = null
+            }
+        }
+    }
+
+    override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) {
+        runSafe("report metric ${metric.name}") {
+            sessionLogger.report(metric, value, subprojectName)
+        }
+
+    }
+
+    override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) {
+        runSafe("report metric ${metric.name}") {
+            sessionLogger.report(metric, value, subprojectName)
+        }
+    }
+
+    override fun report(metric: StringMetrics, value: String, subprojectName: String?) {
+        runSafe("report metric ${metric.name}") {
+            sessionLogger.report(metric, value, subprojectName)
+        }
+    }
+
+    override fun reportBoolean(name: String, value: Boolean, subprojectName: String?) {
+        report(BooleanMetrics.valueOf(name), value, subprojectName)
+    }
+
+    override fun reportNumber(name: String, value: Long, subprojectName: String?) {
+        report(NumericalMetrics.valueOf(name), value, subprojectName)
+    }
+
+    override fun reportString(name: String, value: String, subprojectName: String?) {
+        report(StringMetrics.valueOf(name), value, subprojectName)
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt

@@ -7,8 +7,10 @@ import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.OutputDirectory
 import org.gradle.api.tasks.TaskAction
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
+import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
 import org.jetbrains.kotlin.gradle.utils.isGradleVersionAtLeast
 import org.jetbrains.kotlin.gradle.utils.patternLayoutCompatible
+import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
 import java.io.File
 import java.net.URI
 
@@ -52,7 +54,13 @@ open class NodeJsSetupTask : DefaultTask() {
         val dep = this.project.dependencies.create(ivyDependency)
         val conf = this.project.configurations.detachedConfiguration(dep)
         conf.isTransitive = false
+
+        val startDownloadTime = System.currentTimeMillis()
         val result = conf.resolve().single()
+
+        KotlinBuildStatsService.getInstance()
+            ?.report(NumericalMetrics.ARTIFACTS_DOWNLOAD_SPEED, result.length() * 1000 / (System.currentTimeMillis() - startDownloadTime))
+
         project.repositories.remove(repo)
 
         project.logger.kotlinInfo("Using node distribution from '$result'")

libraries/tools/kotlin-gradle-statistics/build.gradle.kts

@@ -0,0 +1,20 @@
+description = "kotlin-gradle-statistics"
+
+plugins {
+    kotlin("jvm")
+}
+
+dependencies {
+    compileOnly(kotlinStdlib())
+}
+
+sourceSets {
+    "main" { projectDefault() }
+    "test" { projectDefault() }
+}
+
+projectTest {
+    workingDir = rootDir
+}
+
+runtimeJar(rewriteDefaultJarDepsToShadedCompiler())

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSession.kt

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics
+
+class BuildSession(val buildStartedTime: Long?) {
+
+    val projectEvaluatedTime = System.currentTimeMillis()
+
+}

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics
+
+import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger
+import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger
+import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
+import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger
+import org.jetbrains.kotlin.statistics.metrics.*
+import java.io.File
+import java.io.IOException
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+
+class BuildSessionLogger(
+    private val rootPath: File,
+    private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
+    private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE
+) : IStatisticsValuesConsumer {
+
+    companion object {
+        const val STATISTICS_FOLDER_NAME = "kotlin-profile"
+        const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}.profile"
+
+        private const val DEFAULT_MAX_PROFILE_FILES = 1_000
+        private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
+    }
+
+    private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS'.profile'")
+    private val statisticsFolder: File = File(
+        rootPath,
+        STATISTICS_FOLDER_NAME
+    ).also { it.mkdirs() }
+
+    private var buildSession: BuildSession? = null
+    private var trackingFile: IRecordLogger? = null
+
+    private val metricsContainer = MetricsContainer()
+
+    @Synchronized
+    fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
+        report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart)
+
+        buildSession = BuildSession(buildStartedTime)
+        initTrackingFile()
+    }
+
+    @Synchronized
+    fun isBuildSessionStarted() = buildSession != null
+
+    @Synchronized
+    private fun closeTrackingFile() {
+        metricsContainer.flush(trackingFile)
+        trackingFile?.close()
+        trackingFile = null
+    }
+
+    @Synchronized
+    private fun initTrackingFile() {
+        closeTrackingFile()
+
+        // Get list of existing files. Try to create folder if possible, return from function if failed to create folder
+        val fileCandidates =
+            statisticsFolder.listFiles()?.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.toMutableList()
+                ?: if (statisticsFolder.mkdirs()) emptyList<File>() else return
+
+        for (i in 0 until fileCandidates.size - maxProfileFiles) {
+            val file2delete = fileCandidates[i]
+            if (file2delete.isFile) {
+                file2delete.delete()
+            }
+        }
+
+        // emergency check. What if a lot of files are locked due to some reason
+        if (statisticsFolder.listFiles()?.size ?: 0 > maxProfileFiles * 2) {
+            return
+        }
+
+        fun newFile(): File = File(statisticsFolder, profileFileNameFormatter.format(LocalDateTime.now()))
+        val lastFile = fileCandidates.lastOrNull() ?: newFile()
+
+        trackingFile = try {
+            if (lastFile.length() < maxFileSize) {
+                FileRecordLogger(lastFile)
+            } else {
+                FileRecordLogger(newFile())
+            }
+        } catch (e: IOException) {
+            try {
+                FileRecordLogger(newFile())
+            } catch (e: IOException) {
+                NullRecordLogger()
+            }
+        }
+    }
+
+    @Synchronized
+    fun finishBuildSession(action: String?, failure: Throwable?) {
+        // nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
+        // could be affected by system clock correction
+        val finishTime = System.currentTimeMillis()
+        buildSession?.also {
+            if (it.buildStartedTime != null) {
+                report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
+            }
+            report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
+        }
+        buildSession = null
+    }
+
+    @Synchronized
+    fun unlockJournalFile() {
+        closeTrackingFile()
+    }
+
+    override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) {
+        metricsContainer.report(metric, value, subprojectName)
+    }
+
+    override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) {
+        metricsContainer.report(metric, value, subprojectName)
+    }
+
+    override fun report(metric: StringMetrics, value: String, subprojectName: String?) {
+        metricsContainer.report(metric, value, subprojectName)
+    }
+}

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/FileRecordLogger.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.fileloggers
+
+import java.io.File
+import java.io.IOException
+import java.io.OutputStream
+import java.nio.channels.Channels
+import java.nio.channels.FileChannel
+import java.nio.channels.FileLock
+import java.nio.file.Paths
+import java.nio.file.StandardOpenOption
+
+class FileRecordLogger(file: File) : IRecordLogger {
+
+    private val channel: FileChannel =
+        FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
+            ?: throw IOException("Could not open file $file")
+
+    private val outputStream: OutputStream
+    private val lock: FileLock
+
+    init {
+        lock = try {
+            channel.lock()
+        } catch (e: Exception) {
+            channel.close()
+            // wrap in order to unify with FileOverlappingException
+            throw IOException(e.message, e)
+        }
+        outputStream = Channels.newOutputStream(channel)
+    }
+
+    override fun append(s: String) {
+        outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING))
+    }
+
+    override fun close() {
+        channel.use {
+            outputStream.use {
+                lock.release()
+            }
+        }
+    }
+}

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/IRecordLogger.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.fileloggers
+
+import java.io.Closeable
+
+interface IRecordLogger : Closeable {
+
+    fun append(s: String)
+
+}

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.fileloggers
+
+import org.jetbrains.kotlin.statistics.metrics.*
+import org.jetbrains.kotlin.statistics.sha256
+import java.io.*
+import java.nio.channels.Channels
+import java.nio.channels.FileChannel
+import java.nio.file.Paths
+import java.nio.file.StandardOpenOption
+import java.util.*
+
+class MetricsContainer : IStatisticsValuesConsumer {
+
+    data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
+        override fun compareTo(other: MetricDescriptor): Int {
+            return when {
+                name.compareTo(other.name) != 0 -> name.compareTo(other.name)
+                projectHash == other.projectHash -> 0
+                else -> (projectHash ?: "").compareTo(other.projectHash ?: "")
+            }
+        }
+    }
+
+    private val numericalMetrics = TreeMap<MetricDescriptor, IMetricContainer<Long>>()
+
+    private val booleanMetrics = TreeMap<MetricDescriptor, IMetricContainer<Boolean>>()
+
+    private val stringMetrics = TreeMap<MetricDescriptor, IMetricContainer<String>>()
+
+    companion object {
+
+        private const val BUILD_SESSION_SEPARATOR = "BUILD FINISHED"
+
+        val ENCODING = Charsets.UTF_8
+
+        private val stringMetricsMap = StringMetrics.values().map { m -> m.name to m }.toMap()
+
+        private val booleanMetricsMap = BooleanMetrics.values().map { m -> m.name to m }.toMap()
+
+        private val numericalMetricsMap = NumericalMetrics.values().map { m -> m.name to m }.toMap()
+
+        fun readFromFile(file: File, consumer: (MetricsContainer) -> Unit) {
+            val channel = FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.READ)
+            channel.lock()
+
+            val inputStream = Channels.newInputStream(channel)
+            try {
+                var container = MetricsContainer()
+                // Note: close is called at forEachLine
+                BufferedReader(InputStreamReader(inputStream, ENCODING)).forEachLine { line ->
+                    if (BUILD_SESSION_SEPARATOR == line) {
+                        consumer.invoke(container)
+                        container = MetricsContainer()
+                    } else {
+                        // format: metricName.hash=string representation
+                        val lineParts = line.split('=')
+                        if (lineParts.size == 2) {
+                            val name = lineParts[0].split('.')[0]
+                            val subProjectHash = lineParts[0].split('.').getOrNull(1)
+                            val representation = lineParts[1]
+
+                            stringMetricsMap[name]?.also { metricType ->
+                                metricType.type.fromStringRepresentation(representation)?.also {
+                                    container.stringMetrics[MetricDescriptor(name, subProjectHash)] = it
+                                }
+                            }
+
+                            booleanMetricsMap[name]?.also { metricType ->
+                                metricType.type.fromStringRepresentation(representation)?.also {
+                                    container.booleanMetrics[MetricDescriptor(name, subProjectHash)] = it
+                                }
+                            }
+
+                            numericalMetricsMap[name]?.also { metricType ->
+                                metricType.type.fromStringRepresentation(representation)?.also {
+                                    container.numericalMetrics[MetricDescriptor(name, subProjectHash)] = it
+                                }
+                            }
+                        }
+                    }
+                }
+            } finally {
+                channel.close()
+            }
+        }
+    }
+
+    private fun processProjectName(subprojectName: String?, perProject: Boolean) =
+        if (perProject && subprojectName != null) sha256(subprojectName) else null
+
+    override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) {
+        val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject)
+        val metricContainer = booleanMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
+            .also { booleanMetrics[MetricDescriptor(metric.name, projectHash)] = it }
+        metricContainer.addValue(metric.anonymization.anonymize(value))
+    }
+
+    override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) {
+        val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject)
+        val metricContainer = numericalMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
+            .also { numericalMetrics[MetricDescriptor(metric.name, projectHash)] = it }
+        metricContainer.addValue(metric.anonymization.anonymize(value))
+    }
+
+    override fun report(metric: StringMetrics, value: String, subprojectName: String?) {
+        val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject)
+        val metricContainer = stringMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
+            .also { stringMetrics[MetricDescriptor(metric.name, projectHash)] = it }
+        metricContainer.addValue(metric.anonymization.anonymize(value))
+    }
+
+    fun flush(trackingFile: IRecordLogger?) {
+        if (trackingFile == null) return
+        for (entry in numericalMetrics.entries.union(booleanMetrics.entries).union(stringMetrics.entries)) {
+            val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}"
+            trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
+        }
+
+        trackingFile.append(BUILD_SESSION_SEPARATOR)
+
+        stringMetrics.clear()
+        booleanMetrics.clear()
+        numericalMetrics.clear()
+    }
+
+    fun getMetric(metric: NumericalMetrics): IMetricContainer<Long>? = numericalMetrics[MetricDescriptor(metric.name, null)]
+
+    fun getMetric(metric: StringMetrics): IMetricContainer<String>? = stringMetrics[MetricDescriptor(metric.name, null)]
+
+    fun getMetric(metric: BooleanMetrics): IMetricContainer<Boolean>? = booleanMetrics[MetricDescriptor(metric.name, null)]
+}

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/NullRecordLogger.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.fileloggers
+
+class NullRecordLogger : IRecordLogger {
+    override fun append(s: String) {}
+
+    override fun close() {
+    }
+}

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.metrics
+
+interface IMetricContainer<T> {
+    fun addValue(t: T)
+
+    fun toStringRepresentation(): String
+
+    fun getValue(): T?
+
+}
+
+interface IMetricContainerFactory<T> {
+    fun newMetricContainer(): IMetricContainer<T>
+
+    //null if could not parse
+    fun fromStringRepresentation(state: String): IMetricContainer<T>?
+}
+
+class OverrideMetricContainer<T>() : IMetricContainer<T> {
+
+    private var myValue: T? = null
+
+    override fun addValue(t: T) {
+        myValue = t
+    }
+
+    internal constructor(v: T?) : this() {
+        myValue = v
+    }
+
+    override fun toStringRepresentation(): String {
+        return myValue?.toString() ?: "null"
+    }
+
+    override fun getValue() = myValue
+}
+
+class ConcatMetricContainer() : IMetricContainer<String> {
+    private val myValues = HashSet<String>()
+
+    override fun addValue(t: String) {
+        myValues.add(t)
+    }
+
+    override fun toStringRepresentation(): String {
+        return myValues.joinToString(";")
+    }
+
+    override fun getValue() = toStringRepresentation()
+}
+

libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StatisticsValues.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.metrics
+
+interface ReportStatisticsValue<T> {
+    val name: String
+    val value: T
+}
+
+class ReportOnceStatisticsValue<T>(override val name: String, override val value: T) :
+    ReportStatisticsValue<T>
+
+interface AdditiveStatisticsValue<T> : ReportStatisticsValue<T> {
+    fun addValue(t: T)
+}
+
+interface IStatisticsValuesConsumer {
+
+    fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null)
+
+    fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null)
+
+    fun report(metric: StringMetrics, value: String, subprojectName: String? = null)
+
+}

settings.gradle

@@ -219,6 +219,7 @@ include ":kotlin-build-common",
         ":tools:kotlinp",
         ":kotlin-gradle-plugin-api",
         ":kotlin-gradle-plugin-dsl-codegen",
+        ":kotlin-gradle-statistics",
         ":kotlin-gradle-plugin",
         ":kotlin-gradle-plugin-model",
         ":kotlin-gradle-plugin-test-utils-embeddable",
@@ -421,6 +422,7 @@ project(':kotlin-source-sections-compiler-plugin').projectDir = "$rootDir/plugin
 project(':tools:kotlinp').projectDir = "$rootDir/libraries/tools/kotlinp" as File
 project(':kotlin-gradle-plugin-api').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-api" as File
 project(':kotlin-gradle-plugin-dsl-codegen').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-dsl-codegen" as File
+project(':kotlin-gradle-statistics').projectDir = "$rootDir/libraries/tools/kotlin-gradle-statistics" as File
 project(':kotlin-gradle-plugin').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin" as File
 project(':kotlin-gradle-plugin-model').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-model" as File
 project(':kotlin-gradle-plugin-test-utils-embeddable').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-test-utils-embeddable" as File