diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index b93f4a92351..dabb45316c0 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -44,6 +44,8 @@ dependencies { compileOnly(project(":kotlin-annotation-processing")) compileOnly(project(":kotlin-annotation-processing-gradle")) compileOnly(project(":kotlin-scripting-compiler")) + compileOnly(project(":kotlin-gradle-statistics")) + embedded(project(":kotlin-gradle-statistics")) compile("com.google.code.gson:gson:${rootProject.extra["versions.jar.gson"]}") compile("de.undercouch:gradle-download-task:4.0.2") diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt index 025c1ba367f..e18b1c1e92a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt @@ -29,6 +29,7 @@ import org.jetbrains.kotlin.gradle.logging.kotlinDebug import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinUsages import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory +import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin import org.jetbrains.kotlin.gradle.targets.js.npm.addNpmDependencyExtension import org.jetbrains.kotlin.gradle.tasks.KOTLIN_COMPILER_EMBEDDABLE @@ -36,6 +37,9 @@ import org.jetbrains.kotlin.gradle.tasks.KOTLIN_MODULE_GROUP import org.jetbrains.kotlin.gradle.testing.internal.KotlinTestsRegistry import org.jetbrains.kotlin.gradle.utils.checkGradleCompatibility import org.jetbrains.kotlin.gradle.utils.loadPropertyFromResources +import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics +import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics +import org.jetbrains.kotlin.statistics.metrics.StringMetrics import javax.inject.Inject import kotlin.reflect.KClass @@ -51,6 +55,8 @@ abstract class KotlinBasePluginWrapper( DefaultKotlinSourceSetFactory(project, fileResolver) override fun apply(project: Project) { + val statisticsReporter = KotlinBuildStatsService.getOrCreateInstance(project.gradle) + checkGradleCompatibility() project.configurations.maybeCreate(COMPILER_CLASSPATH_CONFIGURATION_NAME).defaultDependencies { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/DaemonReusageCounter.kt.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/DaemonReusageCounter.kt.kt new file mode 100644 index 00000000000..87219aa1039 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/DaemonReusageCounter.kt.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import java.lang.management.ManagementFactory +import java.util.concurrent.atomic.AtomicLong +import javax.management.MBeanServer +import javax.management.ObjectName +import javax.management.StandardMBean + +interface IDaemonReuseCounterMXBean { + + fun getOrdinal(): Long + + fun incrementAndGetOrdinal(): Long + +} + +/** + * This class is used for counting builds involving Kotlin withing one Gradle daemon + * One instance is kept available via JMX bean after build completion. If other builds are executed with other version + * of Kotlin plugin, they are still able to reuse this build counter vis JXM interface + */ +class DaemonReuseCounter private constructor() : IDaemonReuseCounterMXBean { + private val count = AtomicLong() + + override fun getOrdinal(): Long { + return count.get() + } + + override fun incrementAndGetOrdinal(): Long { + return count.incrementAndGet() + } + + companion object { + // Do not rename this bean otherwise compatibility with the older Kotlin Gradle Plugins would be lost + const val JMX_BEAN_NAME = "org.jetbrains.kotlin.gradle.plugin.statistics:type=BuildCounter" + + private fun ensureRegistered(beanName: ObjectName, mbs: MBeanServer) { + if (!mbs.isRegistered(beanName)) { + mbs.registerMBean(StandardMBean(DaemonReuseCounter(), IDaemonReuseCounterMXBean::class.java), beanName) + } + } + + fun incrementAndGetOrdinal(): Long { + val beanName = ObjectName(JMX_BEAN_NAME) + val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() + ensureRegistered(beanName, mbs) + return mbs.invoke(beanName, "incrementAndGetOrdinal", emptyArray(), emptyArray()) as? Long ?: 0 + } + + fun getOrdinal(): Long { + val beanName = ObjectName(KotlinBuildStatsService.JMX_BEAN_NAME) + val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() + ensureRegistered(beanName, mbs) + return mbs.invoke(beanName, "getOrdinal", emptyArray(), emptyArray()) as? Long ?: 0 + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt new file mode 100644 index 00000000000..8d337418d9f --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt @@ -0,0 +1,296 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import org.gradle.BuildAdapter +import org.gradle.BuildResult +import org.gradle.api.invocation.Gradle +import org.gradle.api.logging.Logging +import org.gradle.initialization.BuildCompletionListener +import org.gradle.initialization.BuildRequestMetaData +import org.gradle.invocation.DefaultGradle +import org.jetbrains.kotlin.statistics.BuildSessionLogger +import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME +import org.jetbrains.kotlin.statistics.metrics.* +import java.io.File +import java.lang.management.ManagementFactory +import javax.management.MBeanServer +import javax.management.ObjectName +import javax.management.StandardMBean + +/** + * Interface for populating statistics collection method via JXM interface + * JMX could be used for reporting both from other JVMs, other versions + * of Kotlin Plugin and other classloaders + */ +interface KotlinBuildStatsMXBean { + + fun reportBoolean(name: String, value: Boolean, subprojectName: String?) + + fun reportNumber(name: String, value: Long, subprojectName: String?) + + fun reportString(name: String, value: String, subprojectName: String?) + +} + + +internal abstract class KotlinBuildStatsService internal constructor() : BuildAdapter(), IStatisticsValuesConsumer, + BuildCompletionListener { + + companion object { + + // Do not rename this bean otherwise compatibility with the older Kotlin Gradle Plugins would be lost + const val JMX_BEAN_NAME = "org.jetbrains.kotlin.gradle.plugin.statistics:type=StatsService" + + // Property name for disabling saving statistical information + const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile" + + // default state + const val DEFAULT_STATISTICS_STATE = true + + // "emergency file" collecting statistics is disabled it the file exists + const val DISABLE_STATISTICS_FILE_NAME = "${STATISTICS_FOLDER_NAME}/.disable" + + + /** + * Method for getting IStatisticsValuesConsumer for reporting some statistics + * Could be invoked during any build phase after applying first Kotlin plugin and + * until build is completed + */ + @JvmStatic + @Synchronized + fun getInstance(): IStatisticsValuesConsumer? { + if (statisticsIsEnabled != true) { + return null + } + return instance + } + + /** + * Method for creating new instance of IStatisticsValuesConsumer + * It could be invoked only when applying Kotlin gradle plugin. + * When executed, this method checks, whether it is already executed in current build. + * If it was not executed, the new instance of IStatisticsValuesConsumer is created + * + * If it was already executed in the same classpath (i.e. with the same version of Kotlin plugin), + * the previously returned instance is returned. + * + * If it was already executed in the other classpath, a JXM implementation is returned. + * + * All the created instances are registered as build listeners + */ + @JvmStatic + @Synchronized + internal fun getOrCreateInstance(gradle: Gradle): IStatisticsValuesConsumer? { + return runSafe("${KotlinBuildStatsService::class.java}.getOrCreateInstance") { + statisticsIsEnabled = statisticsIsEnabled ?: checkStatisticsEnabled(gradle) + if (statisticsIsEnabled != true) { + null + } else { + val log = getLogger() + + if (instance != null) { + log.debug("${KotlinBuildStatsService::class.java} is already instantiated. Current instance is $instance") + } else { + val beanName = ObjectName(JMX_BEAN_NAME) + val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() + if (mbs.isRegistered(beanName)) { + log.debug( + "${KotlinBuildStatsService::class.java} is already instantiated in another classpath. Creating JMX-wrapper" + ) + instance = JMXKotlinBuildStatsService(mbs, beanName) + } else { + val newInstance = DefaultKotlinBuildStatsService(gradle, beanName) + instance = newInstance + log.debug("Instantiated ${KotlinBuildStatsService::class.java}: new instance $instance") + mbs.registerMBean(StandardMBean(newInstance, KotlinBuildStatsMXBean::class.java), beanName) + } + } + gradle.addBuildListener(instance) + instance + } + } + } + + @JvmStatic + internal fun getLogger() = Logging.getLogger(KotlinBuildStatsService::class.java) + + internal var instance: KotlinBuildStatsService? = null + + private var statisticsIsEnabled: Boolean? = null + + private fun checkStatisticsEnabled(gradle: Gradle): Boolean { + return if (File(gradle.gradleUserHomeDir, DISABLE_STATISTICS_FILE_NAME).exists()) { + false + } else { + gradle.rootProject.properties[ENABLE_STATISTICS_PROPERTY_NAME]?.toString()?.toBoolean() ?: DEFAULT_STATISTICS_STATE + } + } + + private fun logException(description: String, e: Throwable) { + getLogger().info(description) + getLogger().debug(e.message, e) + } + + internal fun runSafe(methodName: String, action: () -> T?): T? { + return try { + getLogger().debug("Executing [$methodName]") + action.invoke() + } catch (e: Throwable) { + logException("Could not execute [$methodName]", e) + null + } + } + } + + +} + +internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) : + KotlinBuildStatsService() { + + override fun buildFinished(result: BuildResult) { + } + + private fun callJmx(method: String, type: String, metricName: String, value: Any, subprojectName: String?) { + mbs.invoke( + beanName, + method, + arrayOf(metricName, value, subprojectName), + arrayOf("java.lang.String", type, "java.lang.String") + ) + } + + override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) { + runSafe("report metric ${metric.name}") { + callJmx("reportBoolean", "boolean", metric.name, value, subprojectName) + } + } + + override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) { + runSafe("report metric ${metric.name}") { + callJmx("reportNumber", "long", metric.name, value, subprojectName) + } + } + + override fun report(metric: StringMetrics, value: String, subprojectName: String?) { + runSafe("report metric ${metric.name}") { + callJmx("reportString", "java.lang.String", metric.name, value, subprojectName) + } + } + + override fun completed() { + instance = null + } + +} + +internal class DefaultKotlinBuildStatsService internal constructor( + gradle: Gradle, + val beanName: ObjectName +) : KotlinBuildStatsService(), KotlinBuildStatsMXBean { + + private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir) + + private fun gradleBuildStartTime(gradle: Gradle): Long? { + return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime + } + + private fun reportGlobalMetrics(gradle: Gradle) { + System.getProperty("os.name")?.also { + sessionLogger.report(StringMetrics.OS_TYPE, gradle.gradleVersion) + } + sessionLogger.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong()) + sessionLogger.report(StringMetrics.GRADLE_VERSION, gradle.gradleVersion) + sessionLogger.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null) + + + gradle.allprojects { project -> + for (configuration in project.configurations) { + val configurationName = configuration.name + val dependencies = configuration.dependencies + + when (configurationName) { + "kapt" -> { + sessionLogger.report(BooleanMetrics.ENABLED_KAPT, true) + dependencies.forEach { dependency -> + when (dependency.group) { + "com.google.dagger" -> sessionLogger.report(BooleanMetrics.ENABLED_DAGGER, true) + "com.android.databinding" -> sessionLogger.report(BooleanMetrics.ENABLED_DATABINDING, true) + } + } + } + } + } + } + } + + override fun projectsEvaluated(gradle: Gradle) { + runSafe("${DefaultKotlinBuildStatsService::class.java}.projectEvaluated") { + if (!sessionLogger.isBuildSessionStarted()) { + sessionLogger.startBuildSession( + DaemonReuseCounter.incrementAndGetOrdinal(), + gradleBuildStartTime(gradle) + ) + reportGlobalMetrics(gradle) + } + } + } + + @Synchronized + override fun buildFinished(result: BuildResult) { + runSafe("${DefaultKotlinBuildStatsService::class.java}.buildFinished") { + sessionLogger.finishBuildSession(result.action, result.failure) + } + } + + + @Synchronized + override fun completed() { + runSafe("${DefaultKotlinBuildStatsService::class.java}.completed") { + try { + sessionLogger.unlockJournalFile() + } finally { + val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() + if (mbs.isRegistered(beanName)) { + mbs.unregisterMBean(beanName) + } + instance = null + } + } + } + + override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) { + runSafe("report metric ${metric.name}") { + sessionLogger.report(metric, value, subprojectName) + } + + } + + override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) { + runSafe("report metric ${metric.name}") { + sessionLogger.report(metric, value, subprojectName) + } + } + + override fun report(metric: StringMetrics, value: String, subprojectName: String?) { + runSafe("report metric ${metric.name}") { + sessionLogger.report(metric, value, subprojectName) + } + } + + override fun reportBoolean(name: String, value: Boolean, subprojectName: String?) { + report(BooleanMetrics.valueOf(name), value, subprojectName) + } + + override fun reportNumber(name: String, value: Long, subprojectName: String?) { + report(NumericalMetrics.valueOf(name), value, subprojectName) + } + + override fun reportString(name: String, value: String, subprojectName: String?) { + report(StringMetrics.valueOf(name), value, subprojectName) + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt index 061d6c5b4f9..8fcfa7eeea2 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt @@ -7,8 +7,10 @@ import org.gradle.api.tasks.Input import org.gradle.api.tasks.OutputDirectory import org.gradle.api.tasks.TaskAction import org.jetbrains.kotlin.gradle.logging.kotlinInfo +import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService import org.jetbrains.kotlin.gradle.utils.isGradleVersionAtLeast import org.jetbrains.kotlin.gradle.utils.patternLayoutCompatible +import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import java.io.File import java.net.URI @@ -52,7 +54,13 @@ open class NodeJsSetupTask : DefaultTask() { val dep = this.project.dependencies.create(ivyDependency) val conf = this.project.configurations.detachedConfiguration(dep) conf.isTransitive = false + + val startDownloadTime = System.currentTimeMillis() val result = conf.resolve().single() + + KotlinBuildStatsService.getInstance() + ?.report(NumericalMetrics.ARTIFACTS_DOWNLOAD_SPEED, result.length() * 1000 / (System.currentTimeMillis() - startDownloadTime)) + project.repositories.remove(repo) project.logger.kotlinInfo("Using node distribution from '$result'") diff --git a/libraries/tools/kotlin-gradle-statistics/build.gradle.kts b/libraries/tools/kotlin-gradle-statistics/build.gradle.kts new file mode 100644 index 00000000000..3f9df54773e --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/build.gradle.kts @@ -0,0 +1,20 @@ +description = "kotlin-gradle-statistics" + +plugins { + kotlin("jvm") +} + +dependencies { + compileOnly(kotlinStdlib()) +} + +sourceSets { + "main" { projectDefault() } + "test" { projectDefault() } +} + +projectTest { + workingDir = rootDir +} + +runtimeJar(rewriteDefaultJarDepsToShadedCompiler()) diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSession.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSession.kt new file mode 100644 index 00000000000..3a147ea1de3 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSession.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics + +class BuildSession(val buildStartedTime: Long?) { + + val projectEvaluatedTime = System.currentTimeMillis() + +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt new file mode 100644 index 00000000000..52086357baa --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt @@ -0,0 +1,130 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics + +import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger +import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger +import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer +import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger +import org.jetbrains.kotlin.statistics.metrics.* +import java.io.File +import java.io.IOException +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +class BuildSessionLogger( + private val rootPath: File, + private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES, + private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE +) : IStatisticsValuesConsumer { + + companion object { + const val STATISTICS_FOLDER_NAME = "kotlin-profile" + const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}.profile" + + private const val DEFAULT_MAX_PROFILE_FILES = 1_000 + private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L + } + + private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS'.profile'") + private val statisticsFolder: File = File( + rootPath, + STATISTICS_FOLDER_NAME + ).also { it.mkdirs() } + + private var buildSession: BuildSession? = null + private var trackingFile: IRecordLogger? = null + + private val metricsContainer = MetricsContainer() + + @Synchronized + fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) { + report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart) + + buildSession = BuildSession(buildStartedTime) + initTrackingFile() + } + + @Synchronized + fun isBuildSessionStarted() = buildSession != null + + @Synchronized + private fun closeTrackingFile() { + metricsContainer.flush(trackingFile) + trackingFile?.close() + trackingFile = null + } + + @Synchronized + private fun initTrackingFile() { + closeTrackingFile() + + // Get list of existing files. Try to create folder if possible, return from function if failed to create folder + val fileCandidates = + statisticsFolder.listFiles()?.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.toMutableList() + ?: if (statisticsFolder.mkdirs()) emptyList() else return + + for (i in 0 until fileCandidates.size - maxProfileFiles) { + val file2delete = fileCandidates[i] + if (file2delete.isFile) { + file2delete.delete() + } + } + + // emergency check. What if a lot of files are locked due to some reason + if (statisticsFolder.listFiles()?.size ?: 0 > maxProfileFiles * 2) { + return + } + + fun newFile(): File = File(statisticsFolder, profileFileNameFormatter.format(LocalDateTime.now())) + val lastFile = fileCandidates.lastOrNull() ?: newFile() + + trackingFile = try { + if (lastFile.length() < maxFileSize) { + FileRecordLogger(lastFile) + } else { + FileRecordLogger(newFile()) + } + } catch (e: IOException) { + try { + FileRecordLogger(newFile()) + } catch (e: IOException) { + NullRecordLogger() + } + } + } + + @Synchronized + fun finishBuildSession(action: String?, failure: Throwable?) { + // nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration + // could be affected by system clock correction + val finishTime = System.currentTimeMillis() + buildSession?.also { + if (it.buildStartedTime != null) { + report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime) + } + report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime) + } + buildSession = null + } + + @Synchronized + fun unlockJournalFile() { + closeTrackingFile() + } + + override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) { + metricsContainer.report(metric, value, subprojectName) + } + + override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) { + metricsContainer.report(metric, value, subprojectName) + } + + override fun report(metric: StringMetrics, value: String, subprojectName: String?) { + metricsContainer.report(metric, value, subprojectName) + } +} diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/FileRecordLogger.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/FileRecordLogger.kt new file mode 100644 index 00000000000..42099a3e7bd --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/FileRecordLogger.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.fileloggers + +import java.io.File +import java.io.IOException +import java.io.OutputStream +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.channels.FileLock +import java.nio.file.Paths +import java.nio.file.StandardOpenOption + +class FileRecordLogger(file: File) : IRecordLogger { + + private val channel: FileChannel = + FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND) + ?: throw IOException("Could not open file $file") + + private val outputStream: OutputStream + private val lock: FileLock + + init { + lock = try { + channel.lock() + } catch (e: Exception) { + channel.close() + // wrap in order to unify with FileOverlappingException + throw IOException(e.message, e) + } + outputStream = Channels.newOutputStream(channel) + } + + override fun append(s: String) { + outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING)) + } + + override fun close() { + channel.use { + outputStream.use { + lock.release() + } + } + } +} diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/IRecordLogger.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/IRecordLogger.kt new file mode 100644 index 00000000000..64820392e69 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/IRecordLogger.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.fileloggers + +import java.io.Closeable + +interface IRecordLogger : Closeable { + + fun append(s: String) + +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt new file mode 100644 index 00000000000..32e46283fe0 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.fileloggers + +import org.jetbrains.kotlin.statistics.metrics.* +import org.jetbrains.kotlin.statistics.sha256 +import java.io.* +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.file.Paths +import java.nio.file.StandardOpenOption +import java.util.* + +class MetricsContainer : IStatisticsValuesConsumer { + + data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable { + override fun compareTo(other: MetricDescriptor): Int { + return when { + name.compareTo(other.name) != 0 -> name.compareTo(other.name) + projectHash == other.projectHash -> 0 + else -> (projectHash ?: "").compareTo(other.projectHash ?: "") + } + } + } + + private val numericalMetrics = TreeMap>() + + private val booleanMetrics = TreeMap>() + + private val stringMetrics = TreeMap>() + + companion object { + + private const val BUILD_SESSION_SEPARATOR = "BUILD FINISHED" + + val ENCODING = Charsets.UTF_8 + + private val stringMetricsMap = StringMetrics.values().map { m -> m.name to m }.toMap() + + private val booleanMetricsMap = BooleanMetrics.values().map { m -> m.name to m }.toMap() + + private val numericalMetricsMap = NumericalMetrics.values().map { m -> m.name to m }.toMap() + + fun readFromFile(file: File, consumer: (MetricsContainer) -> Unit) { + val channel = FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.READ) + channel.lock() + + val inputStream = Channels.newInputStream(channel) + try { + var container = MetricsContainer() + // Note: close is called at forEachLine + BufferedReader(InputStreamReader(inputStream, ENCODING)).forEachLine { line -> + if (BUILD_SESSION_SEPARATOR == line) { + consumer.invoke(container) + container = MetricsContainer() + } else { + // format: metricName.hash=string representation + val lineParts = line.split('=') + if (lineParts.size == 2) { + val name = lineParts[0].split('.')[0] + val subProjectHash = lineParts[0].split('.').getOrNull(1) + val representation = lineParts[1] + + stringMetricsMap[name]?.also { metricType -> + metricType.type.fromStringRepresentation(representation)?.also { + container.stringMetrics[MetricDescriptor(name, subProjectHash)] = it + } + } + + booleanMetricsMap[name]?.also { metricType -> + metricType.type.fromStringRepresentation(representation)?.also { + container.booleanMetrics[MetricDescriptor(name, subProjectHash)] = it + } + } + + numericalMetricsMap[name]?.also { metricType -> + metricType.type.fromStringRepresentation(representation)?.also { + container.numericalMetrics[MetricDescriptor(name, subProjectHash)] = it + } + } + } + } + } + } finally { + channel.close() + } + } + } + + private fun processProjectName(subprojectName: String?, perProject: Boolean) = + if (perProject && subprojectName != null) sha256(subprojectName) else null + + override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) { + val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject) + val metricContainer = booleanMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer() + .also { booleanMetrics[MetricDescriptor(metric.name, projectHash)] = it } + metricContainer.addValue(metric.anonymization.anonymize(value)) + } + + override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) { + val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject) + val metricContainer = numericalMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer() + .also { numericalMetrics[MetricDescriptor(metric.name, projectHash)] = it } + metricContainer.addValue(metric.anonymization.anonymize(value)) + } + + override fun report(metric: StringMetrics, value: String, subprojectName: String?) { + val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject) + val metricContainer = stringMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer() + .also { stringMetrics[MetricDescriptor(metric.name, projectHash)] = it } + metricContainer.addValue(metric.anonymization.anonymize(value)) + } + + fun flush(trackingFile: IRecordLogger?) { + if (trackingFile == null) return + for (entry in numericalMetrics.entries.union(booleanMetrics.entries).union(stringMetrics.entries)) { + val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}" + trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}") + } + + trackingFile.append(BUILD_SESSION_SEPARATOR) + + stringMetrics.clear() + booleanMetrics.clear() + numericalMetrics.clear() + } + + fun getMetric(metric: NumericalMetrics): IMetricContainer? = numericalMetrics[MetricDescriptor(metric.name, null)] + + fun getMetric(metric: StringMetrics): IMetricContainer? = stringMetrics[MetricDescriptor(metric.name, null)] + + fun getMetric(metric: BooleanMetrics): IMetricContainer? = booleanMetrics[MetricDescriptor(metric.name, null)] +} diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/NullRecordLogger.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/NullRecordLogger.kt new file mode 100644 index 00000000000..d3529c15012 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/NullRecordLogger.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.fileloggers + +class NullRecordLogger : IRecordLogger { + override fun append(s: String) {} + + override fun close() { + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt new file mode 100644 index 00000000000..7f7a19eda6a --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.metrics + +interface IMetricContainer { + fun addValue(t: T) + + fun toStringRepresentation(): String + + fun getValue(): T? + +} + +interface IMetricContainerFactory { + fun newMetricContainer(): IMetricContainer + + //null if could not parse + fun fromStringRepresentation(state: String): IMetricContainer? +} + +class OverrideMetricContainer() : IMetricContainer { + + private var myValue: T? = null + + override fun addValue(t: T) { + myValue = t + } + + internal constructor(v: T?) : this() { + myValue = v + } + + override fun toStringRepresentation(): String { + return myValue?.toString() ?: "null" + } + + override fun getValue() = myValue +} + +class ConcatMetricContainer() : IMetricContainer { + private val myValues = HashSet() + + override fun addValue(t: String) { + myValues.add(t) + } + + override fun toStringRepresentation(): String { + return myValues.joinToString(";") + } + + override fun getValue() = toStringRepresentation() +} + diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StatisticsValues.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StatisticsValues.kt new file mode 100644 index 00000000000..62f666a2423 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StatisticsValues.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.metrics + +interface ReportStatisticsValue { + val name: String + val value: T +} + +class ReportOnceStatisticsValue(override val name: String, override val value: T) : + ReportStatisticsValue + +interface AdditiveStatisticsValue : ReportStatisticsValue { + fun addValue(t: T) +} + +interface IStatisticsValuesConsumer { + + fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null) + + fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null) + + fun report(metric: StringMetrics, value: String, subprojectName: String? = null) + +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 7a6af76056e..b9d138accb0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -219,6 +219,7 @@ include ":kotlin-build-common", ":tools:kotlinp", ":kotlin-gradle-plugin-api", ":kotlin-gradle-plugin-dsl-codegen", + ":kotlin-gradle-statistics", ":kotlin-gradle-plugin", ":kotlin-gradle-plugin-model", ":kotlin-gradle-plugin-test-utils-embeddable", @@ -421,6 +422,7 @@ project(':kotlin-source-sections-compiler-plugin').projectDir = "$rootDir/plugin project(':tools:kotlinp').projectDir = "$rootDir/libraries/tools/kotlinp" as File project(':kotlin-gradle-plugin-api').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-api" as File project(':kotlin-gradle-plugin-dsl-codegen').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-dsl-codegen" as File +project(':kotlin-gradle-statistics').projectDir = "$rootDir/libraries/tools/kotlin-gradle-statistics" as File project(':kotlin-gradle-plugin').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin" as File project(':kotlin-gradle-plugin-model').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-model" as File project(':kotlin-gradle-plugin-test-utils-embeddable').projectDir = "$rootDir/libraries/tools/kotlin-gradle-plugin-test-utils-embeddable" as File