Collecting performance information in Gradle plugin supported
The build performance information is collected during the build when the Kotlin Gradle plugin is applied. Only hashed values are saved if they may contain sensitive information. Numeric metrics obtained on the basis of user's project are saved with random seed. Persisted information is saved in gradleUserHomeDir. #KT-33404 Fixed
This commit is contained in:
Andrey Uskov
@@ -0,0 +1,20 @@
|
||||
description = "kotlin-gradle-statistics"
|
||||
|
||||
plugins {
|
||||
kotlin("jvm")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compileOnly(kotlinStdlib())
|
||||
}
|
||||
|
||||
sourceSets {
|
||||
"main" { projectDefault() }
|
||||
"test" { projectDefault() }
|
||||
}
|
||||
|
||||
projectTest {
|
||||
workingDir = rootDir
|
||||
}
|
||||
|
||||
runtimeJar(rewriteDefaultJarDepsToShadedCompiler())
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics
|
||||
|
||||
class BuildSession(val buildStartedTime: Long?) {
|
||||
|
||||
val projectEvaluatedTime = System.currentTimeMillis()
|
||||
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics
|
||||
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger
|
||||
import org.jetbrains.kotlin.statistics.metrics.*
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.time.LocalDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
|
||||
class BuildSessionLogger(
|
||||
private val rootPath: File,
|
||||
private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
|
||||
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE
|
||||
) : IStatisticsValuesConsumer {
|
||||
|
||||
companion object {
|
||||
const val STATISTICS_FOLDER_NAME = "kotlin-profile"
|
||||
const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}.profile"
|
||||
|
||||
private const val DEFAULT_MAX_PROFILE_FILES = 1_000
|
||||
private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
|
||||
}
|
||||
|
||||
private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS'.profile'")
|
||||
private val statisticsFolder: File = File(
|
||||
rootPath,
|
||||
STATISTICS_FOLDER_NAME
|
||||
).also { it.mkdirs() }
|
||||
|
||||
private var buildSession: BuildSession? = null
|
||||
private var trackingFile: IRecordLogger? = null
|
||||
|
||||
private val metricsContainer = MetricsContainer()
|
||||
|
||||
@Synchronized
|
||||
fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
|
||||
report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart)
|
||||
|
||||
buildSession = BuildSession(buildStartedTime)
|
||||
initTrackingFile()
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun isBuildSessionStarted() = buildSession != null
|
||||
|
||||
@Synchronized
|
||||
private fun closeTrackingFile() {
|
||||
metricsContainer.flush(trackingFile)
|
||||
trackingFile?.close()
|
||||
trackingFile = null
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
private fun initTrackingFile() {
|
||||
closeTrackingFile()
|
||||
|
||||
// Get list of existing files. Try to create folder if possible, return from function if failed to create folder
|
||||
val fileCandidates =
|
||||
statisticsFolder.listFiles()?.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.toMutableList()
|
||||
?: if (statisticsFolder.mkdirs()) emptyList<File>() else return
|
||||
|
||||
for (i in 0 until fileCandidates.size - maxProfileFiles) {
|
||||
val file2delete = fileCandidates[i]
|
||||
if (file2delete.isFile) {
|
||||
file2delete.delete()
|
||||
}
|
||||
}
|
||||
|
||||
// emergency check. What if a lot of files are locked due to some reason
|
||||
if (statisticsFolder.listFiles()?.size ?: 0 > maxProfileFiles * 2) {
|
||||
return
|
||||
}
|
||||
|
||||
fun newFile(): File = File(statisticsFolder, profileFileNameFormatter.format(LocalDateTime.now()))
|
||||
val lastFile = fileCandidates.lastOrNull() ?: newFile()
|
||||
|
||||
trackingFile = try {
|
||||
if (lastFile.length() < maxFileSize) {
|
||||
FileRecordLogger(lastFile)
|
||||
} else {
|
||||
FileRecordLogger(newFile())
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
try {
|
||||
FileRecordLogger(newFile())
|
||||
} catch (e: IOException) {
|
||||
NullRecordLogger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun finishBuildSession(action: String?, failure: Throwable?) {
|
||||
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
|
||||
// could be affected by system clock correction
|
||||
val finishTime = System.currentTimeMillis()
|
||||
buildSession?.also {
|
||||
if (it.buildStartedTime != null) {
|
||||
report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
|
||||
}
|
||||
report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
|
||||
}
|
||||
buildSession = null
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun unlockJournalFile() {
|
||||
closeTrackingFile()
|
||||
}
|
||||
|
||||
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) {
|
||||
metricsContainer.report(metric, value, subprojectName)
|
||||
}
|
||||
|
||||
override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) {
|
||||
metricsContainer.report(metric, value, subprojectName)
|
||||
}
|
||||
|
||||
override fun report(metric: StringMetrics, value: String, subprojectName: String?) {
|
||||
metricsContainer.report(metric, value, subprojectName)
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.fileloggers
|
||||
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.io.OutputStream
|
||||
import java.nio.channels.Channels
|
||||
import java.nio.channels.FileChannel
|
||||
import java.nio.channels.FileLock
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.StandardOpenOption
|
||||
|
||||
class FileRecordLogger(file: File) : IRecordLogger {
|
||||
|
||||
private val channel: FileChannel =
|
||||
FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
|
||||
?: throw IOException("Could not open file $file")
|
||||
|
||||
private val outputStream: OutputStream
|
||||
private val lock: FileLock
|
||||
|
||||
init {
|
||||
lock = try {
|
||||
channel.lock()
|
||||
} catch (e: Exception) {
|
||||
channel.close()
|
||||
// wrap in order to unify with FileOverlappingException
|
||||
throw IOException(e.message, e)
|
||||
}
|
||||
outputStream = Channels.newOutputStream(channel)
|
||||
}
|
||||
|
||||
override fun append(s: String) {
|
||||
outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING))
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
channel.use {
|
||||
outputStream.use {
|
||||
lock.release()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.fileloggers
|
||||
|
||||
import java.io.Closeable
|
||||
|
||||
interface IRecordLogger : Closeable {
|
||||
|
||||
fun append(s: String)
|
||||
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.fileloggers
|
||||
|
||||
import org.jetbrains.kotlin.statistics.metrics.*
|
||||
import org.jetbrains.kotlin.statistics.sha256
|
||||
import java.io.*
|
||||
import java.nio.channels.Channels
|
||||
import java.nio.channels.FileChannel
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.util.*
|
||||
|
||||
class MetricsContainer : IStatisticsValuesConsumer {
|
||||
|
||||
data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
|
||||
override fun compareTo(other: MetricDescriptor): Int {
|
||||
return when {
|
||||
name.compareTo(other.name) != 0 -> name.compareTo(other.name)
|
||||
projectHash == other.projectHash -> 0
|
||||
else -> (projectHash ?: "").compareTo(other.projectHash ?: "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val numericalMetrics = TreeMap<MetricDescriptor, IMetricContainer<Long>>()
|
||||
|
||||
private val booleanMetrics = TreeMap<MetricDescriptor, IMetricContainer<Boolean>>()
|
||||
|
||||
private val stringMetrics = TreeMap<MetricDescriptor, IMetricContainer<String>>()
|
||||
|
||||
companion object {
|
||||
|
||||
private const val BUILD_SESSION_SEPARATOR = "BUILD FINISHED"
|
||||
|
||||
val ENCODING = Charsets.UTF_8
|
||||
|
||||
private val stringMetricsMap = StringMetrics.values().map { m -> m.name to m }.toMap()
|
||||
|
||||
private val booleanMetricsMap = BooleanMetrics.values().map { m -> m.name to m }.toMap()
|
||||
|
||||
private val numericalMetricsMap = NumericalMetrics.values().map { m -> m.name to m }.toMap()
|
||||
|
||||
fun readFromFile(file: File, consumer: (MetricsContainer) -> Unit) {
|
||||
val channel = FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.READ)
|
||||
channel.lock()
|
||||
|
||||
val inputStream = Channels.newInputStream(channel)
|
||||
try {
|
||||
var container = MetricsContainer()
|
||||
// Note: close is called at forEachLine
|
||||
BufferedReader(InputStreamReader(inputStream, ENCODING)).forEachLine { line ->
|
||||
if (BUILD_SESSION_SEPARATOR == line) {
|
||||
consumer.invoke(container)
|
||||
container = MetricsContainer()
|
||||
} else {
|
||||
// format: metricName.hash=string representation
|
||||
val lineParts = line.split('=')
|
||||
if (lineParts.size == 2) {
|
||||
val name = lineParts[0].split('.')[0]
|
||||
val subProjectHash = lineParts[0].split('.').getOrNull(1)
|
||||
val representation = lineParts[1]
|
||||
|
||||
stringMetricsMap[name]?.also { metricType ->
|
||||
metricType.type.fromStringRepresentation(representation)?.also {
|
||||
container.stringMetrics[MetricDescriptor(name, subProjectHash)] = it
|
||||
}
|
||||
}
|
||||
|
||||
booleanMetricsMap[name]?.also { metricType ->
|
||||
metricType.type.fromStringRepresentation(representation)?.also {
|
||||
container.booleanMetrics[MetricDescriptor(name, subProjectHash)] = it
|
||||
}
|
||||
}
|
||||
|
||||
numericalMetricsMap[name]?.also { metricType ->
|
||||
metricType.type.fromStringRepresentation(representation)?.also {
|
||||
container.numericalMetrics[MetricDescriptor(name, subProjectHash)] = it
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
channel.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun processProjectName(subprojectName: String?, perProject: Boolean) =
|
||||
if (perProject && subprojectName != null) sha256(subprojectName) else null
|
||||
|
||||
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?) {
|
||||
val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject)
|
||||
val metricContainer = booleanMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
|
||||
.also { booleanMetrics[MetricDescriptor(metric.name, projectHash)] = it }
|
||||
metricContainer.addValue(metric.anonymization.anonymize(value))
|
||||
}
|
||||
|
||||
override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?) {
|
||||
val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject)
|
||||
val metricContainer = numericalMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
|
||||
.also { numericalMetrics[MetricDescriptor(metric.name, projectHash)] = it }
|
||||
metricContainer.addValue(metric.anonymization.anonymize(value))
|
||||
}
|
||||
|
||||
override fun report(metric: StringMetrics, value: String, subprojectName: String?) {
|
||||
val projectHash = if (subprojectName == null) null else processProjectName(subprojectName, metric.perProject)
|
||||
val metricContainer = stringMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
|
||||
.also { stringMetrics[MetricDescriptor(metric.name, projectHash)] = it }
|
||||
metricContainer.addValue(metric.anonymization.anonymize(value))
|
||||
}
|
||||
|
||||
fun flush(trackingFile: IRecordLogger?) {
|
||||
if (trackingFile == null) return
|
||||
for (entry in numericalMetrics.entries.union(booleanMetrics.entries).union(stringMetrics.entries)) {
|
||||
val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}"
|
||||
trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
|
||||
}
|
||||
|
||||
trackingFile.append(BUILD_SESSION_SEPARATOR)
|
||||
|
||||
stringMetrics.clear()
|
||||
booleanMetrics.clear()
|
||||
numericalMetrics.clear()
|
||||
}
|
||||
|
||||
fun getMetric(metric: NumericalMetrics): IMetricContainer<Long>? = numericalMetrics[MetricDescriptor(metric.name, null)]
|
||||
|
||||
fun getMetric(metric: StringMetrics): IMetricContainer<String>? = stringMetrics[MetricDescriptor(metric.name, null)]
|
||||
|
||||
fun getMetric(metric: BooleanMetrics): IMetricContainer<Boolean>? = booleanMetrics[MetricDescriptor(metric.name, null)]
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.fileloggers
|
||||
|
||||
class NullRecordLogger : IRecordLogger {
|
||||
override fun append(s: String) {}
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.metrics
|
||||
|
||||
interface IMetricContainer<T> {
|
||||
fun addValue(t: T)
|
||||
|
||||
fun toStringRepresentation(): String
|
||||
|
||||
fun getValue(): T?
|
||||
|
||||
}
|
||||
|
||||
interface IMetricContainerFactory<T> {
|
||||
fun newMetricContainer(): IMetricContainer<T>
|
||||
|
||||
//null if could not parse
|
||||
fun fromStringRepresentation(state: String): IMetricContainer<T>?
|
||||
}
|
||||
|
||||
class OverrideMetricContainer<T>() : IMetricContainer<T> {
|
||||
|
||||
private var myValue: T? = null
|
||||
|
||||
override fun addValue(t: T) {
|
||||
myValue = t
|
||||
}
|
||||
|
||||
internal constructor(v: T?) : this() {
|
||||
myValue = v
|
||||
}
|
||||
|
||||
override fun toStringRepresentation(): String {
|
||||
return myValue?.toString() ?: "null"
|
||||
}
|
||||
|
||||
override fun getValue() = myValue
|
||||
}
|
||||
|
||||
class ConcatMetricContainer() : IMetricContainer<String> {
|
||||
private val myValues = HashSet<String>()
|
||||
|
||||
override fun addValue(t: String) {
|
||||
myValues.add(t)
|
||||
}
|
||||
|
||||
override fun toStringRepresentation(): String {
|
||||
return myValues.joinToString(";")
|
||||
}
|
||||
|
||||
override fun getValue() = toStringRepresentation()
|
||||
}
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.metrics
|
||||
|
||||
interface ReportStatisticsValue<T> {
|
||||
val name: String
|
||||
val value: T
|
||||
}
|
||||
|
||||
class ReportOnceStatisticsValue<T>(override val name: String, override val value: T) :
|
||||
ReportStatisticsValue<T>
|
||||
|
||||
interface AdditiveStatisticsValue<T> : ReportStatisticsValue<T> {
|
||||
fun addValue(t: T)
|
||||
}
|
||||
|
||||
interface IStatisticsValuesConsumer {
|
||||
|
||||
fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null)
|
||||
|
||||
fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null)
|
||||
|
||||
fun report(metric: StringMetrics, value: String, subprojectName: String? = null)
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user