Remove Gradle plugin variants from ':kotlin-gradle-statistics'

Code does not use any Gradle API imports and does not use any KGP API
classes.
This commit is contained in:
Yahor Berdnikau
2023-01-17 12:00:42 +01:00
committed by Space Team
parent 92509ad400
commit 5c8b8bf009
14 changed files with 20 additions and 1 deletions
@@ -0,0 +1,44 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics
import java.security.MessageDigest
internal interface ValueAnonymizer<T> {
fun anonymize(t: T): T
fun anonymizeOnIdeSize(): Boolean = false
}
internal val salt: String by lazy {
val env = System.getenv()
"${env["HOSTNAME"]}${env["COMPUTERNAME"]}"
}
fun anonymizeComponentVersion(version: String): String {
val parts = version.toLowerCase().replace('-', '.')
.split(".")
.plus(listOf("0", "0", "0")) // pad with zeros
.take(4)
val mainVersion = parts.take(3).map { s -> s.toIntOrNull()?.toString() ?: "0" }
val suffix = when {
parts[3].matches("(rc|m)\\d{0,1}".toRegex()) -> "-${parts[3]}"
parts[3].matches("(snapshot|dev|beta)".toRegex()) -> "-${parts[3]}"
else -> ""
}
return mainVersion.joinToString(".") + suffix
}
internal fun sha256(s: String): String {
val md = MessageDigest.getInstance("SHA-256")
val digest = md.digest(s.toByteArray())
return digest.fold("", { str, it -> str + "%02x".format(it) })
}
class MetricValueValidationFailed(message: String) : RuntimeException(message)
@@ -0,0 +1,10 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics
class BuildSession(val buildStartedTime: Long?) {
val projectEvaluatedTime = System.currentTimeMillis()
}
@@ -0,0 +1,166 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics
import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger
import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger
import org.jetbrains.kotlin.statistics.metrics.*
import java.io.File
import java.io.IOException
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
class BuildSessionLogger(
rootPath: File,
private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
private val forceValuesValidation: Boolean = false
) : IStatisticsValuesConsumer {
companion object {
const val STATISTICS_FOLDER_NAME = "kotlin-profile"
const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}(.\\d+)?.profile"
private const val DEFAULT_MAX_PROFILE_FILES = 1_000
private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
private const val DEFAULT_MAX_FILE_AGE = 30 * 24 * 3600 * 1000L //30 days
fun listProfileFiles(statisticsFolder: File): List<File>? {
return statisticsFolder.listFiles()?.filterTo(ArrayList()) { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted()
}
}
private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS")
private val profileFileNameSuffix = ".profile"
private val statisticsFolder: File = File(
rootPath,
STATISTICS_FOLDER_NAME
).also { it.mkdirs() }
private var buildSession: BuildSession? = null
private var trackingFile: IRecordLogger? = null
private val metricsContainer = MetricsContainer(forceValuesValidation)
@Synchronized
fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart)
buildSession = BuildSession(buildStartedTime)
initTrackingFile()
}
@Synchronized
fun isBuildSessionStarted() = buildSession != null
@Synchronized
private fun closeTrackingFile() {
trackingFile?.let {
metricsContainer.flush(it)
it.close()
trackingFile = null
}
}
/**
* Initializes a new tracking file
* The following contracts are implemented:
* - number of tracking files should not be more than maxProfileFiles (the earlier file created the earlier deleted)
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
* - files are ordered on the basis of name (creation timestamp)
* - if the last file has size less then maxFileSize, the next record will be append to it (new file created otherwise)
* -
*/
@Synchronized
private fun initTrackingFile() {
closeTrackingFile()
// Get list of existing files. Try to create folder if possible, return from function if failed to create folder
val fileCandidates = listProfileFiles(statisticsFolder) ?: if (statisticsFolder.mkdirs()) emptyList() else return
for ((index, file) in fileCandidates.withIndex()) {
val toDelete = if (index < fileCandidates.size - maxProfileFiles)
true
else {
val lastModified = file.lastModified()
(lastModified > 0) && (System.currentTimeMillis() - maxFileAge > lastModified)
}
if (toDelete) {
file.delete()
}
}
// emergency check. What if a lot of files are locked due to some reason
if ((listProfileFiles(statisticsFolder)?.size ?: 0) > maxProfileFiles * 2) {
trackingFile = NullRecordLogger()
return
}
fun newFile(): File {
val timestamp = profileFileNameFormatter.format(LocalDateTime.now())
var result = File(statisticsFolder, timestamp + profileFileNameSuffix)
var suffixIndex = 0
while (result.exists()) {
result = File(statisticsFolder, "${timestamp}.${suffixIndex++}$profileFileNameSuffix")
}
return result
}
val lastFile = fileCandidates.lastOrNull() ?: newFile()
trackingFile = try {
if (lastFile.length() < maxFileSize) {
FileRecordLogger(lastFile)
} else {
FileRecordLogger(newFile())
}
} catch (e: IOException) {
try {
FileRecordLogger(newFile())
} catch (e: IOException) {
NullRecordLogger()
}
}
}
@Synchronized
fun finishBuildSession(@Suppress("UNUSED_PARAMETER") action: String?, failure: Throwable?) {
try {
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
// could be affected by system clock correction
val finishTime = System.currentTimeMillis()
buildSession?.also {
if (it.buildStartedTime != null) {
report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
}
report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
report(NumericalMetrics.BUILD_FINISH_TIME, finishTime)
report(BooleanMetrics.BUILD_FAILED, failure != null)
}
buildSession = null
} finally {
unlockJournalFile()
}
}
@Synchronized
private fun unlockJournalFile() {
closeTrackingFile()
}
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) =
metricsContainer.report(metric, value, subprojectName, weight)
override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?) =
metricsContainer.report(metric, value, subprojectName, weight)
override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?) =
metricsContainer.report(metric, value, subprojectName, weight)
}
@@ -0,0 +1,47 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
import java.io.File
import java.io.IOException
import java.io.OutputStream
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.channels.FileLock
import java.nio.file.Paths
import java.nio.file.StandardOpenOption
class FileRecordLogger(file: File) : IRecordLogger {
private val channel: FileChannel =
FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
?: throw IOException("Could not open file $file")
private val outputStream: OutputStream
private val lock: FileLock
init {
lock = try {
channel.tryLock() ?: throw IOException("Could not acquire an exclusive lock of file ${file.name}")
} catch (e: Exception) {
channel.close()
// wrap in order to unify with FileOverlappingException
throw IOException(e.message, e)
}
outputStream = Channels.newOutputStream(channel)
}
override fun append(s: String) {
outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING))
}
override fun close() {
channel.use {
outputStream.use {
lock.release()
}
}
}
}
@@ -0,0 +1,12 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
import java.io.Closeable
interface IRecordLogger : Closeable {
fun append(s: String)
}
@@ -0,0 +1,179 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
import org.jetbrains.kotlin.statistics.MetricValueValidationFailed
import org.jetbrains.kotlin.statistics.metrics.*
import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.AllowedListAnonymizer.Companion.UNEXPECTED_VALUE
import org.jetbrains.kotlin.statistics.sha256
import java.io.*
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.file.Paths
import java.nio.file.StandardOpenOption
import java.util.*
class MetricsContainer(private val forceValuesValidation: Boolean = false) : IStatisticsValuesConsumer {
data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
override fun compareTo(other: MetricDescriptor): Int {
val compareNames = name.compareTo(other.name)
return when {
compareNames != 0 -> compareNames
projectHash == other.projectHash -> 0
else -> (projectHash ?: "").compareTo(other.projectHash ?: "")
}
}
}
private val metricsLock = Object()
private val numericalMetrics = TreeMap<MetricDescriptor, IMetricContainer<Long>>()
private val booleanMetrics = TreeMap<MetricDescriptor, IMetricContainer<Boolean>>()
private val stringMetrics = TreeMap<MetricDescriptor, IMetricContainer<String>>()
companion object {
private const val BUILD_SESSION_SEPARATOR = "BUILD FINISHED"
val ENCODING = Charsets.UTF_8
private val stringMetricsMap = StringMetrics.values().associateBy(StringMetrics::name)
private val booleanMetricsMap = BooleanMetrics.values().associateBy(BooleanMetrics::name)
private val numericalMetricsMap = NumericalMetrics.values().associateBy(NumericalMetrics::name)
fun readFromFile(file: File, consumer: (MetricsContainer) -> Unit): Boolean {
val channel = FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.READ)
channel.tryLock() ?: return false
val inputStream = Channels.newInputStream(channel)
try {
var container = MetricsContainer()
// Note: close is called at forEachLine
BufferedReader(InputStreamReader(inputStream, ENCODING)).forEachLine { line ->
if (BUILD_SESSION_SEPARATOR == line) {
consumer.invoke(container)
container = MetricsContainer()
} else {
// format: metricName.hash=string representation
val lineParts = line.split('=')
if (lineParts.size == 2) {
val name = lineParts[0].split('.')[0]
val subProjectHash = lineParts[0].split('.').getOrNull(1)
val representation = lineParts[1]
stringMetricsMap[name]?.also { metricType ->
metricType.type.fromStringRepresentation(representation)?.also {
synchronized(container.metricsLock) {
container.stringMetrics[MetricDescriptor(name, subProjectHash)] = it
}
}
}
booleanMetricsMap[name]?.also { metricType ->
metricType.type.fromStringRepresentation(representation)?.also {
synchronized(container.metricsLock) {
container.booleanMetrics[MetricDescriptor(name, subProjectHash)] = it
}
}
}
numericalMetricsMap[name]?.also { metricType ->
metricType.type.fromStringRepresentation(representation)?.also {
synchronized(container.metricsLock) {
container.numericalMetrics[MetricDescriptor(name, subProjectHash)] = it
}
}
}
}
}
}
} finally {
channel.close()
}
return true
}
}
private fun processProjectName(subprojectName: String?, perProject: Boolean) =
if (perProject && subprojectName != null) sha256(subprojectName) else null
private fun getProjectHash(perProject: Boolean, subprojectName: String?) =
if (subprojectName == null) null else processProjectName(subprojectName, perProject)
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?): Boolean {
val projectHash = getProjectHash(metric.perProject, subprojectName)
synchronized(metricsLock) {
val metricContainer = booleanMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
.also { booleanMetrics[MetricDescriptor(metric.name, projectHash)] = it }
metricContainer.addValue(metric.anonymization.anonymize(value), weight)
}
return true
}
override fun report(metric: NumericalMetrics, value: Long, subprojectName: String?, weight: Long?): Boolean {
val projectHash = getProjectHash(metric.perProject, subprojectName)
synchronized(metricsLock) {
val metricContainer = numericalMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
.also { numericalMetrics[MetricDescriptor(metric.name, projectHash)] = it }
metricContainer.addValue(metric.anonymization.anonymize(value), weight)
}
return true
}
override fun report(metric: StringMetrics, value: String, subprojectName: String?, weight: Long?): Boolean {
val projectHash = getProjectHash(metric.perProject, subprojectName)
synchronized(metricsLock) {
val metricContainer = stringMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
.also { stringMetrics[MetricDescriptor(metric.name, projectHash)] = it }
val anonymizedValue = metric.anonymization.anonymize(value)
if (forceValuesValidation && !metric.anonymization.anonymizeOnIdeSize()) {
if (anonymizedValue.contains(UNEXPECTED_VALUE) || !anonymizedValue.matches(Regex(metric.anonymization.validationRegexp()))) {
throw MetricValueValidationFailed("Metric ${metric.name} has value [${value}], after anonymization [${anonymizedValue}]. Validation regex: ${metric.anonymization.validationRegexp()}.")
}
}
metricContainer.addValue(anonymizedValue, weight)
}
return true
}
fun flush(trackingFile: IRecordLogger?) {
if (trackingFile == null) return
val allMetrics = TreeMap<MetricDescriptor, IMetricContainer<out Any>>()
synchronized(metricsLock) {
allMetrics.putAll(numericalMetrics)
allMetrics.putAll(booleanMetrics)
allMetrics.putAll(stringMetrics)
}
for (entry in allMetrics.entries) {
val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}"
trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
}
trackingFile.append(BUILD_SESSION_SEPARATOR)
synchronized(metricsLock) {
stringMetrics.clear()
booleanMetrics.clear()
numericalMetrics.clear()
}
}
fun getMetric(metric: NumericalMetrics): IMetricContainer<Long>? = synchronized(metricsLock) {
numericalMetrics[MetricDescriptor(metric.name, null)]
}
fun getMetric(metric: StringMetrics): IMetricContainer<String>? = synchronized(metricsLock) {
stringMetrics[MetricDescriptor(metric.name, null)]
}
fun getMetric(metric: BooleanMetrics): IMetricContainer<Boolean>? = synchronized(metricsLock) {
booleanMetrics[MetricDescriptor(metric.name, null)]
}
}
@@ -0,0 +1,13 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
class NullRecordLogger : IRecordLogger {
override fun append(s: String) {}
override fun close() {
}
}
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.metrics
import org.jetbrains.kotlin.statistics.metrics.BooleanAnonymizationPolicy.*
import org.jetbrains.kotlin.statistics.metrics.BooleanOverridePolicy.*
enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: BooleanAnonymizationPolicy, val perProject: Boolean = false) {
// whether the build is executed from IDE or from console
EXECUTED_FROM_IDEA(OVERRIDE, SAFE),
// Build script
//annotation processors
ENABLED_KAPT(OR, SAFE),
ENABLED_DAGGER(OR, SAFE),
ENABLED_DATABINDING(OR, SAFE),
ENABLED_KOVER(OR, SAFE),
ENABLED_COMPILER_PLUGIN_ALL_OPEN(OR, SAFE),
ENABLED_COMPILER_PLUGIN_NO_ARG(OR, SAFE),
ENABLED_COMPILER_PLUGIN_SAM_WITH_RECEIVER(OR, SAFE),
ENABLED_COMPILER_PLUGIN_LOMBOK(OR, SAFE),
ENABLED_COMPILER_PLUGIN_PARSELIZE(OR, SAFE),
ENABLED_COMPILER_PLUGIN_ATOMICFU(OR, SAFE),
ENABLED_HMPP(OR, SAFE),
// Enabled features
BUILD_SRC_EXISTS(OR, SAFE),
BUILD_PREPARE_KOTLIN_BUILD_SCRIPT_MODEL(OR, SAFE),
GRADLE_BUILD_CACHE_USED(OVERRIDE, SAFE),
GRADLE_WORKER_API_USED(OVERRIDE, SAFE),
KOTLIN_OFFICIAL_CODESTYLE(OVERRIDE, SAFE),
KOTLIN_PROGRESSIVE_MODE(OVERRIDE, SAFE),
KOTLIN_KTS_USED(OR, SAFE),
JVM_COMPILER_IR_MODE(OR, SAFE),
JS_GENERATE_EXTERNALS(OR, SAFE),
JS_SOURCE_MAP(OR, SAFE),
JS_KLIB_INCREMENTAL(OR, SAFE),
JS_IR_INCREMENTAL(OR, SAFE),
// User scenarios
DEBUGGER_ENABLED(OVERRIDE, SAFE),
COMPILATION_STARTED(OVERRIDE, SAFE),
TESTS_EXECUTED(OVERRIDE, SAFE),
MAVEN_PUBLISH_EXECUTED(OVERRIDE, SAFE),
BUILD_FAILED(OVERRIDE, SAFE),
KOTLIN_COMPILATION_FAILED(OR, SAFE)
}
@@ -0,0 +1,118 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.metrics
import java.util.*
interface IMetricContainer<T> {
fun addValue(t: T, weight: Long? = null)
fun toStringRepresentation(): String
fun getValue(): T?
}
interface IMetricContainerFactory<T> {
fun newMetricContainer(): IMetricContainer<T>
fun fromStringRepresentation(state: String): IMetricContainer<T>?
}
open class OverrideMetricContainer<T>() : IMetricContainer<T> {
internal var myValue: T? = null
override fun addValue(t: T, weight: Long?) {
myValue = t
}
internal constructor(v: T?) : this() {
myValue = v
}
override fun toStringRepresentation(): String {
return myValue?.toString() ?: "null"
}
override fun getValue() = myValue
}
class OverrideVersionMetricContainer() : OverrideMetricContainer<String>() {
constructor(v: String) : this() {
myValue = v
}
override fun addValue(t: String, weight: Long?) {
if (myValue == null || myValue == "0.0.0") {
myValue = t
}
}
}
class SumMetricContainer() : OverrideMetricContainer<Long>() {
constructor(v: Long) : this() {
myValue = v
}
override fun addValue(t: Long, weight: Long?) {
myValue = (myValue ?: 0) + t
}
}
class AverageMetricContainer() : IMetricContainer<Long> {
private var totalWeight = 0L
private var totalSum: Long? = null
constructor(v: Long) : this() {
totalSum = v
totalWeight = 1
}
override fun addValue(t: Long, weight: Long?) {
val w = weight ?: 1
totalSum = (totalSum ?: 0) + t * w
totalWeight += w
}
override fun toStringRepresentation(): String {
return getValue()?.toString() ?: "null"
}
override fun getValue(): Long? {
return totalSum?.div(if (totalWeight > 0) totalWeight else 1)
}
}
class OrMetricContainer() : OverrideMetricContainer<Boolean>() {
constructor(v: Boolean) : this() {
myValue = v
}
override fun addValue(t: Boolean, weight: Long?) {
myValue = (myValue ?: false) || t
}
}
class ConcatMetricContainer() : IMetricContainer<String> {
private val myValues = TreeSet<String>()
companion object {
const val SEPARATOR = ";"
}
constructor(values: Collection<String>) : this() {
myValues.addAll(values)
}
override fun addValue(t: String, weight: Long?) {
myValues.add(t.replace(SEPARATOR, ","))
}
override fun toStringRepresentation(): String {
return myValues.sorted().joinToString(SEPARATOR)
}
override fun getValue() = toStringRepresentation()
}
@@ -0,0 +1,154 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.metrics
import org.jetbrains.kotlin.statistics.ValueAnonymizer
import org.jetbrains.kotlin.statistics.anonymizeComponentVersion
import org.jetbrains.kotlin.statistics.sha256
import kotlin.math.abs
enum class StringOverridePolicy: IMetricContainerFactory<String> {
OVERRIDE {
override fun newMetricContainer(): IMetricContainer<String> = OverrideMetricContainer<String>()
override fun fromStringRepresentation(state: String): IMetricContainer<String>? = OverrideMetricContainer(state)
},
OVERRIDE_VERSION_IF_NOT_SET {
override fun newMetricContainer(): IMetricContainer<String> = OverrideVersionMetricContainer()
override fun fromStringRepresentation(state: String): IMetricContainer<String>? = OverrideVersionMetricContainer(state)
},
CONCAT {
override fun newMetricContainer(): IMetricContainer<String> = ConcatMetricContainer()
override fun fromStringRepresentation(state: String): IMetricContainer<String>? = ConcatMetricContainer(state.split(ConcatMetricContainer.SEPARATOR))
}
//Should be useful counting container?
}
private fun applyIfLong(v: String, action: (Long) -> IMetricContainer<Long>) : IMetricContainer<Long>? {
val longVal = v.toLongOrNull()
return if (longVal == null) {
null
} else {
action(longVal)
}
}
enum class NumberOverridePolicy: IMetricContainerFactory<Long> {
OVERRIDE {
override fun newMetricContainer(): IMetricContainer<Long> = OverrideMetricContainer<Long>()
override fun fromStringRepresentation(state: String): IMetricContainer<Long>? = applyIfLong(state) {
OverrideMetricContainer(it)
}
},
SUM {
override fun newMetricContainer(): IMetricContainer<Long> = SumMetricContainer()
override fun fromStringRepresentation(state: String): IMetricContainer<Long>? = applyIfLong(state) {
SumMetricContainer(it)
}
},
AVERAGE {
override fun newMetricContainer(): IMetricContainer<Long> = AverageMetricContainer()
override fun fromStringRepresentation(state: String): IMetricContainer<Long>? = applyIfLong(state) {
AverageMetricContainer(it)
}
}
}
enum class BooleanOverridePolicy: IMetricContainerFactory<Boolean> {
OVERRIDE {
override fun newMetricContainer(): IMetricContainer<Boolean> = OverrideMetricContainer<Boolean>()
override fun fromStringRepresentation(state: String): IMetricContainer<Boolean>? = OverrideMetricContainer(state.toBoolean())
},
OR {
override fun newMetricContainer(): IMetricContainer<Boolean> = OrMetricContainer()
override fun fromStringRepresentation(state: String): IMetricContainer<Boolean>? = OrMetricContainer(state.toBoolean())
}
// may be add disctribution counter metric container
}
enum class BooleanAnonymizationPolicy : ValueAnonymizer<Boolean> {
SAFE {
override fun anonymize(t: Boolean) = t
}
}
abstract class StringAnonymizationPolicy : ValueAnonymizer<String> {
abstract fun validationRegexp(): String
class AllowedListAnonymizer(val allowedValues: Collection<String>) : StringAnonymizationPolicy() {
companion object {
const val UNEXPECTED_VALUE = "UNEXPECTED-VALUE"
}
override fun validationRegexp(): String {
return "^((${UNEXPECTED_VALUE}|${allowedValues.joinToString("|")})${ConcatMetricContainer.SEPARATOR}?)+$"
}
override fun anonymize(t: String): String {
return if (t.matches(Regex(validationRegexp()))) {
t
} else {
t.split(ConcatMetricContainer.SEPARATOR).joinToString(ConcatMetricContainer.SEPARATOR) {
if (allowedValues.contains(it))
it
else
UNEXPECTED_VALUE
}
}
}
}
class RegexControlled(private val regex: String, private val anonymizeInIde: Boolean) : StringAnonymizationPolicy() {
override fun validationRegexp() = regex
override fun anonymize(t: String) = t
override fun anonymizeOnIdeSize() = anonymizeInIde
}
class ComponentVersionAnonymizer() : StringAnonymizationPolicy() {
override fun validationRegexp() = "(\\d+).(\\d+).(\\d+)-?(dev|snapshot|m\\d?|rc\\d?|beta\\d?)?"
override fun anonymize(t: String) = anonymizeComponentVersion(t)
}
}
enum class NumberAnonymizationPolicy : ValueAnonymizer<Long> {
SAFE {
override fun anonymize(t: Long) = t
},
RANDOM_10_PERCENT {
override fun anonymize(t: Long): Long {
if (abs(t) < 10) return t
val sign = if (t < 0)
-1
else
1
val absT = t * sign
var div: Long = 1
while (div * 10 < absT) {
div *= 10
}
return sign * if (absT / div < 2)
absT - absT % (div / 10)
else
absT - absT % div
}
}
}
@@ -0,0 +1,66 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.metrics
import org.jetbrains.kotlin.statistics.metrics.NumberAnonymizationPolicy.*
import org.jetbrains.kotlin.statistics.metrics.NumberOverridePolicy.*
enum class NumericalMetrics(val type: NumberOverridePolicy, val anonymization: NumberAnonymizationPolicy, val perProject: Boolean = false) {
// User environment
// Number of CPU cores. No other information (e.g. env.PROCESSOR_IDENTIFIER is not reported)
CPU_NUMBER_OF_CORES(OVERRIDE, SAFE),
//Download speed in Bytes per second
ARTIFACTS_DOWNLOAD_SPEED(OVERRIDE, RANDOM_10_PERCENT),
// Build script
GRADLE_DAEMON_HEAP_SIZE(OVERRIDE, RANDOM_10_PERCENT),
GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON(OVERRIDE, SAFE),
// gradle configuration types
CONFIGURATION_API_COUNT(SUM, RANDOM_10_PERCENT),
CONFIGURATION_IMPLEMENTATION_COUNT(SUM, RANDOM_10_PERCENT),
CONFIGURATION_COMPILE_COUNT(SUM, RANDOM_10_PERCENT),
CONFIGURATION_RUNTIME_COUNT(SUM, RANDOM_10_PERCENT),
// gradle task types
GRADLE_NUMBER_OF_TASKS(SUM, RANDOM_10_PERCENT),
GRADLE_NUMBER_OF_UNCONFIGURED_TASKS(SUM, RANDOM_10_PERCENT),
GRADLE_NUMBER_OF_INCREMENTAL_TASKS(SUM, RANDOM_10_PERCENT),
//Features
BUILD_SRC_COUNT(SUM, RANDOM_10_PERCENT),
// Build performance
// duration of the whole gradle build
GRADLE_BUILD_DURATION(OVERRIDE, SAFE),
//duration of the execution gradle phase
GRADLE_EXECUTION_DURATION(OVERRIDE, SAFE),
//performance of compiler
COMPILATIONS_COUNT(SUM, RANDOM_10_PERCENT),
INCREMENTAL_COMPILATIONS_COUNT(SUM, RANDOM_10_PERCENT),
COMPILATION_DURATION(SUM, SAFE),
COMPILED_LINES_OF_CODE(SUM, RANDOM_10_PERCENT),
COMPILATION_LINES_PER_SECOND(OVERRIDE, SAFE),
ANALYSIS_LINES_PER_SECOND(AVERAGE, SAFE),
CODE_GENERATION_LINES_PER_SECOND(AVERAGE, SAFE),
NUMBER_OF_SUBPROJECTS(SUM, RANDOM_10_PERCENT),
STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD(SUM, RANDOM_10_PERCENT),
STATISTICS_COLLECT_METRICS_OVERHEAD(SUM, RANDOM_10_PERCENT),
// User scenarios
// this value is not reported, only time intervals from the previous build are used
BUILD_FINISH_TIME(OVERRIDE, SAFE)
}
@@ -0,0 +1,26 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.metrics
interface ReportStatisticsValue<T> {
val name: String
val value: T
}
class ReportOnceStatisticsValue<T>(override val name: String, override val value: T) :
ReportStatisticsValue<T>
interface AdditiveStatisticsValue<T> : ReportStatisticsValue<T> {
fun addValue(t: T)
}
interface IStatisticsValuesConsumer {
fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null, weight: Long? = null): Boolean
fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null, weight: Long? = null): Boolean
fun report(metric: StringMetrics, value: String, subprojectName: String? = null, weight: Long? = null): Boolean
}
@@ -0,0 +1,119 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.metrics
import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.*
import org.jetbrains.kotlin.statistics.metrics.StringOverridePolicy.*
enum class StringMetrics(val type: StringOverridePolicy, val anonymization: StringAnonymizationPolicy, val perProject: Boolean = false) {
// User environment
GRADLE_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
PROJECT_PATH(OVERRIDE, RegexControlled("([0-9A-Fa-f]{40,64})|undefined", true)),
OS_TYPE(OVERRIDE, RegexControlled("(Windows|Windows |Windows Server |Mac|Linux|FreeBSD|Solaris|Other|Mac OS X)\\d*", false)),
IDES_INSTALLED(CONCAT, AllowedListAnonymizer(listOf("AS", "OC", "CL", "IU", "IC", "WC"))),
// Build script
MPP_PLATFORMS(
CONCAT, AllowedListAnonymizer(
listOf(
"common",
"metadata",
"jvm",
"js",
"arm32",
"arm64",
"mips32",
"mipsel32",
"x64",
"android",
"androidJvm",
"androidApp",
"androidNativeArm",
"androidNativeArm32",
"android_arm32",
"androidNativeArm64",
"android_arm64",
"androidNative",
"androidNativeX86",
"androidNativeX64",
"iosArm",
"iosArm32",
"ios_arm32",
"iosArm64",
"ios_arm64",
"ios_simulator_arm64",
"ios",
"ios_x64",
"iosSim",
"iosX64",
"watchos",
"watchosArm32",
"watchosArm64",
"watchosX86",
"tvos",
"tvosArm64",
"tvosX64",
"linux",
"linuxArm32Hfp",
"linux_arm32_hfp",
"linuxMips32",
"linux_mips32",
"linuxMipsel32",
"linux_mipsel32",
"linuxX64",
"linux_arm64",
"linux_x64",
"macos",
"osx",
"macosX64",
"macos_x64",
"macos_arm64",
"mingw",
"mingwX64",
"mingw_x64",
"mingwX86",
"mingw_X86",
"mingw_x86",
"wasm32",
"wasm"
)
)
),
JS_COMPILER_MODE(CONCAT, AllowedListAnonymizer(listOf("ir", "legacy", "both", "UNKNOWN"))),
// Component versions
LIBRARY_SPRING_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
LIBRARY_VAADIN_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
LIBRARY_GWT_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
LIBRARY_HIBERNATE_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
KOTLIN_COMPILER_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
KOTLIN_STDLIB_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
KOTLIN_REFLECT_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
KOTLIN_COROUTINES_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
KOTLIN_SERIALIZATION_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
// Features
KOTLIN_LANGUAGE_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
KOTLIN_API_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
USE_CLASSPATH_SNAPSHOT(CONCAT, AllowedListAnonymizer(listOf("true", "false", "default-true"))),
JS_GENERATE_EXECUTABLE_DEFAULT(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
JS_TARGET_MODE(CONCAT, AllowedListAnonymizer(listOf("both", "browser", "nodejs", "none"))),
JS_OUTPUT_GRANULARITY(OVERRIDE, RegexControlled("(whole_program|per_module|per_file)", false)),
// Compiler parameters
JVM_DEFAULTS(CONCAT, AllowedListAnonymizer(listOf("disable", "enable", "compatibility", "all", "all-compatibility"))),
USE_OLD_BACKEND(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
USE_FIR(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
JS_PROPERTY_LAZY_INITIALIZATION(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
}