diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/AnonymizerUtils.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/AnonymizerUtils.kt new file mode 100644 index 00000000000..e06c933196f --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/AnonymizerUtils.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics + +import java.security.MessageDigest + +internal interface ValueAnonymizer { + + fun anonymize(t: T): T + +} + +internal val salt: String by lazy { + val env = System.getenv() + "${env["HOSTNAME"]}${env["COMPUTERNAME"]}" +} + +fun anonymizeComponentVersion(version: String): String { + return version.replace('-', '.') + .split(".") + .filterIndexed { i, _ -> i < 3 } + .map { s -> s.toIntOrNull() ?: "?" } + .joinToString(".") +} + +internal fun sha256(s: String): String { + val md = MessageDigest.getInstance("SHA-256") + val digest = md.digest(s.toByteArray()) + return digest.fold("", { str, it -> str + "%02x".format(it) }) +} diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/BooleanMetrics.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/BooleanMetrics.kt new file mode 100644 index 00000000000..b62d0434ae5 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/BooleanMetrics.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.metrics + +import org.jetbrains.kotlin.statistics.metrics.BooleanAnonymizationPolicy.* +import org.jetbrains.kotlin.statistics.metrics.BooleanOverridePolicy.* + + +enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: BooleanAnonymizationPolicy, val perProject: Boolean = false) { + + // whether the build is executed from IDE or from console + EXECUTED_FROM_IDEA(OVERRIDE, SAFE), + + // Build script + + //annotation processors + ENABLED_KAPT(OR, SAFE), + ENABLED_DAGGER(OR, SAFE), + ENABLED_DATABINDING(OR, SAFE), + + ENABLED_COMPILER_PLUGIN_ALL_OPEN(OR, SAFE), + ENABLED_COMPILER_PLUGIN_NO_ARG(OR, SAFE), + ENABLED_COMPILER_PLUGIN_JPA_SUPPORT(OR, SAFE), + ENABLED_COMPILER_PLUGIN_SAM_WITH_RECEIVER(OR, SAFE), + + ENABLED_HMPP(OR, SAFE), + + // Enabled features + BUILD_SRC_EXISTS(OR, SAFE), + GRADLE_BUILD_CACHE_USED(OVERRIDE, SAFE), + GRADLE_WORKER_API_USED(OVERRIDE, SAFE), + + KOTLIN_OFFICIAL_CODESTYLE(OVERRIDE, SAFE), + KOTLIN_PROGRESSIVE_MODE(OVERRIDE, SAFE), + KOTLIN_KTS_USED(OVERRIDE, SAFE), + + // User scenarios + DEBUGGER_ENABLED(OVERRIDE, SAFE), + COMPILATION_STARTED(OVERRIDE, SAFE), + TESTS_EXECUTED(OVERRIDE, SAFE), + MAVEN_PUBLISH_EXECUTED(OVERRIDE, SAFE), + BUILD_FAILED(OVERRIDE, SAFE) + +} diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt index 7f7a19eda6a..9633c82c168 100644 --- a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.statistics.metrics +import java.util.* + interface IMetricContainer { fun addValue(t: T) @@ -17,13 +19,12 @@ interface IMetricContainer { interface IMetricContainerFactory { fun newMetricContainer(): IMetricContainer - //null if could not parse fun fromStringRepresentation(state: String): IMetricContainer? } -class OverrideMetricContainer() : IMetricContainer { +open class OverrideMetricContainer() : IMetricContainer { - private var myValue: T? = null + internal var myValue: T? = null override fun addValue(t: T) { myValue = t @@ -40,15 +41,67 @@ class OverrideMetricContainer() : IMetricContainer { override fun getValue() = myValue } -class ConcatMetricContainer() : IMetricContainer { - private val myValues = HashSet() +class SumMetricContainer() : OverrideMetricContainer() { - override fun addValue(t: String) { - myValues.add(t) + constructor(v: Long) : this() { + myValue = v + } + + override fun addValue(t: Long) { + myValue = (myValue ?: 0) + t + } +} + +class AverageMetricContainer() : IMetricContainer { + private var count = 0 + private var myValue: Long? = null + + constructor(v: Long) : this() { + myValue = v + } + + override fun addValue(t: Long) { + myValue = (myValue ?: 0) + t + count++ } override fun toStringRepresentation(): String { - return myValues.joinToString(";") + return getValue()?.toString() ?: "null" + } + + override fun getValue(): Long? { + return myValue?.div(count) + } +} + +class OrMetricContainer() : OverrideMetricContainer() { + constructor(v: Boolean) : this() { + myValue = v + } + + override fun addValue(t: Boolean) { + myValue = (myValue ?: false) || t + } +} + +class ConcatMetricContainer() : IMetricContainer { + + private val myValues = TreeSet() + + companion object { + const val SEPARATOR = ";" + } + + constructor(values: Collection) : this() { + myValues.addAll(values) + } + + override fun addValue(t: String) { + myValues.add(t.replace(SEPARATOR, ",")) + } + + override fun toStringRepresentation(): String { + return myValues.joinToString(SEPARATOR) } override fun getValue() = toStringRepresentation() diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricPolicies.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricPolicies.kt new file mode 100644 index 00000000000..483f898daa9 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricPolicies.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.metrics + +import org.jetbrains.kotlin.statistics.ValueAnonymizer +import org.jetbrains.kotlin.statistics.anonymizeComponentVersion +import org.jetbrains.kotlin.statistics.sha256 + + +enum class StringOverridePolicy: IMetricContainerFactory { + OVERRIDE { + override fun newMetricContainer(): IMetricContainer = OverrideMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = OverrideMetricContainer(state) + }, + CONCAT { + override fun newMetricContainer(): IMetricContainer = ConcatMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = ConcatMetricContainer(state.split(ConcatMetricContainer.SEPARATOR)) + } + + //Should be useful counting container? +} + +private fun applyIfLong(v: String, action: (Long) -> IMetricContainer) : IMetricContainer? { + val longVal = v.toLongOrNull() + return if (longVal == null) { + null + } else { + action(longVal) + } +} + +enum class NumberOverridePolicy: IMetricContainerFactory { + OVERRIDE { + override fun newMetricContainer(): IMetricContainer = OverrideMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = applyIfLong(state) { + OverrideMetricContainer(it) + } + }, + SUM { + override fun newMetricContainer(): IMetricContainer = SumMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = applyIfLong(state) { + SumMetricContainer(it) + } + }, + AVERAGE { + override fun newMetricContainer(): IMetricContainer = AverageMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = applyIfLong(state) { + AverageMetricContainer(it) + } + } +} + +enum class BooleanOverridePolicy: IMetricContainerFactory { + OVERRIDE { + override fun newMetricContainer(): IMetricContainer = OverrideMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = OverrideMetricContainer(state.toBoolean()) + }, + OR { + override fun newMetricContainer(): IMetricContainer = OrMetricContainer() + + override fun fromStringRepresentation(state: String): IMetricContainer? = OrMetricContainer(state.toBoolean()) + } + + // may be add disctribution counter metric container +} + +enum class BooleanAnonymizationPolicy : ValueAnonymizer { + SAFE { + override fun anonymize(t: Boolean) = t + } +} + +enum class StringAnonymizationPolicy : ValueAnonymizer { + SAFE { + override fun anonymize(t: String) = t + }, + SHA_256 { + override fun anonymize(t: String) = sha256(t) + }, + COMPONENT_VERSION { + override fun anonymize(t: String) = anonymizeComponentVersion(t) + } +} + +enum class NumberAnonymizationPolicy : ValueAnonymizer { + SAFE { + override fun anonymize(t: Long) = t + }, + RANDOM_10_PERCENT { + override fun anonymize(t: Long) = (t + t * 0.1 * Math.random()).toLong() + }, + RANDOM_01_PERCENT { + override fun anonymize(t: Long) = (t + t * 0.01 * Math.random()).toLong() + } +} + diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/NumericalMetrics.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/NumericalMetrics.kt new file mode 100644 index 00000000000..498b701bb09 --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/NumericalMetrics.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.metrics + +import org.jetbrains.kotlin.statistics.metrics.NumberAnonymizationPolicy.* +import org.jetbrains.kotlin.statistics.metrics.NumberOverridePolicy.* + + +enum class NumericalMetrics(val type: NumberOverridePolicy, val anonymization: NumberAnonymizationPolicy, val perProject: Boolean = false) { + + + // User environment + // Number of CPU cores. No other information (e.g. env.PROCESSOR_IDENTIFIER is not reported) + CPU_NUMBER_OF_CORES(OVERRIDE, SAFE), + + //Download speed in Bytes per second + ARTIFACTS_DOWNLOAD_SPEED(OVERRIDE, RANDOM_10_PERCENT), + + // Build script + GRADLE_DAEMON_HEAP_SIZE(OVERRIDE, RANDOM_10_PERCENT), + + GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON(OVERRIDE, SAFE), + + // gradle configuration types + CONFIGURATION_API_COUNT(SUM, RANDOM_10_PERCENT), + CONFIGURATION_IMPLEMENTATION_COUNT(SUM, RANDOM_10_PERCENT), + CONFIGURATION_COMPILE_COUNT(SUM, RANDOM_10_PERCENT), + CONFIGURATION_RUNTIME_COUNT(SUM, RANDOM_10_PERCENT), + + // gradle task types + GRADLE_NUMBER_OF_TASKS(SUM, RANDOM_10_PERCENT), + GRADLE_NUMBER_OF_UNCONFIGURED_TASKS(SUM, RANDOM_10_PERCENT), + GRADLE_NUMBER_OF_INCREMENTAL_TASKS(SUM, RANDOM_10_PERCENT), + + //Features + BUILD_SRC_COUNT(SUM, RANDOM_10_PERCENT), + + // Build performance + // duration of the whole gradle build + GRADLE_BUILD_DURATION(OVERRIDE, SAFE), + + //duration of the execution gradle phase + GRADLE_EXECUTION_DURATION(OVERRIDE, SAFE), + + NUMBER_OF_SUBPROJECTS(OVERRIDE, RANDOM_10_PERCENT), + + // User scenarios + + // this value is not reported, only time intervals from the previous build are used + BUILD_FINISH_TIME(OVERRIDE, SAFE) +} diff --git a/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StringMetrics.kt b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StringMetrics.kt new file mode 100644 index 00000000000..c9136f1d96e --- /dev/null +++ b/libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StringMetrics.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.statistics.metrics + +import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.* +import org.jetbrains.kotlin.statistics.metrics.StringOverridePolicy.* + + +enum class StringMetrics(val type: StringOverridePolicy, val anonymization: StringAnonymizationPolicy, val perProject: Boolean = false) { + + // User environment + GRADLE_VERSION(OVERRIDE, COMPONENT_VERSION), + + OS_TYPE(OVERRIDE, SAFE), + + //TODO could we collect only JB IDEs or, e.g. WsCode? + IDES_INSTALLED(CONCAT, SAFE), + + // Build script + MPP_PLATFORMS(CONCAT, SAFE), + + // Component versions + LIBRARY_SPRING_VERSION(OVERRIDE, COMPONENT_VERSION), + LIBRARY_VAADIN_VERSION(OVERRIDE, COMPONENT_VERSION), + LIBRARY_GWT_VERSION(OVERRIDE, COMPONENT_VERSION), + LIBRARY_HYBERNATE_VERSION(OVERRIDE, COMPONENT_VERSION), + + KOTLIN_COMPILER_VERSION(OVERRIDE, COMPONENT_VERSION), + KOTLIN_STDLIB_VERSION(OVERRIDE, COMPONENT_VERSION), + KOTLIN_REFLECT_VERSION(OVERRIDE, COMPONENT_VERSION), + KOTLIN_COROUTINES_VERSION(OVERRIDE, COMPONENT_VERSION), + KOTLIN_SERIALIZATION_VERSION(OVERRIDE, COMPONENT_VERSION), + + ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, COMPONENT_VERSION), + + // Features + KOTLIN_LANGUAGE_VERSION(OVERRIDE, COMPONENT_VERSION), + KOTLIN_API_VERSION(OVERRIDE, COMPONENT_VERSION), + + + +} + +