Metric description and anonymization rules added

Added metrics and their anonymization rules added. Persisted gradle statistical information will not contain any sensitive information. #KT-33404 Fixed
2019-12-25 07:35:58 +03:00
parent 763e02b3e8
commit 33d89e15b5
6 changed files with 347 additions and 8 deletions
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics
+
+import java.security.MessageDigest
+
+internal interface ValueAnonymizer<T> {
+
+    fun anonymize(t: T): T
+
+}
+
+internal val salt: String by lazy {
+    val env = System.getenv()
+    "${env["HOSTNAME"]}${env["COMPUTERNAME"]}"
+}
+
+fun anonymizeComponentVersion(version: String): String {
+    return version.replace('-', '.')
+        .split(".")
+        .filterIndexed { i, _ -> i < 3 }
+        .map { s -> s.toIntOrNull() ?: "?" }
+        .joinToString(".")
+}
+
+internal fun sha256(s: String): String {
+    val md = MessageDigest.getInstance("SHA-256")
+    val digest = md.digest(s.toByteArray())
+    return digest.fold("", { str, it -> str + "%02x".format(it) })
+}
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.metrics
+
+import org.jetbrains.kotlin.statistics.metrics.BooleanAnonymizationPolicy.*
+import org.jetbrains.kotlin.statistics.metrics.BooleanOverridePolicy.*
+
+
+enum class BooleanMetrics(val type: BooleanOverridePolicy, val anonymization: BooleanAnonymizationPolicy, val perProject: Boolean = false) {
+
+    // whether the build is executed from IDE or from console
+    EXECUTED_FROM_IDEA(OVERRIDE, SAFE),
+
+    // Build script
+
+    //annotation processors
+    ENABLED_KAPT(OR, SAFE),
+    ENABLED_DAGGER(OR, SAFE),
+    ENABLED_DATABINDING(OR, SAFE),
+
+    ENABLED_COMPILER_PLUGIN_ALL_OPEN(OR, SAFE),
+    ENABLED_COMPILER_PLUGIN_NO_ARG(OR, SAFE),
+    ENABLED_COMPILER_PLUGIN_JPA_SUPPORT(OR, SAFE),
+    ENABLED_COMPILER_PLUGIN_SAM_WITH_RECEIVER(OR, SAFE),
+
+    ENABLED_HMPP(OR, SAFE),
+
+    // Enabled features
+    BUILD_SRC_EXISTS(OR, SAFE),
+    GRADLE_BUILD_CACHE_USED(OVERRIDE, SAFE),
+    GRADLE_WORKER_API_USED(OVERRIDE, SAFE),
+
+    KOTLIN_OFFICIAL_CODESTYLE(OVERRIDE, SAFE),
+    KOTLIN_PROGRESSIVE_MODE(OVERRIDE, SAFE),
+    KOTLIN_KTS_USED(OVERRIDE, SAFE),
+
+    // User scenarios
+    DEBUGGER_ENABLED(OVERRIDE, SAFE),
+    COMPILATION_STARTED(OVERRIDE, SAFE),
+    TESTS_EXECUTED(OVERRIDE, SAFE),
+    MAVEN_PUBLISH_EXECUTED(OVERRIDE, SAFE),
+    BUILD_FAILED(OVERRIDE, SAFE)
+
+}
@@ -5,6 +5,8 @@

 package org.jetbrains.kotlin.statistics.metrics

+import java.util.*
+
 interface IMetricContainer<T> {
    fun addValue(t: T)

@@ -17,13 +19,12 @@ interface IMetricContainer<T> {
 interface IMetricContainerFactory<T> {
    fun newMetricContainer(): IMetricContainer<T>

-    //null if could not parse
    fun fromStringRepresentation(state: String): IMetricContainer<T>?
 }

-class OverrideMetricContainer<T>() : IMetricContainer<T> {
+open class OverrideMetricContainer<T>() : IMetricContainer<T> {

-    private var myValue: T? = null
+    internal var myValue: T? = null

    override fun addValue(t: T) {
        myValue = t
@@ -40,15 +41,67 @@ class OverrideMetricContainer<T>() : IMetricContainer<T> {
    override fun getValue() = myValue
 }

-class ConcatMetricContainer() : IMetricContainer<String> {
-    private val myValues = HashSet<String>()
+class SumMetricContainer() : OverrideMetricContainer<Long>() {

-    override fun addValue(t: String) {
-        myValues.add(t)
+    constructor(v: Long) : this() {
+        myValue = v
+    }
+
+    override fun addValue(t: Long) {
+        myValue = (myValue ?: 0) + t
+    }
+}
+
+class AverageMetricContainer() : IMetricContainer<Long> {
+    private var count = 0
+    private var myValue: Long? = null
+
+    constructor(v: Long) : this() {
+        myValue = v
+    }
+
+    override fun addValue(t: Long) {
+        myValue = (myValue ?: 0) + t
+        count++
    }

    override fun toStringRepresentation(): String {
-        return myValues.joinToString(";")
+        return getValue()?.toString() ?: "null"
+    }
+
+    override fun getValue(): Long? {
+        return myValue?.div(count)
+    }
+}
+
+class OrMetricContainer() : OverrideMetricContainer<Boolean>() {
+    constructor(v: Boolean) : this() {
+        myValue = v
+    }
+
+    override fun addValue(t: Boolean) {
+        myValue = (myValue ?: false) || t
+    }
+}
+
+class ConcatMetricContainer() : IMetricContainer<String> {
+
+    private val myValues = TreeSet<String>()
+
+    companion object {
+        const val SEPARATOR = ";"
+    }
+
+    constructor(values: Collection<String>) : this() {
+        myValues.addAll(values)
+    }
+
+    override fun addValue(t: String) {
+        myValues.add(t.replace(SEPARATOR, ","))
+    }
+
+    override fun toStringRepresentation(): String {
+        return myValues.joinToString(SEPARATOR)
    }

    override fun getValue() = toStringRepresentation()
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.metrics
+
+import org.jetbrains.kotlin.statistics.ValueAnonymizer
+import org.jetbrains.kotlin.statistics.anonymizeComponentVersion
+import org.jetbrains.kotlin.statistics.sha256
+
+
+enum class StringOverridePolicy: IMetricContainerFactory<String> {
+    OVERRIDE {
+        override fun newMetricContainer(): IMetricContainer<String> = OverrideMetricContainer<String>()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<String>? = OverrideMetricContainer(state)
+    },
+    CONCAT {
+        override fun newMetricContainer(): IMetricContainer<String> = ConcatMetricContainer()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<String>? = ConcatMetricContainer(state.split(ConcatMetricContainer.SEPARATOR))
+    }
+
+    //Should be useful counting container?
+}
+
+private fun applyIfLong(v: String, action: (Long) -> IMetricContainer<Long>) : IMetricContainer<Long>? {
+    val longVal = v.toLongOrNull()
+    return if (longVal == null) {
+        null
+    } else {
+        action(longVal)
+    }
+}
+
+enum class NumberOverridePolicy: IMetricContainerFactory<Long> {
+    OVERRIDE {
+        override fun newMetricContainer(): IMetricContainer<Long> = OverrideMetricContainer<Long>()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<Long>? = applyIfLong(state) {
+            OverrideMetricContainer(it)
+        }
+    },
+    SUM {
+        override fun newMetricContainer(): IMetricContainer<Long> = SumMetricContainer()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<Long>? = applyIfLong(state) {
+            SumMetricContainer(it)
+        }
+    },
+    AVERAGE {
+        override fun newMetricContainer(): IMetricContainer<Long> = AverageMetricContainer()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<Long>? = applyIfLong(state) {
+            AverageMetricContainer(it)
+        }
+    }
+}
+
+enum class BooleanOverridePolicy: IMetricContainerFactory<Boolean> {
+    OVERRIDE {
+        override fun newMetricContainer(): IMetricContainer<Boolean> = OverrideMetricContainer<Boolean>()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<Boolean>? = OverrideMetricContainer(state.toBoolean())
+    },
+    OR {
+        override fun newMetricContainer(): IMetricContainer<Boolean> = OrMetricContainer()
+
+        override fun fromStringRepresentation(state: String): IMetricContainer<Boolean>? = OrMetricContainer(state.toBoolean())
+    }
+
+    // may be add disctribution counter metric container
+}
+
+enum class BooleanAnonymizationPolicy : ValueAnonymizer<Boolean> {
+    SAFE {
+        override fun anonymize(t: Boolean) = t
+    }
+}
+
+enum class StringAnonymizationPolicy : ValueAnonymizer<String> {
+    SAFE {
+        override fun anonymize(t: String) = t
+    },
+    SHA_256 {
+        override fun anonymize(t: String) = sha256(t)
+    },
+    COMPONENT_VERSION {
+        override fun anonymize(t: String) = anonymizeComponentVersion(t)
+    }
+}
+
+enum class NumberAnonymizationPolicy : ValueAnonymizer<Long> {
+    SAFE {
+        override fun anonymize(t: Long) = t
+    },
+    RANDOM_10_PERCENT {
+        override fun anonymize(t: Long) = (t + t * 0.1 * Math.random()).toLong()
+    },
+    RANDOM_01_PERCENT {
+        override fun anonymize(t: Long) = (t + t * 0.01 * Math.random()).toLong()
+    }
+}
+
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.metrics
+
+import org.jetbrains.kotlin.statistics.metrics.NumberAnonymizationPolicy.*
+import org.jetbrains.kotlin.statistics.metrics.NumberOverridePolicy.*
+
+
+enum class NumericalMetrics(val type: NumberOverridePolicy, val anonymization: NumberAnonymizationPolicy, val perProject: Boolean = false) {
+
+
+    // User environment
+    // Number of CPU cores. No other information (e.g. env.PROCESSOR_IDENTIFIER is not reported)
+    CPU_NUMBER_OF_CORES(OVERRIDE, SAFE),
+
+    //Download speed in Bytes per second
+    ARTIFACTS_DOWNLOAD_SPEED(OVERRIDE, RANDOM_10_PERCENT),
+
+    // Build script
+    GRADLE_DAEMON_HEAP_SIZE(OVERRIDE, RANDOM_10_PERCENT),
+
+    GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON(OVERRIDE, SAFE),
+
+    // gradle configuration types
+    CONFIGURATION_API_COUNT(SUM, RANDOM_10_PERCENT),
+    CONFIGURATION_IMPLEMENTATION_COUNT(SUM, RANDOM_10_PERCENT),
+    CONFIGURATION_COMPILE_COUNT(SUM, RANDOM_10_PERCENT),
+    CONFIGURATION_RUNTIME_COUNT(SUM, RANDOM_10_PERCENT),
+
+    // gradle task types
+    GRADLE_NUMBER_OF_TASKS(SUM, RANDOM_10_PERCENT),
+    GRADLE_NUMBER_OF_UNCONFIGURED_TASKS(SUM, RANDOM_10_PERCENT),
+    GRADLE_NUMBER_OF_INCREMENTAL_TASKS(SUM, RANDOM_10_PERCENT),
+
+    //Features
+    BUILD_SRC_COUNT(SUM, RANDOM_10_PERCENT),
+
+    // Build performance
+    // duration of the whole gradle build
+    GRADLE_BUILD_DURATION(OVERRIDE, SAFE),
+
+    //duration of the execution gradle phase
+    GRADLE_EXECUTION_DURATION(OVERRIDE, SAFE),
+
+    NUMBER_OF_SUBPROJECTS(OVERRIDE, RANDOM_10_PERCENT),
+
+    // User scenarios
+
+    // this value is not reported, only time intervals from the previous build are used
+    BUILD_FINISH_TIME(OVERRIDE, SAFE)
+}
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.statistics.metrics
+
+import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.*
+import org.jetbrains.kotlin.statistics.metrics.StringOverridePolicy.*
+
+
+enum class StringMetrics(val type: StringOverridePolicy, val anonymization: StringAnonymizationPolicy, val perProject: Boolean = false) {
+
+    // User environment
+    GRADLE_VERSION(OVERRIDE, COMPONENT_VERSION),
+
+    OS_TYPE(OVERRIDE, SAFE),
+
+    //TODO could we collect only JB IDEs or, e.g. WsCode?
+    IDES_INSTALLED(CONCAT, SAFE),
+
+    // Build script
+    MPP_PLATFORMS(CONCAT, SAFE),
+
+    // Component versions
+    LIBRARY_SPRING_VERSION(OVERRIDE, COMPONENT_VERSION),
+    LIBRARY_VAADIN_VERSION(OVERRIDE, COMPONENT_VERSION),
+    LIBRARY_GWT_VERSION(OVERRIDE, COMPONENT_VERSION),
+    LIBRARY_HYBERNATE_VERSION(OVERRIDE, COMPONENT_VERSION),
+
+    KOTLIN_COMPILER_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_STDLIB_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_REFLECT_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_COROUTINES_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_SERIALIZATION_VERSION(OVERRIDE, COMPONENT_VERSION),
+
+    ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, COMPONENT_VERSION),
+
+    // Features
+    KOTLIN_LANGUAGE_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_API_VERSION(OVERRIDE, COMPONENT_VERSION),
+
+
+
+}
+
+