Added validation rules of FUS metrics on Gradle side

#KT-55541 Fixed
2022-12-16 13:05:32 +03:00
parent 0f33722124
commit 4c80538a78
9 changed files with 196 additions and 59 deletions
@@ -37,6 +37,7 @@ data class BuildOptions(
    val useFir: Boolean = false,
    val usePreciseJavaTracking: Boolean? = null,
    val freeArgs: List<String> = emptyList(),
+    val statisticsForceValidation: Boolean = true,
 ) {
    data class KaptOptions(
        val verbose: Boolean = false,
@@ -139,6 +140,10 @@ data class BuildOptions(
            arguments.add("-Pkotlin.incremental.usePreciseJavaTracking=$usePreciseJavaTracking")
        }

+        if (statisticsForceValidation) {
+            arguments.add("-Pkotlin_performance_profile_force_validation=true")
+        }
+
        arguments.addAll(freeArgs)

        return arguments.toList()
@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.statistics.BuildSessionLogger
 import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
 import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
 import org.jetbrains.kotlin.statistics.metrics.StringMetrics
+import org.jetbrains.kotlin.statistics.MetricValueValidationFailed
 import java.io.File
 import java.lang.management.ManagementFactory
 import javax.management.MBeanServer
@@ -32,6 +33,8 @@ class KotlinBuildStatHandler {
            return try {
                getLogger().debug("Executing [$methodName]")
                action.invoke()
+            } catch (e: MetricValueValidationFailed) {
+                throw e
            } catch (e: Throwable) {
                logException("Could not execute [$methodName]", e)
                null
@@ -49,6 +49,9 @@ internal abstract class KotlinBuildStatsService internal constructor() : BuildAd
        // Property name for disabling saving statistical information
        const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile"

+        // Property used for tests. Build will fail fast if collected value doesn't fit regexp
+        const val FORCE_VALUES_VALIDATION = "kotlin_performance_profile_force_validation"
+
        // default state
        const val DEFAULT_STATISTICS_STATE = true

@@ -191,7 +194,8 @@ internal class DefaultKotlinBuildStatsService internal constructor(
    gradle: Gradle,
    val beanName: ObjectName
 ) : KotlinBuildStatsService(), KotlinBuildStatsMXBean {
-    private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir)
+    private val forcePropertiesValidation = gradle.rootProject.properties[FORCE_VALUES_VALIDATION]?.toString()?.toBoolean() ?: false
+    private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir, forceValuesValidation = forcePropertiesValidation)

    private fun gradleBuildStartTime(gradle: Gradle): Long? {
        return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime
@@ -40,3 +40,5 @@ internal fun sha256(s: String): String {
    val digest = md.digest(s.toByteArray())
    return digest.fold("", { str, it -> str + "%02x".format(it) })
 }
+
+class MetricValueValidationFailed(message: String) : RuntimeException(message)
@@ -19,7 +19,8 @@ class BuildSessionLogger(
    rootPath: File,
    private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
    private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
-    private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE
+    private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
+    private val forceValuesValidation: Boolean = false
 ) : IStatisticsValuesConsumer {

    companion object {
@@ -46,7 +47,7 @@ class BuildSessionLogger(
    private var buildSession: BuildSession? = null
    private var trackingFile: IRecordLogger? = null

-    private val metricsContainer = MetricsContainer()
+    private val metricsContainer = MetricsContainer(forceValuesValidation)

    @Synchronized
    fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
@@ -5,7 +5,9 @@

 package org.jetbrains.kotlin.statistics.fileloggers

+import org.jetbrains.kotlin.statistics.MetricValueValidationFailed
 import org.jetbrains.kotlin.statistics.metrics.*
+import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.AllowedListAnonymizer.Companion.UNEXPECTED_VALUE
 import org.jetbrains.kotlin.statistics.sha256
 import java.io.*
 import java.nio.channels.Channels
@@ -14,7 +16,7 @@ import java.nio.file.Paths
 import java.nio.file.StandardOpenOption
 import java.util.*

-class MetricsContainer : IStatisticsValuesConsumer {
+class MetricsContainer(private val forceValuesValidation: Boolean = false) : IStatisticsValuesConsumer {
    data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
        override fun compareTo(other: MetricDescriptor): Int {
            val compareNames = name.compareTo(other.name)
@@ -129,7 +131,14 @@ class MetricsContainer : IStatisticsValuesConsumer {
        synchronized(metricsLock) {
            val metricContainer = stringMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
                .also { stringMetrics[MetricDescriptor(metric.name, projectHash)] = it }
-            metricContainer.addValue(metric.anonymization.anonymize(value), weight)
+
+            val anonymizedValue = metric.anonymization.anonymize(value)
+            if (forceValuesValidation && !metric.anonymization.anonymizeOnIdeSize()) {
+                if (anonymizedValue.contains(UNEXPECTED_VALUE) || !anonymizedValue.matches(Regex(metric.anonymization.validationRegexp()))) {
+                    throw MetricValueValidationFailed("Metric ${metric.name} has value [${value}], after anonymization [${anonymizedValue}]. Validation regex: ${metric.anonymization.validationRegexp()}.")
+                }
+            }
+            metricContainer.addValue(anonymizedValue, weight)
        }
        return true
    }
@@ -85,18 +85,45 @@ enum class BooleanAnonymizationPolicy : ValueAnonymizer<Boolean> {
    }
 }

-enum class StringAnonymizationPolicy : ValueAnonymizer<String> {
-    SAFE {
+abstract class StringAnonymizationPolicy : ValueAnonymizer<String> {
+
+    abstract fun validationRegexp(): String
+
+    class AllowedListAnonymizer(val allowedValues: Collection<String>) : StringAnonymizationPolicy() {
+        companion object {
+            const val UNEXPECTED_VALUE = "UNEXPECTED-VALUE"
+        }
+
+        override fun validationRegexp(): String {
+            return "^((${UNEXPECTED_VALUE}|${allowedValues.joinToString("|")})${ConcatMetricContainer.SEPARATOR}?)+$"
+        }
+
+        override fun anonymize(t: String): String {
+            return if (t.matches(Regex(validationRegexp()))) {
+                t
+            } else {
+                t.split(ConcatMetricContainer.SEPARATOR).joinToString(ConcatMetricContainer.SEPARATOR) {
+                    if (allowedValues.contains(it))
+                        it
+                    else
+                        UNEXPECTED_VALUE
+                }
+            }
+        }
+    }
+
+    class RegexControlled(private val regex: String, private val anonymizeInIde: Boolean) : StringAnonymizationPolicy() {
+        override fun validationRegexp() = regex
+
        override fun anonymize(t: String) = t
-    },
-    ANONYMIZE_IN_IDE {
-        override fun anonymize(t: String) = t
-        override fun anonymizeOnIdeSize() = true
-    },
-    SHA_256 {
-        override fun anonymize(t: String) = sha256(t)
-    },
-    COMPONENT_VERSION {
+
+        override fun anonymizeOnIdeSize() = anonymizeInIde
+
+    }
+
+    class ComponentVersionAnonymizer() : StringAnonymizationPolicy() {
+        override fun validationRegexp() = "(\\d+).(\\d+).(\\d+)-?(dev|snapshot|m\\d?|rc\\d?|beta\\d?)?"
+
        override fun anonymize(t: String) = anonymizeComponentVersion(t)
    }
 }
@@ -12,43 +12,107 @@ import org.jetbrains.kotlin.statistics.metrics.StringOverridePolicy.*
 enum class StringMetrics(val type: StringOverridePolicy, val anonymization: StringAnonymizationPolicy, val perProject: Boolean = false) {

    // User environment
-    GRADLE_VERSION(OVERRIDE, COMPONENT_VERSION),
-    PROJECT_PATH(OVERRIDE, ANONYMIZE_IN_IDE),
+    GRADLE_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    PROJECT_PATH(OVERRIDE, RegexControlled("([0-9A-Fa-f]{40,64})|undefined", true)),

-    OS_TYPE(OVERRIDE, SAFE),
+    OS_TYPE(OVERRIDE, RegexControlled("(Windows|Windows |Windows Server |Mac|Linux|FreeBSD|Solaris|Other|Mac OS X)\\d*", false)),

-    //TODO could we collect only JB IDEs or, e.g. WsCode?
-    IDES_INSTALLED(CONCAT, SAFE),
+    IDES_INSTALLED(CONCAT, AllowedListAnonymizer(listOf("AS", "OC", "CL", "IU", "IC", "WC"))),

    // Build script
-    MPP_PLATFORMS(CONCAT, SAFE),
-    JS_COMPILER_MODE(CONCAT, SAFE),
+    MPP_PLATFORMS(
+        CONCAT, AllowedListAnonymizer(
+            listOf(
+                "common",
+                "metadata",
+                "jvm",
+                "js",
+                "arm32",
+                "arm64",
+                "mips32",
+                "mipsel32",
+                "x64",
+                "android",
+                "androidJvm",
+                "androidApp",
+                "androidNativeArm",
+                "androidNativeArm32",
+                "android_arm32",
+                "androidNativeArm64",
+                "android_arm64",
+                "androidNative",
+                "androidNativeX86",
+                "androidNativeX64",
+                "iosArm",
+                "iosArm32",
+                "ios_arm32",
+                "iosArm64",
+                "ios_arm64",
+                "ios_simulator_arm64",
+                "ios",
+                "ios_x64",
+                "iosSim",
+                "iosX64",
+                "watchos",
+                "watchosArm32",
+                "watchosArm64",
+                "watchosX86",
+                "tvos",
+                "tvosArm64",
+                "tvosX64",
+                "linux",
+                "linuxArm32Hfp",
+                "linux_arm32_hfp",
+                "linuxMips32",
+                "linux_mips32",
+                "linuxMipsel32",
+                "linux_mipsel32",
+                "linuxX64",
+                "linux_arm64",
+                "linux_x64",
+                "macos",
+                "osx",
+                "macosX64",
+                "macos_x64",
+                "macos_arm64",
+                "mingw",
+                "mingwX64",
+                "mingw_x64",
+                "mingwX86",
+                "mingw_X86",
+                "mingw_x86",
+                "wasm32",
+                "wasm"
+            )
+        )
+    ),
+    JS_COMPILER_MODE(CONCAT, AllowedListAnonymizer(listOf("ir", "legacy", "both", "UNKNOWN"))),

    // Component versions
-    LIBRARY_SPRING_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
-    LIBRARY_VAADIN_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
-    LIBRARY_GWT_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
-    LIBRARY_HIBERNATE_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
+    LIBRARY_SPRING_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
+    LIBRARY_VAADIN_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
+    LIBRARY_GWT_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
+    LIBRARY_HIBERNATE_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),

-    KOTLIN_COMPILER_VERSION(OVERRIDE, COMPONENT_VERSION),
-    KOTLIN_STDLIB_VERSION(OVERRIDE, COMPONENT_VERSION),
-    KOTLIN_REFLECT_VERSION(OVERRIDE, COMPONENT_VERSION),
-    KOTLIN_COROUTINES_VERSION(OVERRIDE, COMPONENT_VERSION),
-    KOTLIN_SERIALIZATION_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_COMPILER_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    KOTLIN_STDLIB_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    KOTLIN_REFLECT_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    KOTLIN_COROUTINES_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    KOTLIN_SERIALIZATION_VERSION(OVERRIDE, ComponentVersionAnonymizer()),

-    ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, COMPONENT_VERSION),
+    ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, ComponentVersionAnonymizer()),

    // Features
-    KOTLIN_LANGUAGE_VERSION(OVERRIDE, COMPONENT_VERSION),
-    KOTLIN_API_VERSION(OVERRIDE, COMPONENT_VERSION),
-    JS_GENERATE_EXECUTABLE_DEFAULT(CONCAT, SAFE),
-    JS_TARGET_MODE(CONCAT, SAFE),
-    JS_OUTPUT_GRANULARITY(OVERRIDE, SAFE),
+    KOTLIN_LANGUAGE_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    KOTLIN_API_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
+    JS_GENERATE_EXECUTABLE_DEFAULT(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
+    JS_TARGET_MODE(CONCAT, AllowedListAnonymizer(listOf("both", "browser", "nodejs", "none"))),
+    JS_OUTPUT_GRANULARITY(OVERRIDE, RegexControlled("(whole_program|per_module|per_file)", false)),

    // Compiler parameters
-    JVM_DEFAULTS(CONCAT, SAFE),
-    USE_OLD_BACKEND(CONCAT, SAFE),
-    USE_FIR(CONCAT, SAFE),
+    JVM_DEFAULTS(CONCAT, AllowedListAnonymizer(listOf("disable", "enable", "compatibility", "all", "all-compatibility"))),
+    USE_OLD_BACKEND(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
+    USE_FIR(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),

-    JS_PROPERTY_LAZY_INITIALIZATION(CONCAT, SAFE),
+    JS_PROPERTY_LAZY_INITIALIZATION(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
 }
@@ -2,10 +2,14 @@
 * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
+import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
+import org.jetbrains.kotlin.statistics.metrics.ConcatMetricContainer
 import org.jetbrains.kotlin.statistics.metrics.NumberAnonymizationPolicy.RANDOM_10_PERCENT
-import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.COMPONENT_VERSION
+import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy
+import org.jetbrains.kotlin.statistics.metrics.StringMetrics
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertTrue


 class MetricPolicyTest {
@@ -28,25 +32,43 @@ class MetricPolicyTest {

    @Test
    fun versionAnonymization() {
-        assertEquals("0.0.0", COMPONENT_VERSION.anonymize("some.invalid.string"))
-        assertEquals("1.0.0", COMPONENT_VERSION.anonymize("1"))
-        assertEquals("1.2.0", COMPONENT_VERSION.anonymize("1.2"))
+        assertEquals("0.0.0", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("some.invalid.string"))
+        assertEquals("1.0.0", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1"))
+        assertEquals("1.2.0", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2"))

-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3"))
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3"))
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3.4"))
-        assertEquals("1.2.3-m", COMPONENT_VERSION.anonymize("1.2.3-M"))
-        assertEquals("1.2.3-m1", COMPONENT_VERSION.anonymize("1.2.3-M1"))
-        assertEquals("1.2.3-m2", COMPONENT_VERSION.anonymize("1.2.3.M2"))
-        assertEquals("1.2.3-rc", COMPONENT_VERSION.anonymize("1.2.3-RC"))
-        assertEquals("1.2.3-rc5", COMPONENT_VERSION.anonymize("1.2.3-RC5"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3.4"))
+        assertEquals("1.2.3-m", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-M"))
+        assertEquals("1.2.3-m1", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-M1"))
+        assertEquals("1.2.3-m2", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3.M2"))
+        assertEquals("1.2.3-rc", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-RC"))
+        assertEquals("1.2.3-rc5", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-RC5"))

-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3.unknown suffix"))
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3-unknown suffix"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3.unknown suffix"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-unknown suffix"))

-        assertEquals("123.234.345-dev", COMPONENT_VERSION.anonymize("123.234.345-dev-12345"))
-        assertEquals("1.8.255-snapshot", COMPONENT_VERSION.anonymize("1.8.255-SNAPSHOT"))
+        assertEquals("123.234.345-dev", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("123.234.345-dev-12345"))
+        assertEquals("1.8.255-snapshot", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.8.255-SNAPSHOT"))

-        assertEquals("1.8.255-beta", COMPONENT_VERSION.anonymize("1.8.255-beta"))
+        assertEquals("1.8.255-beta", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.8.255-beta"))
+    }
+
+    @Test
+    fun versionStringValidation() {
+        val separator = ConcatMetricContainer.SEPARATOR
+        val container = MetricsContainer()
+        fun whenAdded(newValue: String, expected: String) {
+            container.report(StringMetrics.MPP_PLATFORMS, newValue)
+            val currentValue = container.getMetric(StringMetrics.MPP_PLATFORMS)!!.toStringRepresentation()
+            assertEquals(expected, currentValue)
+            val regex = StringMetrics.MPP_PLATFORMS.anonymization.validationRegexp()
+            assertTrue(currentValue.matches(Regex(regex)), "'${currentValue}' should match '${regex}'")
+        }
+        whenAdded("js", "js")
+        whenAdded("common", "common${separator}js")
+        whenAdded("jvm", "common${separator}js${separator}jvm")
+        whenAdded("AAA", "UNEXPECTED-VALUE${separator}common${separator}js${separator}jvm")
+        whenAdded("BBB", "UNEXPECTED-VALUE${separator}common${separator}js${separator}jvm")
    }
 }