Added validation rules of FUS metrics on Gradle side

#KT-55541 Fixed

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt

@@ -37,6 +37,7 @@ data class BuildOptions(
     val useFir: Boolean = false,
     val usePreciseJavaTracking: Boolean? = null,
     val freeArgs: List<String> = emptyList(),
+    val statisticsForceValidation: Boolean = true,
 ) {
     data class KaptOptions(
         val verbose: Boolean = false,
@@ -139,6 +140,10 @@ data class BuildOptions(
             arguments.add("-Pkotlin.incremental.usePreciseJavaTracking=$usePreciseJavaTracking")
         }
 
+        if (statisticsForceValidation) {
+            arguments.add("-Pkotlin_performance_profile_force_validation=true")
+        }
+
         arguments.addAll(freeArgs)
 
         return arguments.toList()

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatHandler.kt

@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.statistics.BuildSessionLogger
 import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
 import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
 import org.jetbrains.kotlin.statistics.metrics.StringMetrics
+import org.jetbrains.kotlin.statistics.MetricValueValidationFailed
 import java.io.File
 import java.lang.management.ManagementFactory
 import javax.management.MBeanServer
@@ -32,6 +33,8 @@ class KotlinBuildStatHandler {
             return try {
                 getLogger().debug("Executing [$methodName]")
                 action.invoke()
+            } catch (e: MetricValueValidationFailed) {
+                throw e
             } catch (e: Throwable) {
                 logException("Could not execute [$methodName]", e)
                 null

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt

@@ -49,6 +49,9 @@ internal abstract class KotlinBuildStatsService internal constructor() : BuildAd
         // Property name for disabling saving statistical information
         const val ENABLE_STATISTICS_PROPERTY_NAME = "enable_kotlin_performance_profile"
 
+        // Property used for tests. Build will fail fast if collected value doesn't fit regexp
+        const val FORCE_VALUES_VALIDATION = "kotlin_performance_profile_force_validation"
+
         // default state
         const val DEFAULT_STATISTICS_STATE = true
 
@@ -191,7 +194,8 @@ internal class DefaultKotlinBuildStatsService internal constructor(
     gradle: Gradle,
     val beanName: ObjectName
 ) : KotlinBuildStatsService(), KotlinBuildStatsMXBean {
-    private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir)
+    private val forcePropertiesValidation = gradle.rootProject.properties[FORCE_VALUES_VALIDATION]?.toString()?.toBoolean() ?: false
+    private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir, forceValuesValidation = forcePropertiesValidation)
 
     private fun gradleBuildStartTime(gradle: Gradle): Long? {
         return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime

libraries/tools/kotlin-gradle-statistics/src/common/kotlin/org/jetbrains/kotlin/statistics/AnonymizerUtils.kt

@@ -40,3 +40,5 @@ internal fun sha256(s: String): String {
     val digest = md.digest(s.toByteArray())
     return digest.fold("", { str, it -> str + "%02x".format(it) })
 }
+
+class MetricValueValidationFailed(message: String) : RuntimeException(message)

libraries/tools/kotlin-gradle-statistics/src/common/kotlin/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt

@@ -19,7 +19,8 @@ class BuildSessionLogger(
     rootPath: File,
     private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
     private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
-    private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE
+    private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
+    private val forceValuesValidation: Boolean = false
 ) : IStatisticsValuesConsumer {
 
     companion object {
@@ -46,7 +47,7 @@ class BuildSessionLogger(
     private var buildSession: BuildSession? = null
     private var trackingFile: IRecordLogger? = null
 
-    private val metricsContainer = MetricsContainer()
+    private val metricsContainer = MetricsContainer(forceValuesValidation)
 
     @Synchronized
     fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {

libraries/tools/kotlin-gradle-statistics/src/common/kotlin/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt

@@ -5,7 +5,9 @@
 
 package org.jetbrains.kotlin.statistics.fileloggers
 
+import org.jetbrains.kotlin.statistics.MetricValueValidationFailed
 import org.jetbrains.kotlin.statistics.metrics.*
+import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.AllowedListAnonymizer.Companion.UNEXPECTED_VALUE
 import org.jetbrains.kotlin.statistics.sha256
 import java.io.*
 import java.nio.channels.Channels
@@ -14,7 +16,7 @@ import java.nio.file.Paths
 import java.nio.file.StandardOpenOption
 import java.util.*
 
-class MetricsContainer : IStatisticsValuesConsumer {
+class MetricsContainer(private val forceValuesValidation: Boolean = false) : IStatisticsValuesConsumer {
     data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
         override fun compareTo(other: MetricDescriptor): Int {
             val compareNames = name.compareTo(other.name)
@@ -129,7 +131,14 @@ class MetricsContainer : IStatisticsValuesConsumer {
         synchronized(metricsLock) {
             val metricContainer = stringMetrics[MetricDescriptor(metric.name, projectHash)] ?: metric.type.newMetricContainer()
                 .also { stringMetrics[MetricDescriptor(metric.name, projectHash)] = it }
-            metricContainer.addValue(metric.anonymization.anonymize(value), weight)
+
+            val anonymizedValue = metric.anonymization.anonymize(value)
+            if (forceValuesValidation && !metric.anonymization.anonymizeOnIdeSize()) {
+                if (anonymizedValue.contains(UNEXPECTED_VALUE) || !anonymizedValue.matches(Regex(metric.anonymization.validationRegexp()))) {
+                    throw MetricValueValidationFailed("Metric ${metric.name} has value [${value}], after anonymization [${anonymizedValue}]. Validation regex: ${metric.anonymization.validationRegexp()}.")
+                }
+            }
+            metricContainer.addValue(anonymizedValue, weight)
         }
         return true
     }

libraries/tools/kotlin-gradle-statistics/src/common/kotlin/org/jetbrains/kotlin/statistics/metrics/MetricPolicies.kt

@@ -85,18 +85,45 @@ enum class BooleanAnonymizationPolicy : ValueAnonymizer<Boolean> {
     }
 }
 
-enum class StringAnonymizationPolicy : ValueAnonymizer<String> {
+abstract class StringAnonymizationPolicy : ValueAnonymizer<String> {
-    SAFE {
+
+    abstract fun validationRegexp(): String
+
+    class AllowedListAnonymizer(val allowedValues: Collection<String>) : StringAnonymizationPolicy() {
+        companion object {
+            const val UNEXPECTED_VALUE = "UNEXPECTED-VALUE"
+        }
+
+        override fun validationRegexp(): String {
+            return "^((${UNEXPECTED_VALUE}|${allowedValues.joinToString("|")})${ConcatMetricContainer.SEPARATOR}?)+$"
+        }
+
+        override fun anonymize(t: String): String {
+            return if (t.matches(Regex(validationRegexp()))) {
+                t
+            } else {
+                t.split(ConcatMetricContainer.SEPARATOR).joinToString(ConcatMetricContainer.SEPARATOR) {
+                    if (allowedValues.contains(it))
+                        it
+                    else
+                        UNEXPECTED_VALUE
+                }
+            }
+        }
+    }
+
+    class RegexControlled(private val regex: String, private val anonymizeInIde: Boolean) : StringAnonymizationPolicy() {
+        override fun validationRegexp() = regex
+
         override fun anonymize(t: String) = t
-    },
+
-    ANONYMIZE_IN_IDE {
+        override fun anonymizeOnIdeSize() = anonymizeInIde
-        override fun anonymize(t: String) = t
+
-        override fun anonymizeOnIdeSize() = true
+    }
-    },
+
-    SHA_256 {
+    class ComponentVersionAnonymizer() : StringAnonymizationPolicy() {
-        override fun anonymize(t: String) = sha256(t)
+        override fun validationRegexp() = "(\\d+).(\\d+).(\\d+)-?(dev|snapshot|m\\d?|rc\\d?|beta\\d?)?"
-    },
+
-    COMPONENT_VERSION {
         override fun anonymize(t: String) = anonymizeComponentVersion(t)
     }
 }

libraries/tools/kotlin-gradle-statistics/src/common/kotlin/org/jetbrains/kotlin/statistics/metrics/StringMetrics.kt

@@ -12,43 +12,107 @@ import org.jetbrains.kotlin.statistics.metrics.StringOverridePolicy.*
 enum class StringMetrics(val type: StringOverridePolicy, val anonymization: StringAnonymizationPolicy, val perProject: Boolean = false) {
 
     // User environment
-    GRADLE_VERSION(OVERRIDE, COMPONENT_VERSION),
+    GRADLE_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    PROJECT_PATH(OVERRIDE, ANONYMIZE_IN_IDE),
+    PROJECT_PATH(OVERRIDE, RegexControlled("([0-9A-Fa-f]{40,64})|undefined", true)),
 
-    OS_TYPE(OVERRIDE, SAFE),
+    OS_TYPE(OVERRIDE, RegexControlled("(Windows|Windows |Windows Server |Mac|Linux|FreeBSD|Solaris|Other|Mac OS X)\\d*", false)),
 
-    //TODO could we collect only JB IDEs or, e.g. WsCode?
+    IDES_INSTALLED(CONCAT, AllowedListAnonymizer(listOf("AS", "OC", "CL", "IU", "IC", "WC"))),
-    IDES_INSTALLED(CONCAT, SAFE),
 
     // Build script
-    MPP_PLATFORMS(CONCAT, SAFE),
+    MPP_PLATFORMS(
-    JS_COMPILER_MODE(CONCAT, SAFE),
+        CONCAT, AllowedListAnonymizer(
+            listOf(
+                "common",
+                "metadata",
+                "jvm",
+                "js",
+                "arm32",
+                "arm64",
+                "mips32",
+                "mipsel32",
+                "x64",
+                "android",
+                "androidJvm",
+                "androidApp",
+                "androidNativeArm",
+                "androidNativeArm32",
+                "android_arm32",
+                "androidNativeArm64",
+                "android_arm64",
+                "androidNative",
+                "androidNativeX86",
+                "androidNativeX64",
+                "iosArm",
+                "iosArm32",
+                "ios_arm32",
+                "iosArm64",
+                "ios_arm64",
+                "ios_simulator_arm64",
+                "ios",
+                "ios_x64",
+                "iosSim",
+                "iosX64",
+                "watchos",
+                "watchosArm32",
+                "watchosArm64",
+                "watchosX86",
+                "tvos",
+                "tvosArm64",
+                "tvosX64",
+                "linux",
+                "linuxArm32Hfp",
+                "linux_arm32_hfp",
+                "linuxMips32",
+                "linux_mips32",
+                "linuxMipsel32",
+                "linux_mipsel32",
+                "linuxX64",
+                "linux_arm64",
+                "linux_x64",
+                "macos",
+                "osx",
+                "macosX64",
+                "macos_x64",
+                "macos_arm64",
+                "mingw",
+                "mingwX64",
+                "mingw_x64",
+                "mingwX86",
+                "mingw_X86",
+                "mingw_x86",
+                "wasm32",
+                "wasm"
+            )
+        )
+    ),
+    JS_COMPILER_MODE(CONCAT, AllowedListAnonymizer(listOf("ir", "legacy", "both", "UNKNOWN"))),
 
     // Component versions
-    LIBRARY_SPRING_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
+    LIBRARY_SPRING_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
-    LIBRARY_VAADIN_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
+    LIBRARY_VAADIN_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
-    LIBRARY_GWT_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
+    LIBRARY_GWT_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
-    LIBRARY_HIBERNATE_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
+    LIBRARY_HIBERNATE_VERSION(OVERRIDE_VERSION_IF_NOT_SET, ComponentVersionAnonymizer()),
 
-    KOTLIN_COMPILER_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_COMPILER_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    KOTLIN_STDLIB_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_STDLIB_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    KOTLIN_REFLECT_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_REFLECT_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    KOTLIN_COROUTINES_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_COROUTINES_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    KOTLIN_SERIALIZATION_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_SERIALIZATION_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
 
-    ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, COMPONENT_VERSION),
+    ANDROID_GRADLE_PLUGIN_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
 
     // Features
-    KOTLIN_LANGUAGE_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_LANGUAGE_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    KOTLIN_API_VERSION(OVERRIDE, COMPONENT_VERSION),
+    KOTLIN_API_VERSION(OVERRIDE, ComponentVersionAnonymizer()),
-    JS_GENERATE_EXECUTABLE_DEFAULT(CONCAT, SAFE),
+    JS_GENERATE_EXECUTABLE_DEFAULT(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
-    JS_TARGET_MODE(CONCAT, SAFE),
+    JS_TARGET_MODE(CONCAT, AllowedListAnonymizer(listOf("both", "browser", "nodejs", "none"))),
-    JS_OUTPUT_GRANULARITY(OVERRIDE, SAFE),
+    JS_OUTPUT_GRANULARITY(OVERRIDE, RegexControlled("(whole_program|per_module|per_file)", false)),
 
     // Compiler parameters
-    JVM_DEFAULTS(CONCAT, SAFE),
+    JVM_DEFAULTS(CONCAT, AllowedListAnonymizer(listOf("disable", "enable", "compatibility", "all", "all-compatibility"))),
-    USE_OLD_BACKEND(CONCAT, SAFE),
+    USE_OLD_BACKEND(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
-    USE_FIR(CONCAT, SAFE),
+    USE_FIR(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
 
-    JS_PROPERTY_LAZY_INITIALIZATION(CONCAT, SAFE),
+    JS_PROPERTY_LAZY_INITIALIZATION(CONCAT, AllowedListAnonymizer(listOf("true", "false"))),
 }

libraries/tools/kotlin-gradle-statistics/src/test/kotlin/MetricPolicyTest.kt

@@ -2,10 +2,14 @@
  * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
+import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
+import org.jetbrains.kotlin.statistics.metrics.ConcatMetricContainer
 import org.jetbrains.kotlin.statistics.metrics.NumberAnonymizationPolicy.RANDOM_10_PERCENT
-import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy.COMPONENT_VERSION
+import org.jetbrains.kotlin.statistics.metrics.StringAnonymizationPolicy
+import org.jetbrains.kotlin.statistics.metrics.StringMetrics
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertTrue
 
 
 class MetricPolicyTest {
@@ -28,25 +32,43 @@ class MetricPolicyTest {
 
     @Test
     fun versionAnonymization() {
-        assertEquals("0.0.0", COMPONENT_VERSION.anonymize("some.invalid.string"))
+        assertEquals("0.0.0", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("some.invalid.string"))
-        assertEquals("1.0.0", COMPONENT_VERSION.anonymize("1"))
+        assertEquals("1.0.0", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1"))
-        assertEquals("1.2.0", COMPONENT_VERSION.anonymize("1.2"))
+        assertEquals("1.2.0", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2"))
 
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3"))
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3"))
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3.4"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3.4"))
-        assertEquals("1.2.3-m", COMPONENT_VERSION.anonymize("1.2.3-M"))
+        assertEquals("1.2.3-m", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-M"))
-        assertEquals("1.2.3-m1", COMPONENT_VERSION.anonymize("1.2.3-M1"))
+        assertEquals("1.2.3-m1", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-M1"))
-        assertEquals("1.2.3-m2", COMPONENT_VERSION.anonymize("1.2.3.M2"))
+        assertEquals("1.2.3-m2", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3.M2"))
-        assertEquals("1.2.3-rc", COMPONENT_VERSION.anonymize("1.2.3-RC"))
+        assertEquals("1.2.3-rc", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-RC"))
-        assertEquals("1.2.3-rc5", COMPONENT_VERSION.anonymize("1.2.3-RC5"))
+        assertEquals("1.2.3-rc5", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-RC5"))
 
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3.unknown suffix"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3.unknown suffix"))
-        assertEquals("1.2.3", COMPONENT_VERSION.anonymize("1.2.3-unknown suffix"))
+        assertEquals("1.2.3", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.2.3-unknown suffix"))
 
-        assertEquals("123.234.345-dev", COMPONENT_VERSION.anonymize("123.234.345-dev-12345"))
+        assertEquals("123.234.345-dev", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("123.234.345-dev-12345"))
-        assertEquals("1.8.255-snapshot", COMPONENT_VERSION.anonymize("1.8.255-SNAPSHOT"))
+        assertEquals("1.8.255-snapshot", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.8.255-SNAPSHOT"))
 
-        assertEquals("1.8.255-beta", COMPONENT_VERSION.anonymize("1.8.255-beta"))
+        assertEquals("1.8.255-beta", StringAnonymizationPolicy.ComponentVersionAnonymizer().anonymize("1.8.255-beta"))
+    }
+
+    @Test
+    fun versionStringValidation() {
+        val separator = ConcatMetricContainer.SEPARATOR
+        val container = MetricsContainer()
+        fun whenAdded(newValue: String, expected: String) {
+            container.report(StringMetrics.MPP_PLATFORMS, newValue)
+            val currentValue = container.getMetric(StringMetrics.MPP_PLATFORMS)!!.toStringRepresentation()
+            assertEquals(expected, currentValue)
+            val regex = StringMetrics.MPP_PLATFORMS.anonymization.validationRegexp()
+            assertTrue(currentValue.matches(Regex(regex)), "'${currentValue}' should match '${regex}'")
+        }
+        whenAdded("js", "js")
+        whenAdded("common", "common${separator}js")
+        whenAdded("jvm", "common${separator}js${separator}jvm")
+        whenAdded("AAA", "UNEXPECTED-VALUE${separator}common${separator}js${separator}jvm")
+        whenAdded("BBB", "UNEXPECTED-VALUE${separator}common${separator}js${separator}jvm")
     }
 }