azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added FUS extensions and new metrics for reporting statistics from gradle

Browse Source
This commit is contained in:
Andrey Uskov
2020-01-17 09:14:59 +03:00
parent bac0309a9b
commit beb3165839
28 changed files with 440 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
idea/idea-core/src/org/jetbrains/kotlin/idea/statistics/FUSEventGroups.kt
+16 -1
View File
@@ -21,11 +21,26 @@ enum class FUSEventGroups(groupIdSuffix: String, val events: Set<String> = setOf
Debug("ide.debugger"),
J2K("ide.j2k"),
Editor("ide.editor"),
Settings("ide.settings");
Settings("ide.settings"),
GradlePerformance("gradle.performance");
val GROUP_ID: String = "kotlin.$groupIdSuffix"
}
@Suppress("EnumEntryName")
enum class GradleStatisticsEvents {
Environment,
Kapt,
CompilerPlugins,
MPP,
Libraries,
GradleConfiguration,
ComponentVersions,
KotlinFeatures,
GradlePerformance,
UseScenarios
}
val gradleTargetEvents = setOf(
"kotlin-android",
"kotlin-platform-common",
idea/idea-gradle/build.gradle.kts
+2
View File
@@ -26,6 +26,8 @@ dependencies {
compileOnly(intellijPluginDep("junit"))
compileOnly(intellijPluginDep("testng"))
compileOnly(project(":kotlin-gradle-statistics"))
Platform[192].orHigher {
compileOnly(intellijPluginDep("java"))
testCompileOnly(intellijPluginDep("java"))
idea/idea-gradle/src/org/jetbrains/kotlin/idea/configuration/KotlinGradleProjectResolverExtension.kt
+2
View File
@@ -34,6 +34,7 @@ import org.jetbrains.kotlin.idea.util.NotNullableCopyableDataNodeUserDataPropert
import org.jetbrains.kotlin.idea.util.PsiPrecedences
import org.jetbrains.kotlin.idea.statistics.FUSEventGroups
import org.jetbrains.kotlin.idea.statistics.KotlinFUSLogger
import org.jetbrains.kotlin.idea.statistics.KotlinGradleFUSLogger
import org.jetbrains.plugins.gradle.model.ExternalProjectDependency
import org.jetbrains.plugins.gradle.model.ExternalSourceSet
import org.jetbrains.plugins.gradle.model.FileCollectionDependency
@@ -294,6 +295,7 @@ class KotlinGradleProjectResolverExtension : AbstractProjectResolverExtension()
nextResolver.populateModuleContentRoots(gradleModule, ideModule)
val moduleNamePrefix = GradleProjectResolverUtil.getModuleId(resolverCtx, gradleModule)
resolverCtx.getExtraProject(gradleModule, KotlinGradleModel::class.java)?.let { gradleModel ->
KotlinGradleFUSLogger.populateGradleUserDir(gradleModel.gradleUserHome)
ideModule.pureKotlinSourceFolders =
gradleModel.kotlinTaskProperties.flatMap { it.value.pureKotlinSourceFolders ?: emptyList() }.map { it.absolutePath }
val gradleSourceSets = ideModule.children.filter { it.data is GradleSourceSetData } as Collection<DataNode<GradleSourceSetData>>
idea/idea-gradle/src/org/jetbrains/kotlin/idea/configuration/KotlinGradleSourceSetDataService.kt
+6
View File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.idea.configuration
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.externalSystem.model.DataNode
import com.intellij.openapi.externalSystem.model.ProjectKeys
@@ -54,6 +55,7 @@ import org.jetbrains.kotlin.idea.inspections.gradle.findKotlinPluginVersion
import org.jetbrains.kotlin.idea.inspections.gradle.getResolvedVersionByModuleData
import org.jetbrains.kotlin.idea.platform.tooling
import org.jetbrains.kotlin.idea.roots.migrateNonJvmSourceFolders
import org.jetbrains.kotlin.idea.statistics.KotlinGradleFUSLogger
import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
import org.jetbrains.kotlin.platform.IdePlatformKind
import org.jetbrains.kotlin.platform.impl.isCommon
@@ -153,6 +155,10 @@ class KotlinGradleProjectDataService : AbstractProjectDataService<ModuleData, Vo
val codeStyleStr = GradlePropertiesFileFacade.forProject(project).readProperty(KOTLIN_CODE_STYLE_GRADLE_SETTING)
ProjectCodeStyleImporter.apply(project, codeStyleStr)
ApplicationManager.getApplication().executeOnPooledThread {
KotlinGradleFUSLogger.reportStatistics()
}
}
}
idea/idea-gradle/src/org/jetbrains/kotlin/idea/statistics/KotlinGradleFUSLogger.kt
+219
View File
@@ -0,0 +1,219 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.idea.statistics
import com.intellij.ide.util.PropertiesComponent
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.project.DumbAware
import com.intellij.openapi.project.Project
import com.intellij.openapi.startup.StartupActivity
import com.intellij.util.concurrency.AppExecutorUtil
import org.jetbrains.kotlin.statistics.BuildSessionLogger
import java.io.File
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics
class KotlinGradleFUSLogger : StartupActivity, DumbAware, Runnable {
override fun runActivity(project: Project) {
AppExecutorUtil.getAppScheduledExecutorService()
.scheduleWithFixedDelay(this, EXECUTION_DELAY_MIN, EXECUTION_DELAY_MIN, TimeUnit.MINUTES)
}
override fun run() {
reportStatistics()
}
companion object {
/**
* Maximum amount of directories which were reported as gradle user dirs
* These directories should be monitored for reported gradle statistics.
*/
const val MAXIMUM_USER_DIRS = 10
/**
* Delay between sequential checks of gradle statistics
*/
const val EXECUTION_DELAY_MIN = 60L
/**
* Property name used for persisting gradle user dirs
*/
private const val GRADLE_USER_DIRS_PROPERTY_NAME = "kotlin-gradle-user-dirs"
private val isRunning = AtomicBoolean(false)
private fun MetricsContainer.log(event: GradleStatisticsEvents, vararg metrics: Any) {
val data = HashMap<String, String>()
fun putIfNotNull(key: String, value: String?) {
if (value != null) {
data[key.toLowerCase()] = value
}
}
for (metric in metrics) {
when (metric) {
is BooleanMetrics -> putIfNotNull(metric.name, this.getMetric(metric)?.toStringRepresentation())
is StringMetrics -> putIfNotNull(metric.name, this.getMetric(metric)?.toStringRepresentation())
is NumericalMetrics -> putIfNotNull(metric.name, this.getMetric(metric)?.toStringRepresentation())
is Pair<*, *> -> putIfNotNull(metric.first.toString(), metric.second.toString())
}
}
if (data.size > 0) {
KotlinFUSLogger.log(FUSEventGroups.GradlePerformance, event.name, data)
}
}
private fun processMetricsContainer(container: MetricsContainer, previous: MetricsContainer?) {
container.log(
GradleStatisticsEvents.Environment,
NumericalMetrics.CPU_NUMBER_OF_CORES,
StringMetrics.GRADLE_VERSION,
NumericalMetrics.ARTIFACTS_DOWNLOAD_SPEED,
StringMetrics.IDES_INSTALLED,
BooleanMetrics.EXECUTED_FROM_IDEA
)
container.log(
GradleStatisticsEvents.Kapt,
BooleanMetrics.ENABLED_KAPT,
BooleanMetrics.ENABLED_DAGGER,
BooleanMetrics.ENABLED_DATABINDING
)
container.log(
GradleStatisticsEvents.CompilerPlugins,
BooleanMetrics.ENABLED_COMPILER_PLUGIN_ALL_OPEN,
BooleanMetrics.ENABLED_COMPILER_PLUGIN_NO_ARG,
BooleanMetrics.ENABLED_COMPILER_PLUGIN_JPA_SUPPORT,
BooleanMetrics.ENABLED_COMPILER_PLUGIN_SAM_WITH_RECEIVER
)
container.log(
GradleStatisticsEvents.MPP,
StringMetrics.MPP_PLATFORMS,
BooleanMetrics.ENABLED_HMPP
)
container.log(
GradleStatisticsEvents.Libraries,
StringMetrics.LIBRARY_SPRING_VERSION,
StringMetrics.LIBRARY_VAADIN_VERSION,
StringMetrics.LIBRARY_GWT_VERSION,
StringMetrics.LIBRARY_HIBERNATE_VERSION
)
container.log(
GradleStatisticsEvents.GradleConfiguration,
NumericalMetrics.GRADLE_DAEMON_HEAP_SIZE,
NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON,
NumericalMetrics.CONFIGURATION_API_COUNT,
NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT,
NumericalMetrics.CONFIGURATION_COMPILE_COUNT,
NumericalMetrics.CONFIGURATION_RUNTIME_COUNT,
NumericalMetrics.GRADLE_NUMBER_OF_TASKS,
NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS,
NumericalMetrics.GRADLE_NUMBER_OF_INCREMENTAL_TASKS
)
container.log(
GradleStatisticsEvents.ComponentVersions,
StringMetrics.KOTLIN_COMPILER_VERSION,
StringMetrics.KOTLIN_STDLIB_VERSION,
StringMetrics.KOTLIN_REFLECT_VERSION,
StringMetrics.KOTLIN_COROUTINES_VERSION,
StringMetrics.KOTLIN_SERIALIZATION_VERSION,
StringMetrics.ANDROID_GRADLE_PLUGIN_VERSION
)
container.log(
GradleStatisticsEvents.KotlinFeatures,
StringMetrics.KOTLIN_LANGUAGE_VERSION,
StringMetrics.KOTLIN_API_VERSION,
BooleanMetrics.BUILD_SRC_EXISTS,
NumericalMetrics.BUILD_SRC_COUNT,
BooleanMetrics.GRADLE_BUILD_CACHE_USED,
BooleanMetrics.GRADLE_WORKER_API_USED,
BooleanMetrics.KOTLIN_OFFICIAL_CODESTYLE,
BooleanMetrics.KOTLIN_PROGRESSIVE_MODE,
BooleanMetrics.KOTLIN_KTS_USED
)
container.log(
GradleStatisticsEvents.GradlePerformance,
NumericalMetrics.GRADLE_BUILD_DURATION,
NumericalMetrics.GRADLE_EXECUTION_DURATION,
NumericalMetrics.NUMBER_OF_SUBPROJECTS,
NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD
)
val finishTime = container.getMetric(NumericalMetrics.BUILD_FINISH_TIME)?.getValue()
val prevFinishTime = previous?.getMetric(NumericalMetrics.BUILD_FINISH_TIME)?.getValue()
val betweenBuilds = if (finishTime != null && prevFinishTime != null) finishTime - prevFinishTime else null
container.log(
GradleStatisticsEvents.UseScenarios,
Pair("time_between_builds", betweenBuilds),
BooleanMetrics.DEBUGGER_ENABLED,
BooleanMetrics.COMPILATION_STARTED,
BooleanMetrics.TESTS_EXECUTED,
BooleanMetrics.MAVEN_PUBLISH_EXECUTED,
BooleanMetrics.BUILD_FAILED
)
}
fun reportStatistics() {
if (isRunning.compareAndSet(false, true)) {
try {
for (gradleUserHome in gradleUserDirs) {
BuildSessionLogger.listProfileFiles(File(gradleUserHome, STATISTICS_FOLDER_NAME))?.forEach { statisticFile ->
try {
var previousEvent: MetricsContainer? = null
MetricsContainer.readFromFile(statisticFile) { metricContainer ->
processMetricsContainer(metricContainer, previousEvent)
previousEvent = metricContainer
}
} catch (e: Exception) {
Logger.getInstance(KotlinFUSLogger::class.java)
.info("Failed to process file ${statisticFile.absolutePath}: ${e.message}", e)
} finally {
if (!statisticFile.delete()) {
Logger.getInstance(KotlinFUSLogger::class.java)
.warn("[FUS] Failed to delete file ${statisticFile.absolutePath}")
}
}
}
}
} finally {
isRunning.set(false)
}
}
}
private var gradleUserDirs: Array<String>
set(value) = PropertiesComponent.getInstance().setValues(
GRADLE_USER_DIRS_PROPERTY_NAME, value
)
get() = PropertiesComponent.getInstance().getValues(GRADLE_USER_DIRS_PROPERTY_NAME) ?: emptyArray()
fun populateGradleUserDir(path: String) {
val currentState = gradleUserDirs
if (path in currentState) return
val result = ArrayList<String>()
result.add(path)
result.addAll(currentState)
gradleUserDirs =
result.filter { filePath -> File(filePath).exists() }.take(MAXIMUM_USER_DIRS).toTypedArray()
}
}
}
idea/kotlin-gradle-tooling/src/KotlinGradleModelBuilder.kt
+5 -2
View File
@@ -54,6 +54,7 @@ interface KotlinGradleModel : Serializable {
val implements: List<String>
val kotlinTarget: String?
val kotlinTaskProperties: KotlinTaskPropertiesBySourceSet
val gradleUserHome: String
}
data class KotlinGradleModelImpl(
@@ -63,7 +64,8 @@ data class KotlinGradleModelImpl(
override val platformPluginId: String?,
override val implements: List<String>,
override val kotlinTarget: String? = null,
override val kotlinTaskProperties: KotlinTaskPropertiesBySourceSet
override val kotlinTaskProperties: KotlinTaskPropertiesBySourceSet,
override val gradleUserHome: String
) : KotlinGradleModel
abstract class AbstractKotlinGradleModelBuilder : ModelBuilderService {
@@ -189,7 +191,8 @@ class KotlinGradleModelBuilder : AbstractKotlinGradleModelBuilder() {
platform,
implementedProjects.map { it.pathOrName() },
platform ?: kotlinPluginId,
extraProperties
extraProperties,
project.gradle.gradleUserHomeDir.absolutePath
)
}
}
idea/resources/META-INF/plugin-common.xml
+2
View File
@@ -880,6 +880,8 @@
<postStartupActivity implementation="org.jetbrains.kotlin.idea.framework.CreateKotlinSdkActivity"/>
<postStartupActivity implementation="org.jetbrains.kotlin.idea.statistics.KotlinGradleFUSLogger"/>
<moduleConfigurationEditorProvider implementation="org.jetbrains.kotlin.idea.roots.ui.NonJvmKotlinModuleEditorsProvider"/>
<projectStructure.sourceRootEditHandler implementation="org.jetbrains.kotlin.idea.roots.ui.KotlinModuleSourceRootEditHandler$Source"/>
idea/resources/META-INF/plugin.xml
+1
View File
@@ -78,6 +78,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio.
<statistics.counterUsagesCollector groupId="kotlin.ide.debugger" version="2"/>
<statistics.counterUsagesCollector groupId="kotlin.ide.j2k" version="1"/>
<statistics.counterUsagesCollector groupId="kotlin.ide.editor" version="1"/>
<statistics.counterUsagesCollector groupId="kotlin.gradle.performance" version="1"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.statistics.IDESettingsFUSCollector"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.formatter.KotlinFormatterUsageCollector"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.statistics.ProjectConfigurationCollector"/>
idea/resources/META-INF/plugin.xml.191
+1
View File
@@ -78,6 +78,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio.
<statistics.counterUsagesCollector groupId="kotlin.ide.debugger" version="2"/>
<statistics.counterUsagesCollector groupId="kotlin.ide.j2k" version="1"/>
<statistics.counterUsagesCollector groupId="kotlin.ide.editor" version="1"/>
<statistics.counterUsagesCollector groupId="kotlin.gradle.performance" version="1"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.statistics.IDESettingsFUSCollector"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.formatter.KotlinFormatterUsageCollector"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.statistics.ProjectConfigurationCollector"/>
idea/resources/META-INF/plugin.xml.192
+1
View File
@@ -78,6 +78,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio.
<statistics.counterUsagesCollector groupId="kotlin.ide.debugger" version="2"/>
<statistics.counterUsagesCollector groupId="kotlin.ide.j2k" version="1"/>
<statistics.counterUsagesCollector groupId="kotlin.ide.editor" version="1"/>
<statistics.counterUsagesCollector groupId="kotlin.gradle.performance" version="1"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.statistics.IDESettingsFUSCollector"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.formatter.KotlinFormatterUsageCollector"/>
<statistics.projectUsagesCollector implementation="org.jetbrains.kotlin.idea.statistics.ProjectConfigurationCollector"/>
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMultiplatformPlugin.kt
+4
View File
@@ -28,12 +28,14 @@ import org.jetbrains.kotlin.gradle.dsl.multiplatformExtension
import org.jetbrains.kotlin.gradle.plugin.*
import org.jetbrains.kotlin.gradle.plugin.sources.DefaultLanguageSettingsBuilder
import org.jetbrains.kotlin.gradle.plugin.sources.checkSourceSetVisibilityRequirements
import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
import org.jetbrains.kotlin.gradle.scripting.internal.ScriptingGradleSubplugin
import org.jetbrains.kotlin.gradle.targets.metadata.isKotlinGranularMetadataEnabled
import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.target.KonanTarget.*
import org.jetbrains.kotlin.konan.target.presetName
import org.jetbrains.kotlin.statistics.metrics.StringMetrics
class KotlinMultiplatformPlugin(
private val kotlinPluginVersion: String,
@@ -261,6 +263,8 @@ class KotlinMultiplatformPlugin(
target.compilations.findByName(KotlinCompilation.TEST_COMPILATION_NAME)?.let { testCompilation ->
sourceSets.findByName(testCompilation.defaultSourceSetName)?.dependsOn(test)
}
KotlinBuildStatsService.getInstance()?.report(StringMetrics.MPP_PLATFORMS, target.targetName)
}
UnusedSourceSetsChecker.checkSourceSets(project)
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/DaemonReusageCounter.kt.kt → libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/DaemonReusageCounter.kt
-2
View File
@@ -12,11 +12,9 @@ import javax.management.ObjectName
import javax.management.StandardMBean
interface IDaemonReuseCounterMXBean {
fun getOrdinal(): Long
fun incrementAndGetOrdinal(): Long
}
/**
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/KotlinBuildStatsService.kt
+59 -38
View File
@@ -7,9 +7,9 @@ package org.jetbrains.kotlin.gradle.plugin.statistics
import org.gradle.BuildAdapter
import org.gradle.BuildResult
import org.gradle.api.artifacts.DependencySet
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging
import org.gradle.initialization.BuildCompletionListener
import org.gradle.initialization.BuildRequestMetaData
import org.gradle.invocation.DefaultGradle
import org.jetbrains.kotlin.statistics.BuildSessionLogger
@@ -20,6 +20,7 @@ import java.lang.management.ManagementFactory
import javax.management.MBeanServer
import javax.management.ObjectName
import javax.management.StandardMBean
import kotlin.system.measureTimeMillis
/**
* Interface for populating statistics collection method via JXM interface
@@ -27,21 +28,16 @@ import javax.management.StandardMBean
* of Kotlin Plugin and other classloaders
*/
interface KotlinBuildStatsMXBean {
fun reportBoolean(name: String, value: Boolean, subprojectName: String?)
fun reportNumber(name: String, value: Long, subprojectName: String?)
fun reportString(name: String, value: String, subprojectName: String?)
}
internal abstract class KotlinBuildStatsService internal constructor() : BuildAdapter(), IStatisticsValuesConsumer,
BuildCompletionListener {
internal abstract class KotlinBuildStatsService internal constructor() : BuildAdapter(), IStatisticsValuesConsumer {
companion object {
// Do not rename this bean otherwise compatibility with the older Kotlin Gradle Plugins would be lost
const val JMX_BEAN_NAME = "org.jetbrains.kotlin.gradle.plugin.statistics:type=StatsService"
@@ -54,7 +50,6 @@ internal abstract class KotlinBuildStatsService internal constructor() : BuildAd
// "emergency file" collecting statistics is disabled it the file exists
const val DISABLE_STATISTICS_FILE_NAME = "${STATISTICS_FOLDER_NAME}/.disable"
/**
* Method for getting IStatisticsValuesConsumer for reporting some statistics
* Could be invoked during any build phase after applying first Kotlin plugin and
@@ -145,16 +140,11 @@ internal abstract class KotlinBuildStatsService internal constructor() : BuildAd
}
}
}
}
internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) :
KotlinBuildStatsService() {
override fun buildFinished(result: BuildResult) {
}
private fun callJmx(method: String, type: String, metricName: String, value: Any, subprojectName: String?) {
mbs.invoke(
beanName,
@@ -182,50 +172,89 @@ internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private
}
}
override fun completed() {
override fun buildFinished(result: BuildResult) {
instance = null
}
}
internal class DefaultKotlinBuildStatsService internal constructor(
gradle: Gradle,
val beanName: ObjectName
) : KotlinBuildStatsService(), KotlinBuildStatsMXBean {
private val sessionLogger = BuildSessionLogger(gradle.gradleUserHomeDir)
private fun gradleBuildStartTime(gradle: Gradle): Long? {
return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime
}
private fun reportLibrariesVersions(dependencies: DependencySet?) {
dependencies?.forEach { dependency ->
when {
dependency.group?.startsWith("org.springframework") ?: false -> sessionLogger.report(
StringMetrics.LIBRARY_SPRING_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("com.vaadin") ?: false -> sessionLogger.report(
StringMetrics.LIBRARY_VAADIN_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("com.google.gwt") ?: false -> sessionLogger.report(
StringMetrics.LIBRARY_GWT_VERSION,
dependency.version ?: "0.0.0"
)
dependency.group?.startsWith("org.hibernate") ?: false -> sessionLogger.report(
StringMetrics.LIBRARY_HIBERNATE_VERSION,
dependency.version ?: "0.0.0"
)
}
}
}
private fun reportGlobalMetrics(gradle: Gradle) {
System.getProperty("os.name")?.also {
sessionLogger.report(StringMetrics.OS_TYPE, gradle.gradleVersion)
sessionLogger.report(StringMetrics.OS_TYPE, System.getProperty("os.name"))
}
sessionLogger.report(NumericalMetrics.CPU_NUMBER_OF_CORES, Runtime.getRuntime().availableProcessors().toLong())
sessionLogger.report(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
sessionLogger.report(BooleanMetrics.EXECUTED_FROM_IDEA, System.getProperty("idea.active") != null)
val statisticOverhead = measureTimeMillis {
gradle.allprojects { project ->
for (configuration in project.configurations) {
val configurationName = configuration.name
val dependencies = configuration.dependencies
gradle.allprojects { project ->
for (configuration in project.configurations) {
val configurationName = configuration.name
val dependencies = configuration.dependencies
when (configurationName) {
"kapt" -> {
sessionLogger.report(BooleanMetrics.ENABLED_KAPT, true)
dependencies.forEach { dependency ->
when (dependency.group) {
"com.google.dagger" -> sessionLogger.report(BooleanMetrics.ENABLED_DAGGER, true)
"com.android.databinding" -> sessionLogger.report(BooleanMetrics.ENABLED_DATABINDING, true)
when (configurationName) {
"kapt" -> {
sessionLogger.report(BooleanMetrics.ENABLED_KAPT, true)
dependencies?.forEach { dependency ->
when (dependency.group) {
"com.google.dagger" -> sessionLogger.report(BooleanMetrics.ENABLED_DAGGER, true)
"com.android.databinding" -> sessionLogger.report(BooleanMetrics.ENABLED_DATABINDING, true)
}
}
}
"api" -> {
sessionLogger.report(NumericalMetrics.CONFIGURATION_API_COUNT, 1)
reportLibrariesVersions(dependencies)
}
"implementation" -> {
sessionLogger.report(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1)
reportLibrariesVersions(dependencies)
}
"compile" -> {
sessionLogger.report(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1)
reportLibrariesVersions(dependencies)
}
"runtime" -> {
sessionLogger.report(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1)
reportLibrariesVersions(dependencies)
}
}
}
}
}
sessionLogger.report(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
}
override fun projectsEvaluated(gradle: Gradle) {
@@ -243,16 +272,8 @@ internal class DefaultKotlinBuildStatsService internal constructor(
@Synchronized
override fun buildFinished(result: BuildResult) {
runSafe("${DefaultKotlinBuildStatsService::class.java}.buildFinished") {
sessionLogger.finishBuildSession(result.action, result.failure)
}
}
@Synchronized
override fun completed() {
runSafe("${DefaultKotlinBuildStatsService::class.java}.completed") {
try {
sessionLogger.unlockJournalFile()
sessionLogger.finishBuildSession(result.action, result.failure)
} finally {
val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer()
if (mbs.isRegistered(beanName)) {
libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/BuildStatServiceTest.kt
-2
View File
@@ -25,7 +25,5 @@ class BuildStatServiceTest {
jmxService.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3")
jmxService.report(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 10)
jmxService.report(BooleanMetrics.ENABLED_DATABINDING, true)
jmxService.completed()
}
}
libraries/tools/kotlin-gradle-statistics/build.gradle.kts
+1 -3
View File
@@ -7,7 +7,7 @@ plugins {
dependencies {
compileOnly(kotlinStdlib())
testCompile(project(":kotlin-test::kotlin-test-junit"))
testCompile(project(":kotlin-test:kotlin-test-junit"))
testCompile("junit:junit:4.12")
}
@@ -19,5 +19,3 @@ sourceSets {
projectTest {
workingDir = rootDir
}
runtimeJar(rewriteDefaultJarDepsToShadedCompiler())
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/AnonymizerUtils.kt
+3 -2
View File
@@ -21,8 +21,9 @@ internal val salt: String by lazy {
fun anonymizeComponentVersion(version: String): String {
return version.replace('-', '.')
.split(".")
.filterIndexed { i, _ -> i < 3 }
.map { s -> s.toIntOrNull() ?: "?" }
.plus(listOf("0", "0")) // pad with zeros
.take(3)
.map { s -> s.toIntOrNull() ?: "0" }
.joinToString(".")
}
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSession.kt
+1 -3
View File
@@ -6,7 +6,5 @@
package org.jetbrains.kotlin.statistics
class BuildSession(val buildStartedTime: Long?) {
val projectEvaluatedTime = System.currentTimeMillis()
}
}
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt
+48 -23
View File
@@ -16,17 +16,23 @@ import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
class BuildSessionLogger(
private val rootPath: File,
rootPath: File,
private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE
) : IStatisticsValuesConsumer {
companion object {
const val STATISTICS_FOLDER_NAME = "kotlin-profile"
const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}.profile"
internal const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}.profile"
private const val DEFAULT_MAX_PROFILE_FILES = 1_000
private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
private const val DEFAULT_MAX_FILE_AGE = 30 * 24 * 3600 * 1000L //30 days
fun listProfileFiles(statisticsFolder: File): List<File>? {
return statisticsFolder.listFiles()?.filterTo(ArrayList()) { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted()
}
}
private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS'.profile'")
@@ -53,29 +59,44 @@ class BuildSessionLogger(
@Synchronized
private fun closeTrackingFile() {
metricsContainer.flush(trackingFile)
trackingFile?.close()
trackingFile = null
trackingFile?.let {
metricsContainer.flush(it)
it.close()
trackingFile = null
}
}
/**
* Initializes a new tracking file
* The following contracts are implemented:
* - number of tracking files should not be more than maxProfileFiles (the earlier file created the earlier deleted)
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
* - files are ordered on the basis of name (creation timestamp)
* - if the last file has size less then maxFileSize, the next record will be append to it (new file created otherwise)
* -
*/
@Synchronized
private fun initTrackingFile() {
closeTrackingFile()
// Get list of existing files. Try to create folder if possible, return from function if failed to create folder
val fileCandidates =
statisticsFolder.listFiles()?.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.toMutableList()
?: if (statisticsFolder.mkdirs()) emptyList<File>() else return
val fileCandidates = listProfileFiles(statisticsFolder) ?: if (statisticsFolder.mkdirs()) emptyList<File>() else return
for (i in 0 until fileCandidates.size - maxProfileFiles) {
val file2delete = fileCandidates[i]
if (file2delete.isFile) {
file2delete.delete()
for ((index, file) in fileCandidates.withIndex()) {
val toDelete = if (index < fileCandidates.size - maxProfileFiles)
true
else {
val lastModified = file.lastModified()
(lastModified > 0) && (System.currentTimeMillis() - maxFileAge > lastModified)
}
if (toDelete) {
file.delete()
}
}
// emergency check. What if a lot of files are locked due to some reason
if (statisticsFolder.listFiles()?.size ?: 0 > maxProfileFiles * 2) {
if (listProfileFiles(statisticsFolder)?.size ?: 0 > maxProfileFiles * 2) {
trackingFile = NullRecordLogger()
return
}
@@ -99,20 +120,24 @@ class BuildSessionLogger(
@Synchronized
fun finishBuildSession(action: String?, failure: Throwable?) {
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
// could be affected by system clock correction
val finishTime = System.currentTimeMillis()
buildSession?.also {
if (it.buildStartedTime != null) {
report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
try {
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
// could be affected by system clock correction
val finishTime = System.currentTimeMillis()
buildSession?.also {
if (it.buildStartedTime != null) {
report(NumericalMetrics.GRADLE_BUILD_DURATION, finishTime - it.buildStartedTime)
}
report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
}
report(NumericalMetrics.GRADLE_EXECUTION_DURATION, finishTime - it.projectEvaluatedTime)
buildSession = null
} finally {
unlockJournalFile()
}
buildSession = null
}
@Synchronized
fun unlockJournalFile() {
private fun unlockJournalFile() {
closeTrackingFile()
}
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/FileRecordLogger.kt
-1
View File
@@ -15,7 +15,6 @@ import java.nio.file.Paths
import java.nio.file.StandardOpenOption
class FileRecordLogger(file: File) : IRecordLogger {
private val channel: FileChannel =
FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
?: throw IOException("Could not open file $file")
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/IRecordLogger.kt
+1 -3
View File
@@ -8,7 +8,5 @@ package org.jetbrains.kotlin.statistics.fileloggers
import java.io.Closeable
interface IRecordLogger : Closeable {
fun append(s: String)
}
}
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt
+5 -6
View File
@@ -15,11 +15,11 @@ import java.nio.file.StandardOpenOption
import java.util.*
class MetricsContainer : IStatisticsValuesConsumer {
data class MetricDescriptor(val name: String, val projectHash: String?) : Comparable<MetricDescriptor> {
override fun compareTo(other: MetricDescriptor): Int {
val compareNames = name.compareTo(other.name)
return when {
name.compareTo(other.name) != 0 -> name.compareTo(other.name)
compareNames != 0 -> compareNames
projectHash == other.projectHash -> 0
else -> (projectHash ?: "").compareTo(other.projectHash ?: "")
}
@@ -33,16 +33,15 @@ class MetricsContainer : IStatisticsValuesConsumer {
private val stringMetrics = TreeMap<MetricDescriptor, IMetricContainer<String>>()
companion object {
private const val BUILD_SESSION_SEPARATOR = "BUILD FINISHED"
val ENCODING = Charsets.UTF_8
private val stringMetricsMap = StringMetrics.values().map { m -> m.name to m }.toMap()
private val stringMetricsMap = StringMetrics.values().associateBy(StringMetrics::name)
private val booleanMetricsMap = BooleanMetrics.values().map { m -> m.name to m }.toMap()
private val booleanMetricsMap = BooleanMetrics.values().associateBy(BooleanMetrics::name)
private val numericalMetricsMap = NumericalMetrics.values().map { m -> m.name to m }.toMap()
private val numericalMetricsMap = NumericalMetrics.values().associateBy(NumericalMetrics::name)
fun readFromFile(file: File, consumer: (MetricsContainer) -> Unit) {
val channel = FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.READ)
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricContainers.kt
+13 -6
View File
@@ -13,7 +13,6 @@ interface IMetricContainer<T> {
fun toStringRepresentation(): String
fun getValue(): T?
}
interface IMetricContainerFactory<T> {
@@ -23,7 +22,6 @@ interface IMetricContainerFactory<T> {
}
open class OverrideMetricContainer<T>() : IMetricContainer<T> {
internal var myValue: T? = null
override fun addValue(t: T) {
@@ -41,8 +39,19 @@ open class OverrideMetricContainer<T>() : IMetricContainer<T> {
override fun getValue() = myValue
}
class SumMetricContainer() : OverrideMetricContainer<Long>() {
class OverrideVersionMetricContainer() : OverrideMetricContainer<String>() {
constructor(v: String) : this() {
myValue = v
}
override fun addValue(t: String) {
if (myValue == null || myValue == "0.0.0") {
myValue = t
}
}
}
class SumMetricContainer() : OverrideMetricContainer<Long>() {
constructor(v: Long) : this() {
myValue = v
}
@@ -85,7 +94,6 @@ class OrMetricContainer() : OverrideMetricContainer<Boolean>() {
}
class ConcatMetricContainer() : IMetricContainer<String> {
private val myValues = TreeSet<String>()
companion object {
@@ -101,9 +109,8 @@ class ConcatMetricContainer() : IMetricContainer<String> {
}
override fun toStringRepresentation(): String {
return myValues.joinToString(SEPARATOR)
return myValues.sorted().joinToString(SEPARATOR)
}
override fun getValue() = toStringRepresentation()
}
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/MetricPolicies.kt
+5
View File
@@ -16,6 +16,11 @@ enum class StringOverridePolicy: IMetricContainerFactory<String> {
override fun fromStringRepresentation(state: String): IMetricContainer<String>? = OverrideMetricContainer(state)
},
OVERRIDE_VERSION_IF_NOT_SET {
override fun newMetricContainer(): IMetricContainer<String> = OverrideVersionMetricContainer()
override fun fromStringRepresentation(state: String): IMetricContainer<String>? = OverrideVersionMetricContainer(state)
},
CONCAT {
override fun newMetricContainer(): IMetricContainer<String> = ConcatMetricContainer()
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/NumericalMetrics.kt
+2
View File
@@ -47,6 +47,8 @@ enum class NumericalMetrics(val type: NumberOverridePolicy, val anonymization: N
NUMBER_OF_SUBPROJECTS(OVERRIDE, RANDOM_10_PERCENT),
STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD(SUM, RANDOM_10_PERCENT),
// User scenarios
// this value is not reported, only time intervals from the previous build are used
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StatisticsValues.kt
+1 -3
View File
@@ -18,11 +18,9 @@ interface AdditiveStatisticsValue<T> : ReportStatisticsValue<T> {
}
interface IStatisticsValuesConsumer {
fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String? = null)
fun report(metric: NumericalMetrics, value: Long, subprojectName: String? = null)
fun report(metric: StringMetrics, value: String, subprojectName: String? = null)
}
}
libraries/tools/kotlin-gradle-statistics/src/org/jetbrains/kotlin/statistics/metrics/StringMetrics.kt
+4 -9
View File
@@ -23,10 +23,10 @@ enum class StringMetrics(val type: StringOverridePolicy, val anonymization: Stri
MPP_PLATFORMS(CONCAT, SAFE),
// Component versions
LIBRARY_SPRING_VERSION(OVERRIDE, COMPONENT_VERSION),
LIBRARY_VAADIN_VERSION(OVERRIDE, COMPONENT_VERSION),
LIBRARY_GWT_VERSION(OVERRIDE, COMPONENT_VERSION),
LIBRARY_HYBERNATE_VERSION(OVERRIDE, COMPONENT_VERSION),
LIBRARY_SPRING_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
LIBRARY_VAADIN_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
LIBRARY_GWT_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
LIBRARY_HIBERNATE_VERSION(OVERRIDE_VERSION_IF_NOT_SET, COMPONENT_VERSION),
KOTLIN_COMPILER_VERSION(OVERRIDE, COMPONENT_VERSION),
KOTLIN_STDLIB_VERSION(OVERRIDE, COMPONENT_VERSION),
@@ -39,9 +39,4 @@ enum class StringMetrics(val type: StringOverridePolicy, val anonymization: Stri
// Features
KOTLIN_LANGUAGE_VERSION(OVERRIDE, COMPONENT_VERSION),
KOTLIN_API_VERSION(OVERRIDE, COMPONENT_VERSION),
}
libraries/tools/kotlin-gradle-statistics/test/org/jetbrains/kotlin/statistics/BuildSessionLoggerTest.kt
+37 -15
View File
@@ -5,25 +5,21 @@
package org.jetbrains.kotlin.statistics
import com.sun.javafx.font.Metrics
import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.STATISTICS_FILE_NAME_PATTERN
import org.jetbrains.kotlin.statistics.BuildSessionLogger.Companion.listProfileFiles
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.statistics.metrics.OverrideMetricContainer
import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import org.junit.After
import org.junit.Before
import java.io.File
import java.lang.IllegalStateException
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
import kotlin.test.fail
import kotlin.test.*
class BuildSessionLoggerTest {
lateinit var rootFolder: File
private lateinit var rootFolder: File
private fun statFilesCount() = rootFolder.listFiles().single().listFiles().size
@@ -74,7 +70,6 @@ class BuildSessionLoggerTest {
val logger = BuildSessionLogger(rootFolder, 100, maxFileSize.toLong())
logger.startBuildSession(0, null)
logger.finishBuildSession("", null)
logger.unlockJournalFile()
assertEquals(1, statFilesCount())
@@ -82,7 +77,6 @@ class BuildSessionLoggerTest {
logger.startBuildSession(0, 0)
logger.finishBuildSession("", null)
logger.unlockJournalFile()
//new file should not be created
assertEquals(1, statFilesCount())
@@ -90,7 +84,6 @@ class BuildSessionLoggerTest {
logger.startBuildSession(0, 0)
logger.finishBuildSession("", null)
logger.unlockJournalFile()
// a new file should be created as maximal file size
assertEquals(2, statFilesCount())
@@ -102,7 +95,6 @@ class BuildSessionLoggerTest {
val logger = BuildSessionLogger(rootFolder, maxFiles)
logger.startBuildSession(0, null)
logger.finishBuildSession("", null)
logger.unlockJournalFile()
assertEquals(1, statFilesCount())
val statsFolder = rootFolder.listFiles().single()
@@ -116,7 +108,6 @@ class BuildSessionLoggerTest {
logger.startBuildSession(0, 0)
logger.finishBuildSession("", null)
logger.unlockJournalFile()
assertTrue(
statsFolder.listFiles().filter { it.name == singleStatFile.name }.count() == 1,
@@ -135,6 +126,10 @@ class BuildSessionLoggerTest {
statsFolder.listFiles().filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }.count(),
"Some files which should not be affected, were removed"
)
assertEquals(
statsFolder.listFiles().filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }.sorted(),
listProfileFiles(statsFolder)
)
}
@Test
@@ -142,10 +137,9 @@ class BuildSessionLoggerTest {
val logger = BuildSessionLogger(rootFolder)
logger.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3.4-snapshot")
val startTime = System.currentTimeMillis() - 1000
val startTime = System.currentTimeMillis() - 1001
logger.startBuildSession(1, startTime)
logger.finishBuildSession("Build", null)
logger.unlockJournalFile()
assertEquals(1, statFilesCount())
val statFile = rootFolder.listFiles().single().listFiles().single()
@@ -165,7 +159,6 @@ class BuildSessionLoggerTest {
logger.finishBuildSession("", null)
logger.unlockJournalFile()
val metrics = ArrayList<MetricsContainer>()
MetricsContainer.readFromFile(statFile) {
@@ -186,4 +179,33 @@ class BuildSessionLoggerTest {
val buildDuration = metrics[0].getMetric(NumericalMetrics.GRADLE_BUILD_DURATION)?.getValue() ?: 0L
assertTrue(buildDuration > 1000, "It was expected that build duration is > 1000, but got $buildDuration")
}
@Test
fun testSaveAndReadAllMetrics() {
val logger = BuildSessionLogger(rootFolder)
logger.startBuildSession(1, 1)
for (metric in StringMetrics.values()) {
logger.report(metric, "value")
logger.report(metric, metric.name)
}
for (metric in BooleanMetrics.values()) {
logger.report(metric, true)
}
for (metric in NumericalMetrics.values()) {
logger.report(metric, System.currentTimeMillis())
}
logger.finishBuildSession("Build", null)
MetricsContainer.readFromFile(rootFolder.listFiles().single().listFiles().single()) {
for (metric in StringMetrics.values()) {
assertNotNull(it.getMetric(metric), "Could not find metric ${metric.name}")
}
for (metric in BooleanMetrics.values()) {
assertTrue(it.getMetric(metric)?.getValue() ?: false, "Could not find metric ${metric.name}")
}
for (metric in NumericalMetrics.values()) {
assertNotNull(it.getMetric(metric), "Could not find metric ${metric.name}")
}
}
}
}
prepare/idea-plugin/build.gradle.kts
+1
View File
@@ -67,6 +67,7 @@ val projectsToShadow by extra(listOf(
":js:js.translator",
":kotlin-native:kotlin-native-utils",
":kotlin-native:kotlin-native-library-reader",
":kotlin-gradle-statistics",
":compiler:light-classes",
":compiler:plugin-api",
":kotlin-preloader",