Use unique buildId for file name to store FUS metrics

#KT-58768 In Progress
This commit is contained in:
Nataliya.Valtman
2023-08-31 13:43:03 +02:00
committed by Space Team
parent 30aad31ece
commit b19c496c44
13 changed files with 260 additions and 318 deletions
@@ -10,7 +10,9 @@ import org.jetbrains.kotlin.gradle.report.BuildReportType
import org.jetbrains.kotlin.gradle.testbase.* import org.jetbrains.kotlin.gradle.testbase.*
import org.jetbrains.kotlin.gradle.util.replaceText import org.jetbrains.kotlin.gradle.util.replaceText
import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.DisplayName
import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import kotlin.streams.toList
@DisplayName("FUS statistic") @DisplayName("FUS statistic")
//Tests for FUS statistics have to create new instance of KotlinBuildStatsService //Tests for FUS statistics have to create new instance of KotlinBuildStatsService
@@ -92,7 +94,27 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
@DisplayName("for project with buildSrc") @DisplayName("for project with buildSrc")
@GradleTest @GradleTest
@GradleTestVersions( @GradleTestVersions(
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0], maxVersion = TestVersions.Gradle.G_7_6
)
fun testProjectWithBuildSrcForGradleVersion7(gradleVersion: GradleVersion) {
project(
"instantExecutionWithBuildSrc",
gradleVersion,
) {
build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") {
assertFilesCombinedContains(
Files.list(projectPath.resolve("kotlin-profile")).toList(),
*expectedMetrics,
"BUILD_SRC_EXISTS=true"
)
}
}
}
@DisplayName("for project with buildSrc")
@GradleTest
@GradleTestVersions(
minVersion = TestVersions.Gradle.G_8_0
) )
fun testProjectWithBuildSrc(gradleVersion: GradleVersion) { fun testProjectWithBuildSrc(gradleVersion: GradleVersion) {
project( project(
@@ -177,8 +199,10 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
"incrementalMultiproject", gradleVersion, "incrementalMultiproject", gradleVersion,
) { ) {
//Collect metrics from BuildMetricsService also //Collect metrics from BuildMetricsService also
build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath", build(
buildOptions = defaultBuildOptions.copy(buildReport = listOf(BuildReportType.FILE))) { "compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath",
buildOptions = defaultBuildOptions.copy(buildReport = listOf(BuildReportType.FILE))
) {
assertFileContains( assertFileContains(
fusStatisticsPath, fusStatisticsPath,
"CONFIGURATION_IMPLEMENTATION_COUNT=2", "CONFIGURATION_IMPLEMENTATION_COUNT=2",
@@ -195,16 +219,41 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0], additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
) )
fun testFusStatisticsWithConfigurationCache(gradleVersion: GradleVersion) { fun testFusStatisticsWithConfigurationCache(gradleVersion: GradleVersion) {
testFusStatisticsWithConfigurationCache(gradleVersion, false)
}
@DisplayName("general fields with configuration cache and project isolation")
@GradleTest
@GradleTestVersions(
minVersion = TestVersions.Gradle.G_7_1,
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
)
fun testFusStatisticsWithConfigurationCacheAndProjectIsolation(gradleVersion: GradleVersion) {
testFusStatisticsWithConfigurationCache(gradleVersion, true)
}
fun testFusStatisticsWithConfigurationCache(gradleVersion: GradleVersion, isProjectIsolationEnabled: Boolean) {
project( project(
"simpleProject", "simpleProject",
gradleVersion, gradleVersion,
buildOptions = defaultBuildOptions.copy(configurationCache = true), buildOptions = defaultBuildOptions.copy(
configurationCache = true,
projectIsolation = isProjectIsolationEnabled,
buildReport = listOf(BuildReportType.FILE)
),
) { ) {
build( build(
"compileKotlin", "compileKotlin",
"-Pkotlin.session.logger.root.path=$projectPath", "-Pkotlin.session.logger.root.path=$projectPath",
) { ) {
assertConfigurationCacheStored() assertConfigurationCacheStored()
assertFileContains(
fusStatisticsPath,
*expectedMetrics,
"CONFIGURATION_IMPLEMENTATION_COUNT=1",
"NUMBER_OF_SUBPROJECTS=1",
"COMPILATIONS_COUNT=1"
)
} }
build( build(
@@ -212,6 +261,13 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
"-Pkotlin.session.logger.root.path=$projectPath", "-Pkotlin.session.logger.root.path=$projectPath",
) { ) {
assertConfigurationCacheReused() assertConfigurationCacheReused()
assertFileContains(
fusStatisticsPath,
*expectedMetrics,
"CONFIGURATION_IMPLEMENTATION_COUNT=1",
"NUMBER_OF_SUBPROJECTS=1",
"COMPILATIONS_COUNT=1"
)
} }
} }
} }
@@ -226,15 +226,29 @@ fun assertFileContains(
file: Path, file: Path,
vararg expectedText: String, vararg expectedText: String,
): String { ): String {
assertFileExists(file) return assertFilesCombinedContains(listOf(file), *expectedText)
val text = file.readText() }
/**
* Asserts files together contains all the lines from [expectedText]
*/
fun assertFilesCombinedContains(
files: List<Path>,
vararg expectedText: String,
): String {
files.forEach { assertFileExists(it) }
val text = files.joinToString(separator = "\n") {
it.readText()
}
val textNotInTheFile = expectedText.filterNot { text.contains(it) } val textNotInTheFile = expectedText.filterNot { text.contains(it) }
assert(textNotInTheFile.isEmpty()) { assert(textNotInTheFile.isEmpty()) {
""" """
|$file does not contain: |$files does not contain:
|${textNotInTheFile.joinToString(separator = "\n")} |${textNotInTheFile.joinToString(separator = "\n")}
| |
|actual file content: |actual content:
|"$text" |"$text"
| |
""".trimMargin() """.trimMargin()
@@ -61,7 +61,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
override fun apply(project: Project) { override fun apply(project: Project) {
project.registerDefaultVariantImplementations() project.registerDefaultVariantImplementations()
val buildFusService = BuildFusService.registerIfAbsent(project, pluginVersion) BuildFusService.registerIfAbsent(project, pluginVersion)
project.gradle.projectsEvaluated { project.gradle.projectsEvaluated {
whenBuildEvaluated(project) whenBuildEvaluated(project)
@@ -80,7 +80,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
kotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, pluginVersion) kotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, pluginVersion)
} }
BuildMetricsService.registerIfAbsent(project, buildFusService) BuildMetricsService.registerIfAbsent(project)
} }
private fun addKotlinCompilerConfiguration(project: Project) { private fun addKotlinCompilerConfiguration(project: Project) {
@@ -14,30 +14,37 @@ import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal import org.gradle.api.tasks.Internal
import org.gradle.tooling.events.FailureResult
import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.FinishEvent
import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.OperationCompletionListener
import org.gradle.tooling.events.task.TaskFailureResult import org.gradle.tooling.events.task.TaskFailureResult
import org.gradle.tooling.events.task.TaskFinishEvent import org.gradle.tooling.events.task.TaskFinishEvent
import org.gradle.util.GradleVersion import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
import org.jetbrains.kotlin.gradle.logging.kotlinDebug import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager
import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested
import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
import org.jetbrains.kotlin.gradle.report.TaskExecutionResult
import org.jetbrains.kotlin.gradle.report.reportingSettings import org.jetbrains.kotlin.gradle.report.reportingSettings
import org.jetbrains.kotlin.gradle.tasks.withType import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.gradle.utils.currentBuildId
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import java.io.Serializable import java.io.Serializable
import java.util.UUID.*
internal interface UsesBuildFusService : Task { internal interface UsesBuildFusService : Task {
@get:Internal @get:Internal
val buildFusService: Property<BuildFusService?> val buildFusService: Property<BuildFusService?>
} }
abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoCloseable, OperationCompletionListener { abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoCloseable, OperationCompletionListener {
private var buildFailed: Boolean = false private var buildFailed: Boolean = false
private val log = Logging.getLogger(this.javaClass) private val log = Logging.getLogger(this.javaClass)
@@ -48,42 +55,30 @@ abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoC
interface Parameters : BuildServiceParameters { interface Parameters : BuildServiceParameters {
val configurationMetrics: ListProperty<MetricContainer> val configurationMetrics: ListProperty<MetricContainer>
val fusStatisticsAvailable: Property<Boolean> val useBuildFinishFlowAction: Property<Boolean>
} }
internal val fusMetricsConsumer: NonSynchronizedMetricsContainer? by lazy { private val fusMetricsConsumer = NonSynchronizedMetricsContainer()
// parameters should be already injected on this property access
if (parameters.fusStatisticsAvailable.get()) internal fun getFusMetricsConsumer(): StatisticsValuesConsumer = fusMetricsConsumer
NonSynchronizedMetricsContainer()
else
null
}
internal fun reportFusMetrics(reportAction: (StatisticsValuesConsumer) -> Unit) { internal fun reportFusMetrics(reportAction: (StatisticsValuesConsumer) -> Unit) {
fusMetricsConsumer?.let { reportAction(it) } reportAction(fusMetricsConsumer)
} }
private val buildId = randomUUID().toString()
companion object { companion object {
private val serviceName = "${BuildFusService::class.simpleName}_${BuildFusService::class.java.classLoader.hashCode()}" private val serviceName = "${BuildFusService::class.simpleName}_${BuildFusService::class.java.classLoader.hashCode()}"
private fun fusStatisticsAvailable(project: Project): Boolean { fun registerIfAbsent(project: Project, pluginVersion: String) =
return when { registerIfAbsentImpl(project, pluginVersion).also { serviceProvider ->
//known issue for Gradle with configurationCache: https://github.com/gradle/gradle/issues/20001 SingleActionPerProject.run(project, UsesBuildFusService::class.java.name) {
GradleVersion.current().baseVersion < GradleVersion.version("7.4") -> !project.isConfigurationCacheRequested project.tasks.withType<UsesBuildFusService>().configureEach { task ->
GradleVersion.current().baseVersion < GradleVersion.version("8.1") -> true task.buildFusService.value(serviceProvider).disallowChanges()
//known issue. Cant reuse cache if file is changed in gradle_user_home dir: KT-58768 task.usesService(serviceProvider)
else -> !project.isConfigurationCacheRequested }
}
}
fun registerIfAbsent(project: Project, pluginVersion: String) = registerIfAbsentImpl(project, pluginVersion).also { serviceProvider ->
SingleActionPerProject.run(project, UsesBuildFusService::class.java.name) {
project.tasks.withType<UsesBuildFusService>().configureEach { task ->
task.buildFusService.value(serviceProvider).disallowChanges()
task.usesService(serviceProvider)
} }
} }
}
private fun registerIfAbsentImpl( private fun registerIfAbsentImpl(
project: Project, project: Project,
@@ -96,71 +91,103 @@ abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoC
@Suppress("UNCHECKED_CAST") @Suppress("UNCHECKED_CAST")
return (it.service as Provider<BuildFusService>).also { return (it.service as Provider<BuildFusService>).also {
it.get().parameters.configurationMetrics.add(project.provider { it.get().parameters.configurationMetrics.add(project.provider {
KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project, isProjectIsolationEnabled) KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project)
}) })
} }
} }
//init buildStatsService //init buildStatsService
KotlinBuildStatsService.getOrCreateInstance(project) KotlinBuildStatsService.getOrCreateInstance(project)?.also {
it.recordProjectsEvaluated(project)
}
val fusStatisticsAvailable = fusStatisticsAvailable(project)
val buildReportOutputs = reportingSettings(project).buildReportOutputs val buildReportOutputs = reportingSettings(project).buildReportOutputs
val gradle = project.gradle
//Workaround for known issues for Gradle 8+: https://github.com/gradle/gradle/issues/24887: //Workaround for known issues for Gradle 8+: https://github.com/gradle/gradle/issues/24887:
// when this OperationCompletionListener is called services can be already closed for Gradle 8, // when this OperationCompletionListener is called services can be already closed for Gradle 8,
// so there is a change that no VariantImplementationFactory will be found // so there is a change that no VariantImplementationFactory will be found
return project.gradle.sharedServices.registerIfAbsent(serviceName, BuildFusService::class.java) { spec -> return gradle.sharedServices.registerIfAbsent(serviceName, BuildFusService::class.java) { spec ->
if (fusStatisticsAvailable) {
KotlinBuildStatsService.applyIfInitialised {
it.recordProjectsEvaluated(project.gradle)
}
}
spec.parameters.configurationMetrics.add(project.provider { spec.parameters.configurationMetrics.add(project.provider {
KotlinBuildStatHandler.collectGeneralConfigurationTimeMetrics(project, isProjectIsolationEnabled, buildReportOutputs, pluginVersion) KotlinBuildStatHandler.collectGeneralConfigurationTimeMetrics(
gradle,
buildReportOutputs,
pluginVersion,
isProjectIsolationEnabled
)
}) })
spec.parameters.configurationMetrics.add(project.provider { spec.parameters.configurationMetrics.add(project.provider {
KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project, isProjectIsolationEnabled) KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project)
}) })
spec.parameters.fusStatisticsAvailable.set(fusStatisticsAvailable) spec.parameters.useBuildFinishFlowAction.set(GradleVersion.current().baseVersion >= GradleVersion.version("8.1"))
}.also { buildService -> }.also { buildService ->
if (fusStatisticsAvailable) { @Suppress("DEPRECATION")
when { if (GradleVersion.current().baseVersion >= GradleVersion.version("7.4")
GradleVersion.current().baseVersion < GradleVersion.version("8.1") -> || !project.isConfigurationCacheRequested
BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildService) || project.currentBuildId().name != "buildSrc"
else -> StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult() ) {
} BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildService)
} }
if (GradleVersion.current().baseVersion >= GradleVersion.version("8.1")) {
StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult()
}
} }
} }
} }
override fun onFinish(event: FinishEvent?) { override fun onFinish(event: FinishEvent?) {
parameters.fusStatisticsAvailable.get() //force to calculate configuration metrics before build finish if (event is TaskFinishEvent) {
if ((event is TaskFinishEvent) && (event.result is TaskFailureResult)) { if (event.result is TaskFailureResult) {
buildFailed = true buildFailed = true
}
val taskExecutionResult = TaskExecutionResults[event.descriptor.taskPath]
taskExecutionResult?.also { reportTaskMetrics(it, event) }
} }
} }
override fun close() { override fun close() {
if (parameters.fusStatisticsAvailable.get()) { if (!parameters.useBuildFinishFlowAction.get()) {
recordBuildFinished(null, buildFailed) recordBuildFinished(null, buildFailed)
} }
KotlinBuildStatsService.closeServices()
log.kotlinDebug("Close ${this.javaClass.simpleName}") log.kotlinDebug("Close ${this.javaClass.simpleName}")
} }
internal fun recordBuildFinished(action: String?, buildFailed: Boolean) { internal fun recordBuildFinished(action: String?, buildFailed: Boolean) {
fusMetricsConsumer?.let { metricsConsumer -> KotlinBuildStatHandler.reportGlobalMetrics(fusMetricsConsumer)
KotlinBuildStatHandler.reportGlobalMetrics(metricsConsumer) parameters.configurationMetrics.orElse(emptyList()).get().forEach { it.addToConsumer(fusMetricsConsumer) }
parameters.configurationMetrics.orElse(emptyList()).get().forEach { it.addToConsumer(metricsConsumer) } KotlinBuildStatsService.applyIfInitialised {
KotlinBuildStatsService.applyIfInitialised { it.recordBuildFinish(action, buildFailed, fusMetricsConsumer, buildId)
it.recordBuildFinish(action, buildFailed, metricsConsumer) }
KotlinBuildStatsService.closeServices()
}
private fun reportTaskMetrics(taskExecutionResult: TaskExecutionResult, event: TaskFinishEvent) {
val totalTimeMs = event.result.endTime - event.result.startTime
val buildMetrics = taskExecutionResult.buildMetrics
fusMetricsConsumer.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
fusMetricsConsumer.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
fusMetricsConsumer.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
val linesOfCode = metricsMap[GradleBuildPerformanceMetric.ANALYZED_LINES_NUMBER]
if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) {
fusMetricsConsumer.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode)
fusMetricsConsumer.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode)
metricsMap[GradleBuildPerformanceMetric.ANALYSIS_LPS]?.also { value ->
fusMetricsConsumer.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, value, null, linesOfCode)
}
metricsMap[GradleBuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value ->
fusMetricsConsumer.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
} }
} }
fusMetricsConsumer.report(
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
)
} }
} }
@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.gradle.plugin.statistics
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.api.artifacts.DependencySet import org.gradle.api.artifacts.DependencySet
import org.gradle.api.artifacts.ProjectDependency
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging import org.gradle.api.logging.Logging
import org.jetbrains.kotlin.gradle.plugin.statistics.plugins.ObservablePlugins import org.jetbrains.kotlin.gradle.plugin.statistics.plugins.ObservablePlugins
import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.report.BuildReportType
@@ -49,10 +51,10 @@ class KotlinBuildStatHandler {
* Collect general configuration metrics * Collect general configuration metrics
*/ */
internal fun collectGeneralConfigurationTimeMetrics( internal fun collectGeneralConfigurationTimeMetrics(
project: Project, gradle: Gradle,
isProjectIsolationEnabled: Boolean,
buildReportOutputs: List<BuildReportType>, buildReportOutputs: List<BuildReportType>,
pluginVersion: String, pluginVersion: String,
isProjectIsolationEnabled: Boolean,
): MetricContainer { ): MetricContainer {
val configurationTimeMetrics = MetricContainer() val configurationTimeMetrics = MetricContainer()
@@ -68,15 +70,17 @@ class KotlinBuildStatHandler {
BuildReportType.TRY_NEXT_CONSOLE -> {}//ignore BuildReportType.TRY_NEXT_CONSOLE -> {}//ignore
} }
} }
val gradle = project.gradle
configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath) configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion) configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
//will be updated with KT-58266
if (!isProjectIsolationEnabled) { if (!isProjectIsolationEnabled) {
gradle.taskGraph.whenReady { taskExecutionGraph -> gradle.taskGraph.whenReady { taskExecutionGraph ->
val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct() val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct()
configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install")) configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install"))
} }
} }
} }
configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
@@ -89,14 +93,9 @@ class KotlinBuildStatHandler {
*/ */
internal fun collectProjectConfigurationTimeMetrics( internal fun collectProjectConfigurationTimeMetrics(
project: Project, project: Project,
isProjectIsolationEnabled: Boolean,
): MetricContainer { ): MetricContainer {
val configurationTimeMetrics = MetricContainer() val configurationTimeMetrics = MetricContainer()
if (isProjectIsolationEnabled) { //support project isolation - KT-58768
return configurationTimeMetrics
}
val statisticOverhead = measureTimeMillis { val statisticOverhead = measureTimeMillis {
collectAppliedPluginsStatistics(project, configurationTimeMetrics) collectAppliedPluginsStatistics(project, configurationTimeMetrics)
@@ -202,8 +201,11 @@ class KotlinBuildStatHandler {
} }
} }
private fun reportLibrariesVersions(configurationTimeMetrics: MetricContainer, dependencies: DependencySet?) { private fun reportLibrariesVersions(
dependencies?.forEach { dependency -> configurationTimeMetrics: MetricContainer,
dependencies: DependencySet?,
) {
dependencies?.filter { it !is ProjectDependency }?.forEach { dependency ->
when { when {
dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put( dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put(
StringMetrics.LIBRARY_SPRING_VERSION, StringMetrics.LIBRARY_SPRING_VERSION,
@@ -272,11 +274,12 @@ class KotlinBuildStatHandler {
sessionLogger: BuildSessionLogger, sessionLogger: BuildSessionLogger,
action: String?, action: String?,
buildFailed: Boolean, buildFailed: Boolean,
metrics: NonSynchronizedMetricsContainer, buildId: String,
metricsContainer: NonSynchronizedMetricsContainer,
) { ) {
runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") { runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") {
metrics.sendToConsumer(sessionLogger) metricsContainer.sendToConsumer(sessionLogger)
sessionLogger.finishBuildSession(action, buildFailed) sessionLogger.finishBuildSession(action, buildFailed, buildId)
} }
} }
@@ -187,9 +187,12 @@ internal abstract class KotlinBuildStatsService internal constructor() : Statist
override fun close() { override fun close() {
} }
open fun recordBuildFinish(action: String?, buildFailed: Boolean, metric: NonSynchronizedMetricsContainer) {} /**
* Collects metrics at the end of a build
*/
open fun recordBuildFinish(action: String?, buildFailed: Boolean, metricsContainer: NonSynchronizedMetricsContainer, buildId: String) {}
open fun recordProjectsEvaluated(gradle: Gradle) {} open fun recordProjectsEvaluated(project: Project) {}
} }
internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) : internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) :
@@ -256,12 +259,12 @@ internal abstract class AbstractKotlinBuildStatsService(
return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime
} }
override fun recordProjectsEvaluated(gradle: Gradle) { override fun recordProjectsEvaluated(project: Project) {
runSafe("${DefaultKotlinBuildStatsService::class.java}.projectEvaluated") { runSafe("${DefaultKotlinBuildStatsService::class.java}.projectEvaluated") {
if (!sessionLogger.isBuildSessionStarted()) { if (!sessionLogger.isBuildSessionStarted()) {
sessionLogger.startBuildSession( sessionLogger.startBuildSession(
DaemonReuseCounter.incrementAndGetOrdinal(), DaemonReuseCounter.incrementAndGetOrdinal(),
gradleBuildStartTime(gradle) gradleBuildStartTime(project.gradle),
) )
} }
} }
@@ -299,7 +302,7 @@ internal class DefaultKotlinBuildStatsService internal constructor(
report(StringMetrics.valueOf(name), value, subprojectName, weight) report(StringMetrics.valueOf(name), value, subprojectName, weight)
//only one jmx bean service should report global metrics //only one jmx bean service should report global metrics
override fun recordBuildFinish(action: String?, buildFailed: Boolean, metrics: NonSynchronizedMetricsContainer) { override fun recordBuildFinish(action: String?, buildFailed: Boolean, metricsContainer: NonSynchronizedMetricsContainer, buildId: String) {
KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, metrics) KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, buildId, metricsContainer)
} }
} }
@@ -15,7 +15,6 @@ import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal import org.gradle.api.tasks.Internal
import org.gradle.tooling.events.FailureResult
import org.gradle.tooling.events.FinishEvent import org.gradle.tooling.events.FinishEvent
import org.gradle.tooling.events.OperationCompletionListener import org.gradle.tooling.events.OperationCompletionListener
import org.gradle.tooling.events.task.TaskExecutionResult import org.gradle.tooling.events.task.TaskExecutionResult
@@ -35,14 +34,11 @@ import org.jetbrains.kotlin.gradle.report.BuildReportsService.Companion.getStart
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.gradle.tasks.withType import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.ConcurrentLinkedQueue
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager
import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested
import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFusService
import java.lang.management.ManagementFactory import java.lang.management.ManagementFactory
internal interface UsesBuildMetricsService : Task { internal interface UsesBuildMetricsService : Task {
@@ -63,7 +59,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
val projectName: Property<String> val projectName: Property<String>
val kotlinVersion: Property<String> val kotlinVersion: Property<String>
val buildConfigurationTags: ListProperty<StatTag> val buildConfigurationTags: ListProperty<StatTag>
val fusService: Property<BuildFusService>
} }
private val log = Logging.getLogger(this.javaClass) private val log = Logging.getLogger(this.javaClass)
@@ -114,30 +109,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
val taskExecutionResult = TaskExecutionResults[taskPath] val taskExecutionResult = TaskExecutionResults[taskPath]
taskExecutionResult?.buildMetrics?.also { taskExecutionResult?.buildMetrics?.also {
buildMetrics.addAll(it) buildMetrics.addAll(it)
parameters.fusService.orNull?.reportFusMetrics { collector ->
collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
val linesOfCode = metricsMap[GradleBuildPerformanceMetric.ANALYZED_LINES_NUMBER]
if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) {
collector.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode)
collector.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode)
metricsMap[GradleBuildPerformanceMetric.ANALYSIS_LPS]?.also { value ->
collector.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, value, null, linesOfCode)
}
metricsMap[GradleBuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value ->
collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
}
}
collector.report(
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
)
}
} }
val buildOperation = TaskRecord( val buildOperation = TaskRecord(
@@ -190,7 +161,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
private fun registerIfAbsentImpl( private fun registerIfAbsentImpl(
project: Project, project: Project,
fusService: Provider<BuildFusService>
): Provider<BuildMetricsService>? { ): Provider<BuildMetricsService>? {
// Return early if the service was already registered to avoid the overhead of reading the reporting settings below // Return early if the service was already registered to avoid the overhead of reading the reporting settings below
project.gradle.sharedServices.registrations.findByName(serviceName)?.let { project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
@@ -224,7 +194,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
it.parameters.projectDir.set(project.rootProject.layout.projectDirectory) it.parameters.projectDir.set(project.rootProject.layout.projectDirectory)
//init gradle tags for build scan and http reports //init gradle tags for build scan and http reports
it.parameters.buildConfigurationTags.value(setupTags(project)) it.parameters.buildConfigurationTags.value(setupTags(project))
it.parameters.fusService.set(fusService)
}.also { }.also {
subscribeForTaskEvents(project, it) subscribeForTaskEvents(project, it)
} }
@@ -291,8 +260,8 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
} }
} }
fun registerIfAbsent(project: Project, fusService: Provider<BuildFusService>) = fun registerIfAbsent(project: Project) =
registerIfAbsentImpl(project, fusService)?.also { serviceProvider -> registerIfAbsentImpl(project)?.also { serviceProvider ->
SingleActionPerProject.run(project, UsesBuildMetricsService::class.java.name) { SingleActionPerProject.run(project, UsesBuildMetricsService::class.java.name) {
project.tasks.withType<UsesBuildMetricsService>().configureEach { task -> project.tasks.withType<UsesBuildMetricsService>().configureEach { task ->
task.buildMetricsService.value(serviceProvider).disallowChanges() task.buildMetricsService.value(serviceProvider).disallowChanges()
@@ -205,7 +205,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
classLoadersCachingService, classLoadersCachingService,
buildFinishedListenerService, buildFinishedListenerService,
buildIdService, buildIdService,
buildFusService.orNull?.fusMetricsConsumer buildFusService.orNull?.getFusMetricsConsumer()
) )
} }
} }
@@ -13,8 +13,9 @@ import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import org.junit.After import org.junit.After
import org.junit.Before import org.junit.Before
import java.io.File import java.io.File
import java.lang.IllegalStateException
import java.nio.file.Files import java.nio.file.Files
import java.util.*
import kotlin.collections.ArrayList
import kotlin.test.* import kotlin.test.*
class BuildSessionLoggerTest { class BuildSessionLoggerTest {
@@ -44,57 +45,32 @@ class BuildSessionLoggerTest {
assertEquals(false, logger1.isBuildSessionStarted()) assertEquals(false, logger1.isBuildSessionStarted())
logger1.startBuildSession(0, 0) logger1.startBuildSession(0, 0)
// created single folder with kotlin statistics
assertEquals(1, statFilesCount())
// check that file locks work properly and do not use the same file // check that file locks work properly and do not use the same file
logger2.startBuildSession(0, 0) logger2.startBuildSession(0, 0)
assertEquals(2, statFilesCount())
assertEquals(true, logger1.isBuildSessionStarted()) assertEquals(true, logger1.isBuildSessionStarted())
logger1.finishBuildSession("", false) logger1.finishBuildSession("", false, buildId = "1")
logger2.finishBuildSession("", false) // created single folder with kotlin statistics
assertEquals(1, statFilesCount())
logger2.finishBuildSession("", false, buildId = "2")
assertEquals(2, statFilesCount())
rootFolder.listFiles()?.first()?.listFiles()?.forEach { file -> rootFolder.listFiles()?.first()?.listFiles()?.forEach { file ->
assertTrue( assertTrue(
file.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()), file.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN),
"Check that file name ${file.name} matches pattern ${BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN}" "Check that file name \'${file.name}\' matches pattern \'${BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN}\'"
) )
} }
} }
@Test
fun testSizeLimitation() {
val maxFileSize = 10_000
val logger = BuildSessionLogger(rootFolder, 100, maxFileSize.toLong())
logger.startBuildSession(0, null)
logger.finishBuildSession("", false)
assertEquals(1, statFilesCount())
val file2edit = rootFolder.listFiles()?.first()?.listFiles()?.first() ?: fail("Could not find single stat file")
logger.startBuildSession(0, 0)
logger.finishBuildSession("", false)
//new file should not be created
assertEquals(1, statFilesCount())
file2edit.appendBytes(ByteArray(maxFileSize))
logger.startBuildSession(0, 0)
logger.finishBuildSession("", false)
// a new file should be created as maximal file size
assertEquals(2, statFilesCount())
}
@Test @Test
fun testDeleteExtraFiles() { fun testDeleteExtraFiles() {
val maxFiles = 100 val maxFiles = 100
val logger = BuildSessionLogger(rootFolder, maxFiles) val logger = BuildSessionLogger(rootFolder, maxFiles)
logger.startBuildSession(0, null) logger.startBuildSession(0, null)
logger.finishBuildSession("", false) logger.finishBuildSession("", false, buildId = "1")
assertEquals(1, statFilesCount()) assertEquals(1, statFilesCount())
val statsFolder = rootFolder.listFiles()?.single() ?: fail("${rootFolder.absolutePath} was not created") val statsFolder = rootFolder.listFiles()?.single() ?: fail("${rootFolder.absolutePath} was not created")
@@ -102,12 +78,11 @@ class BuildSessionLoggerTest {
for (i in 1..200) { for (i in 1..200) {
File(statsFolder, "$i").createNewFile() File(statsFolder, "$i").createNewFile()
val formattedIndex = "%04d".format(i) File(statsFolder, "${UUID.randomUUID()}.profile").createNewFile()
File(statsFolder, "$formattedIndex-12-30-12-59-59-123.profile").createNewFile()
} }
logger.startBuildSession(0, 0) logger.startBuildSession(0, 0)
logger.finishBuildSession("", false) logger.finishBuildSession("", false, buildId = "1")
assertTrue( assertTrue(
statsFolder.listFiles()?.count { it.name == singleStatFile.name } == 1, statsFolder.listFiles()?.count { it.name == singleStatFile.name } == 1,
@@ -117,48 +92,39 @@ class BuildSessionLoggerTest {
// files not matching the pattern should not be affected // files not matching the pattern should not be affected
assertEquals( assertEquals(
200, 200,
statsFolder.listFiles()?.count { !it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()) }, statsFolder.listFiles()?.count { !it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN) },
"Some files which should not be affected, were removed" "Some files which should not be affected, were removed"
) )
assertEquals( assertEquals(
maxFiles, maxFiles,
statsFolder.listFiles()?.count { it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()) }, statsFolder.listFiles()?.count { it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN) },
"Some files which should not be affected, were removed" "Some files which should not be affected, were removed"
) )
assertEquals(
statsFolder.listFiles()?.filter { it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted(),
BuildSessionLogger.listProfileFiles(statsFolder)
)
} }
@Test @Test
fun testReadWriteMetrics() { fun testReadWriteMetrics() {
val buildId = "test"
val logger = BuildSessionLogger(rootFolder) val logger = BuildSessionLogger(rootFolder)
logger.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3.4-snapshot") logger.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3.4-snapshot")
val startTime = System.currentTimeMillis() - 1001 val startTime = System.currentTimeMillis() - 1001
logger.startBuildSession(1, startTime) logger.startBuildSession(1, startTime)
logger.finishBuildSession("Build", false) val reportFile = rootFolder.resolve(BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME)
.resolve(buildId + BuildSessionLogger.Companion.PROFILE_FILE_NAME_SUFFIX)
reportFile.createNewFile()
reportFile.appendText("${StringMetrics.USE_FIR.name}=true\n")
logger.finishBuildSession("Build", false, buildId)
assertEquals(1, statFilesCount()) assertEquals(1, statFilesCount())
val statFile = rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file") val statFile = rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")
statFile.appendBytes("break format of the file".toByteArray()) statFile.appendBytes("break format of the file".toByteArray()) //this line should be filtered by MetricsContainer.readFromFile
logger.startBuildSession(1, startTime) logger.startBuildSession(1, startTime)
logger.finishBuildSession("", false, buildId)
// the file should be locked
try {
MetricsContainer.readFromFile(statFile) {
fail("No metrics should be read due to locked file")
}
fail("Method should have failed")
} catch (e: IllegalStateException) {
//all right
}
logger.finishBuildSession("", false)
val metrics = ArrayList<MetricsContainer>() val metrics = ArrayList<MetricsContainer>()
MetricsContainer.readFromFile(statFile) { MetricsContainer.readFromFile(statFile) {
@@ -166,6 +132,11 @@ class BuildSessionLoggerTest {
} }
assertEquals(2, metrics.size, "Invalid number of MerticContainers was read") assertEquals(2, metrics.size, "Invalid number of MerticContainers was read")
assertEquals(
"true",
metrics[0].getMetric(StringMetrics.USE_FIR)?.getValue()
)
assertEquals( assertEquals(
"1.2.3", "1.2.3",
metrics[0].getMetric(StringMetrics.KOTLIN_COMPILER_VERSION)?.getValue() metrics[0].getMetric(StringMetrics.KOTLIN_COMPILER_VERSION)?.getValue()
@@ -194,7 +165,7 @@ class BuildSessionLoggerTest {
for (metric in NumericalMetrics.values()) { for (metric in NumericalMetrics.values()) {
logger.report(metric, System.currentTimeMillis()) logger.report(metric, System.currentTimeMillis())
} }
logger.finishBuildSession("Build", false) logger.finishBuildSession("Build", false, buildId = "test")
MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) { MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) {
for (metric in StringMetrics.values()) { for (metric in StringMetrics.values()) {
@@ -217,7 +188,7 @@ class BuildSessionLoggerTest {
logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 10, null, 9) logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 10, null, 9)
logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 100, null, 1) logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 100, null, 1)
logger.finishBuildSession("Build", false) logger.finishBuildSession("Build", false, buildId = "1")
MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) { MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) {
assertEquals(19L, it.getMetric(NumericalMetrics.ANALYSIS_LINES_PER_SECOND)?.getValue()) assertEquals(19L, it.getMetric(NumericalMetrics.ANALYSIS_LINES_PER_SECOND)?.getValue())
} }
@@ -5,85 +5,79 @@
package org.jetbrains.kotlin.statistics package org.jetbrains.kotlin.statistics
import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger
import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger
import org.jetbrains.kotlin.statistics.metrics.* import org.jetbrains.kotlin.statistics.metrics.*
import java.io.File import java.io.File
import java.io.FileOutputStream
import java.io.IOException import java.io.IOException
import java.time.LocalDateTime import java.nio.file.Files
import java.time.format.DateTimeFormatter
class BuildSessionLogger( class BuildSessionLogger(
rootPath: File, rootPath: File,
private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES, private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE, private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
private val forceValuesValidation: Boolean = false, forceValuesValidation: Boolean = false,
) : StatisticsValuesConsumer { ) : StatisticsValuesConsumer {
companion object { companion object {
const val PROFILE_FILE_NAME_SUFFIX = ".profile"
const val STATISTICS_FOLDER_NAME = "kotlin-profile" const val STATISTICS_FOLDER_NAME = "kotlin-profile"
const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}(.\\d+)?.profile" val STATISTICS_FILE_NAME_PATTERN = "[\\w-]*$PROFILE_FILE_NAME_SUFFIX".toRegex()
private const val DEFAULT_MAX_PROFILE_FILES = 1_000 private const val DEFAULT_MAX_PROFILE_FILES = 1_000
private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
private const val DEFAULT_MAX_FILE_AGE = 30 * 24 * 3600 * 1000L //30 days private const val DEFAULT_MAX_FILE_AGE = 30 * 24 * 3600 * 1000L //30 days
fun listProfileFiles(statisticsFolder: File): List<File>? { fun listProfileFiles(statisticsFolder: File): List<File> {
return statisticsFolder.listFiles()?.filterTo(ArrayList()) { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted() return Files.newDirectoryStream(statisticsFolder.toPath()).use { dirStream ->
dirStream.map { it.toFile() }
.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN) }
.sortedBy { it.lastModified() }
}
} }
} }
private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS")
private val profileFileNameSuffix = ".profile"
private val statisticsFolder: File = File( private val statisticsFolder: File = File(
rootPath, rootPath,
STATISTICS_FOLDER_NAME STATISTICS_FOLDER_NAME
).also { it.mkdirs() } ).also { it.mkdirs() }
private var buildSession: BuildSession? = null private var buildSession: BuildSession? = null
private var trackingFile: IRecordLogger? = null
private val metricsContainer = MetricsContainer(forceValuesValidation) private val metricsContainer = MetricsContainer(forceValuesValidation)
@Synchronized @Synchronized
fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) { fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
buildSession = BuildSession(buildStartedTime)
report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart) report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart)
buildSession = BuildSession(buildStartedTime)
initTrackingFile()
} }
@Synchronized @Synchronized
fun isBuildSessionStarted() = buildSession != null fun isBuildSessionStarted() = buildSession != null
@Synchronized /**
private fun closeTrackingFile() { * Initializes a new build report file
trackingFile?.let { * The following contracts are implemented:
metricsContainer.flush(it) * - each file contains metrics for one build
it.close() * - any other process can add metrics to the file during build
trackingFile = null * - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
*/
private fun storeMetricsIntoFile(buildId: String) {
try {
statisticsFolder.mkdirs()
val file = File(statisticsFolder, buildId + PROFILE_FILE_NAME_SUFFIX)
FileOutputStream(file, true).bufferedWriter().use {
metricsContainer.flush(it)
}
} catch (_: IOException) {
//ignore io exception
} }
} }
/** private fun clearOldFiles() {
* Initializes a new tracking file
* The following contracts are implemented:
* - number of tracking files should not be more than maxProfileFiles (the earlier file created the earlier deleted)
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
* - files are ordered on the basis of name (creation timestamp)
* - if the last file has size less then maxFileSize, the next record will be append to it (new file created otherwise)
* -
*/
@Synchronized
private fun initTrackingFile() {
closeTrackingFile()
// Get list of existing files. Try to create folder if possible, return from function if failed to create folder // Get list of existing files. Try to create folder if possible, return from function if failed to create folder
val fileCandidates = listProfileFiles(statisticsFolder) ?: if (statisticsFolder.mkdirs()) emptyList() else return val fileCandidates = listProfileFiles(statisticsFolder)
for ((index, file) in fileCandidates.withIndex()) { for ((index, file) in fileCandidates.withIndex()) {
val toDelete = if (index < fileCandidates.size - maxProfileFiles) val toDelete = if (index < fileCandidates.size - maxProfileFiles)
@@ -96,44 +90,14 @@ class BuildSessionLogger(
file.delete() file.delete()
} }
} }
// emergency check. What if a lot of files are locked due to some reason
if ((listProfileFiles(statisticsFolder)?.size ?: 0) > maxProfileFiles * 2) {
trackingFile = NullRecordLogger()
return
}
fun newFile(): File {
val timestamp = profileFileNameFormatter.format(LocalDateTime.now())
var result = File(statisticsFolder, timestamp + profileFileNameSuffix)
var suffixIndex = 0
while (result.exists()) {
result = File(statisticsFolder, "${timestamp}.${suffixIndex++}$profileFileNameSuffix")
}
return result
}
val lastFile = fileCandidates.lastOrNull() ?: newFile()
trackingFile = try {
if (lastFile.length() < maxFileSize) {
FileRecordLogger(lastFile)
} else {
FileRecordLogger(newFile())
}
} catch (e: IOException) {
try {
FileRecordLogger(newFile())
} catch (e: IOException) {
NullRecordLogger()
}
}
} }
@Synchronized @Synchronized
fun finishBuildSession( fun finishBuildSession(
@Suppress("UNUSED_PARAMETER") action: String?, @Suppress("UNUSED_PARAMETER") action: String?,
buildFailed: Boolean, buildFailed: Boolean,
buildId: String,
) { ) {
try { try {
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration // nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
@@ -147,17 +111,13 @@ class BuildSessionLogger(
report(NumericalMetrics.BUILD_FINISH_TIME, finishTime) report(NumericalMetrics.BUILD_FINISH_TIME, finishTime)
report(BooleanMetrics.BUILD_FAILED, buildFailed) report(BooleanMetrics.BUILD_FAILED, buildFailed)
} }
buildSession = null
} finally { } finally {
unlockJournalFile() buildSession = null
storeMetricsIntoFile(buildId)
clearOldFiles()
} }
} }
@Synchronized
private fun unlockJournalFile() {
closeTrackingFile()
}
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) = override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) =
metricsContainer.report(metric, value, subprojectName, weight) metricsContainer.report(metric, value, subprojectName, weight)
@@ -1,47 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
import java.io.File
import java.io.IOException
import java.io.OutputStream
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.channels.FileLock
import java.nio.file.Paths
import java.nio.file.StandardOpenOption
class FileRecordLogger(file: File) : IRecordLogger {
private val channel: FileChannel =
FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
?: throw IOException("Could not open file $file")
private val outputStream: OutputStream
private val lock: FileLock
init {
lock = try {
channel.tryLock() ?: throw IOException("Could not acquire an exclusive lock of file ${file.name}")
} catch (e: Exception) {
channel.close()
// wrap in order to unify with FileOverlappingException
throw IOException(e.message, e)
}
outputStream = Channels.newOutputStream(channel)
}
override fun append(s: String) {
outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING))
}
override fun close() {
channel.use {
outputStream.use {
lock.release()
}
}
}
}
@@ -143,8 +143,7 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta
return true return true
} }
fun flush(trackingFile: IRecordLogger?) { fun flush(writer: BufferedWriter) {
if (trackingFile == null) return
val allMetrics = TreeMap<MetricDescriptor, IMetricContainer<out Any>>() val allMetrics = TreeMap<MetricDescriptor, IMetricContainer<out Any>>()
synchronized(metricsLock) { synchronized(metricsLock) {
allMetrics.putAll(numericalMetrics) allMetrics.putAll(numericalMetrics)
@@ -153,10 +152,10 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta
} }
for (entry in allMetrics.entries) { for (entry in allMetrics.entries) {
val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}" val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}"
trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}") writer.appendLine("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
} }
trackingFile.append(BUILD_SESSION_SEPARATOR) writer.appendLine(BUILD_SESSION_SEPARATOR)
synchronized(metricsLock) { synchronized(metricsLock) {
stringMetrics.clear() stringMetrics.clear()
@@ -1,13 +0,0 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
class NullRecordLogger : IRecordLogger {
override fun append(s: String) {}
override fun close() {
}
}