Use unique buildId for file name to store FUS metrics
#KT-58768 In Progress
This commit is contained in:
committed by
Space Team
parent
30aad31ece
commit
b19c496c44
+60
-4
@@ -10,7 +10,9 @@ import org.jetbrains.kotlin.gradle.report.BuildReportType
|
||||
import org.jetbrains.kotlin.gradle.testbase.*
|
||||
import org.jetbrains.kotlin.gradle.util.replaceText
|
||||
import org.junit.jupiter.api.DisplayName
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.streams.toList
|
||||
|
||||
@DisplayName("FUS statistic")
|
||||
//Tests for FUS statistics have to create new instance of KotlinBuildStatsService
|
||||
@@ -92,7 +94,27 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
|
||||
@DisplayName("for project with buildSrc")
|
||||
@GradleTest
|
||||
@GradleTestVersions(
|
||||
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
|
||||
maxVersion = TestVersions.Gradle.G_7_6
|
||||
)
|
||||
fun testProjectWithBuildSrcForGradleVersion7(gradleVersion: GradleVersion) {
|
||||
project(
|
||||
"instantExecutionWithBuildSrc",
|
||||
gradleVersion,
|
||||
) {
|
||||
build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath") {
|
||||
assertFilesCombinedContains(
|
||||
Files.list(projectPath.resolve("kotlin-profile")).toList(),
|
||||
*expectedMetrics,
|
||||
"BUILD_SRC_EXISTS=true"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@DisplayName("for project with buildSrc")
|
||||
@GradleTest
|
||||
@GradleTestVersions(
|
||||
minVersion = TestVersions.Gradle.G_8_0
|
||||
)
|
||||
fun testProjectWithBuildSrc(gradleVersion: GradleVersion) {
|
||||
project(
|
||||
@@ -177,8 +199,10 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
|
||||
"incrementalMultiproject", gradleVersion,
|
||||
) {
|
||||
//Collect metrics from BuildMetricsService also
|
||||
build("compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath",
|
||||
buildOptions = defaultBuildOptions.copy(buildReport = listOf(BuildReportType.FILE))) {
|
||||
build(
|
||||
"compileKotlin", "-Pkotlin.session.logger.root.path=$projectPath",
|
||||
buildOptions = defaultBuildOptions.copy(buildReport = listOf(BuildReportType.FILE))
|
||||
) {
|
||||
assertFileContains(
|
||||
fusStatisticsPath,
|
||||
"CONFIGURATION_IMPLEMENTATION_COUNT=2",
|
||||
@@ -195,16 +219,41 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
|
||||
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
|
||||
)
|
||||
fun testFusStatisticsWithConfigurationCache(gradleVersion: GradleVersion) {
|
||||
testFusStatisticsWithConfigurationCache(gradleVersion, false)
|
||||
}
|
||||
|
||||
@DisplayName("general fields with configuration cache and project isolation")
|
||||
@GradleTest
|
||||
@GradleTestVersions(
|
||||
minVersion = TestVersions.Gradle.G_7_1,
|
||||
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
|
||||
)
|
||||
fun testFusStatisticsWithConfigurationCacheAndProjectIsolation(gradleVersion: GradleVersion) {
|
||||
testFusStatisticsWithConfigurationCache(gradleVersion, true)
|
||||
}
|
||||
|
||||
fun testFusStatisticsWithConfigurationCache(gradleVersion: GradleVersion, isProjectIsolationEnabled: Boolean) {
|
||||
project(
|
||||
"simpleProject",
|
||||
gradleVersion,
|
||||
buildOptions = defaultBuildOptions.copy(configurationCache = true),
|
||||
buildOptions = defaultBuildOptions.copy(
|
||||
configurationCache = true,
|
||||
projectIsolation = isProjectIsolationEnabled,
|
||||
buildReport = listOf(BuildReportType.FILE)
|
||||
),
|
||||
) {
|
||||
build(
|
||||
"compileKotlin",
|
||||
"-Pkotlin.session.logger.root.path=$projectPath",
|
||||
) {
|
||||
assertConfigurationCacheStored()
|
||||
assertFileContains(
|
||||
fusStatisticsPath,
|
||||
*expectedMetrics,
|
||||
"CONFIGURATION_IMPLEMENTATION_COUNT=1",
|
||||
"NUMBER_OF_SUBPROJECTS=1",
|
||||
"COMPILATIONS_COUNT=1"
|
||||
)
|
||||
}
|
||||
|
||||
build(
|
||||
@@ -212,6 +261,13 @@ class FusStatisticsIT : KGPDaemonsBaseTest() {
|
||||
"-Pkotlin.session.logger.root.path=$projectPath",
|
||||
) {
|
||||
assertConfigurationCacheReused()
|
||||
assertFileContains(
|
||||
fusStatisticsPath,
|
||||
*expectedMetrics,
|
||||
"CONFIGURATION_IMPLEMENTATION_COUNT=1",
|
||||
"NUMBER_OF_SUBPROJECTS=1",
|
||||
"COMPILATIONS_COUNT=1"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-4
@@ -226,15 +226,29 @@ fun assertFileContains(
|
||||
file: Path,
|
||||
vararg expectedText: String,
|
||||
): String {
|
||||
assertFileExists(file)
|
||||
val text = file.readText()
|
||||
return assertFilesCombinedContains(listOf(file), *expectedText)
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts files together contains all the lines from [expectedText]
|
||||
*/
|
||||
fun assertFilesCombinedContains(
|
||||
files: List<Path>,
|
||||
vararg expectedText: String,
|
||||
): String {
|
||||
files.forEach { assertFileExists(it) }
|
||||
|
||||
val text = files.joinToString(separator = "\n") {
|
||||
it.readText()
|
||||
}
|
||||
|
||||
val textNotInTheFile = expectedText.filterNot { text.contains(it) }
|
||||
assert(textNotInTheFile.isEmpty()) {
|
||||
"""
|
||||
|$file does not contain:
|
||||
|$files does not contain:
|
||||
|${textNotInTheFile.joinToString(separator = "\n")}
|
||||
|
|
||||
|actual file content:
|
||||
|actual content:
|
||||
|"$text"
|
||||
|
|
||||
""".trimMargin()
|
||||
|
||||
+2
-2
@@ -61,7 +61,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
|
||||
|
||||
override fun apply(project: Project) {
|
||||
project.registerDefaultVariantImplementations()
|
||||
val buildFusService = BuildFusService.registerIfAbsent(project, pluginVersion)
|
||||
BuildFusService.registerIfAbsent(project, pluginVersion)
|
||||
|
||||
project.gradle.projectsEvaluated {
|
||||
whenBuildEvaluated(project)
|
||||
@@ -80,7 +80,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
|
||||
kotlinGradleBuildServices.detectKotlinPluginLoadedInMultipleProjects(project, pluginVersion)
|
||||
}
|
||||
|
||||
BuildMetricsService.registerIfAbsent(project, buildFusService)
|
||||
BuildMetricsService.registerIfAbsent(project)
|
||||
}
|
||||
|
||||
private fun addKotlinCompilerConfiguration(project: Project) {
|
||||
|
||||
+81
-54
@@ -14,30 +14,37 @@ import org.gradle.api.provider.Provider
|
||||
import org.gradle.api.services.BuildService
|
||||
import org.gradle.api.services.BuildServiceParameters
|
||||
import org.gradle.api.tasks.Internal
|
||||
import org.gradle.tooling.events.FailureResult
|
||||
import org.gradle.tooling.events.FinishEvent
|
||||
import org.gradle.tooling.events.OperationCompletionListener
|
||||
import org.gradle.tooling.events.task.TaskFailureResult
|
||||
import org.gradle.tooling.events.task.TaskFinishEvent
|
||||
import org.gradle.util.GradleVersion
|
||||
import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
|
||||
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
|
||||
import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
|
||||
import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager
|
||||
import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested
|
||||
import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled
|
||||
import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
|
||||
import org.jetbrains.kotlin.gradle.report.TaskExecutionResult
|
||||
import org.jetbrains.kotlin.gradle.report.reportingSettings
|
||||
import org.jetbrains.kotlin.gradle.tasks.withType
|
||||
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
|
||||
import org.jetbrains.kotlin.gradle.utils.currentBuildId
|
||||
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
|
||||
import org.jetbrains.kotlin.statistics.metrics.StatisticsValuesConsumer
|
||||
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
|
||||
import org.jetbrains.kotlin.statistics.metrics.StringMetrics
|
||||
import java.io.Serializable
|
||||
import java.util.UUID.*
|
||||
|
||||
|
||||
internal interface UsesBuildFusService : Task {
|
||||
@get:Internal
|
||||
val buildFusService: Property<BuildFusService?>
|
||||
}
|
||||
|
||||
abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoCloseable, OperationCompletionListener {
|
||||
private var buildFailed: Boolean = false
|
||||
private val log = Logging.getLogger(this.javaClass)
|
||||
@@ -48,42 +55,30 @@ abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoC
|
||||
|
||||
interface Parameters : BuildServiceParameters {
|
||||
val configurationMetrics: ListProperty<MetricContainer>
|
||||
val fusStatisticsAvailable: Property<Boolean>
|
||||
val useBuildFinishFlowAction: Property<Boolean>
|
||||
}
|
||||
|
||||
internal val fusMetricsConsumer: NonSynchronizedMetricsContainer? by lazy {
|
||||
// parameters should be already injected on this property access
|
||||
if (parameters.fusStatisticsAvailable.get())
|
||||
NonSynchronizedMetricsContainer()
|
||||
else
|
||||
null
|
||||
}
|
||||
private val fusMetricsConsumer = NonSynchronizedMetricsContainer()
|
||||
|
||||
internal fun getFusMetricsConsumer(): StatisticsValuesConsumer = fusMetricsConsumer
|
||||
|
||||
internal fun reportFusMetrics(reportAction: (StatisticsValuesConsumer) -> Unit) {
|
||||
fusMetricsConsumer?.let { reportAction(it) }
|
||||
reportAction(fusMetricsConsumer)
|
||||
}
|
||||
private val buildId = randomUUID().toString()
|
||||
|
||||
companion object {
|
||||
private val serviceName = "${BuildFusService::class.simpleName}_${BuildFusService::class.java.classLoader.hashCode()}"
|
||||
|
||||
private fun fusStatisticsAvailable(project: Project): Boolean {
|
||||
return when {
|
||||
//known issue for Gradle with configurationCache: https://github.com/gradle/gradle/issues/20001
|
||||
GradleVersion.current().baseVersion < GradleVersion.version("7.4") -> !project.isConfigurationCacheRequested
|
||||
GradleVersion.current().baseVersion < GradleVersion.version("8.1") -> true
|
||||
//known issue. Cant reuse cache if file is changed in gradle_user_home dir: KT-58768
|
||||
else -> !project.isConfigurationCacheRequested
|
||||
}
|
||||
}
|
||||
|
||||
fun registerIfAbsent(project: Project, pluginVersion: String) = registerIfAbsentImpl(project, pluginVersion).also { serviceProvider ->
|
||||
SingleActionPerProject.run(project, UsesBuildFusService::class.java.name) {
|
||||
project.tasks.withType<UsesBuildFusService>().configureEach { task ->
|
||||
task.buildFusService.value(serviceProvider).disallowChanges()
|
||||
task.usesService(serviceProvider)
|
||||
fun registerIfAbsent(project: Project, pluginVersion: String) =
|
||||
registerIfAbsentImpl(project, pluginVersion).also { serviceProvider ->
|
||||
SingleActionPerProject.run(project, UsesBuildFusService::class.java.name) {
|
||||
project.tasks.withType<UsesBuildFusService>().configureEach { task ->
|
||||
task.buildFusService.value(serviceProvider).disallowChanges()
|
||||
task.usesService(serviceProvider)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun registerIfAbsentImpl(
|
||||
project: Project,
|
||||
@@ -96,71 +91,103 @@ abstract class BuildFusService : BuildService<BuildFusService.Parameters>, AutoC
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
return (it.service as Provider<BuildFusService>).also {
|
||||
it.get().parameters.configurationMetrics.add(project.provider {
|
||||
KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project, isProjectIsolationEnabled)
|
||||
KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
//init buildStatsService
|
||||
KotlinBuildStatsService.getOrCreateInstance(project)
|
||||
KotlinBuildStatsService.getOrCreateInstance(project)?.also {
|
||||
it.recordProjectsEvaluated(project)
|
||||
}
|
||||
|
||||
val fusStatisticsAvailable = fusStatisticsAvailable(project)
|
||||
val buildReportOutputs = reportingSettings(project).buildReportOutputs
|
||||
val gradle = project.gradle
|
||||
|
||||
//Workaround for known issues for Gradle 8+: https://github.com/gradle/gradle/issues/24887:
|
||||
// when this OperationCompletionListener is called services can be already closed for Gradle 8,
|
||||
// so there is a change that no VariantImplementationFactory will be found
|
||||
return project.gradle.sharedServices.registerIfAbsent(serviceName, BuildFusService::class.java) { spec ->
|
||||
if (fusStatisticsAvailable) {
|
||||
KotlinBuildStatsService.applyIfInitialised {
|
||||
it.recordProjectsEvaluated(project.gradle)
|
||||
}
|
||||
}
|
||||
return gradle.sharedServices.registerIfAbsent(serviceName, BuildFusService::class.java) { spec ->
|
||||
|
||||
spec.parameters.configurationMetrics.add(project.provider {
|
||||
KotlinBuildStatHandler.collectGeneralConfigurationTimeMetrics(project, isProjectIsolationEnabled, buildReportOutputs, pluginVersion)
|
||||
KotlinBuildStatHandler.collectGeneralConfigurationTimeMetrics(
|
||||
gradle,
|
||||
buildReportOutputs,
|
||||
pluginVersion,
|
||||
isProjectIsolationEnabled
|
||||
)
|
||||
})
|
||||
|
||||
spec.parameters.configurationMetrics.add(project.provider {
|
||||
KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project, isProjectIsolationEnabled)
|
||||
KotlinBuildStatHandler.collectProjectConfigurationTimeMetrics(project)
|
||||
})
|
||||
spec.parameters.fusStatisticsAvailable.set(fusStatisticsAvailable)
|
||||
spec.parameters.useBuildFinishFlowAction.set(GradleVersion.current().baseVersion >= GradleVersion.version("8.1"))
|
||||
}.also { buildService ->
|
||||
if (fusStatisticsAvailable) {
|
||||
when {
|
||||
GradleVersion.current().baseVersion < GradleVersion.version("8.1") ->
|
||||
BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildService)
|
||||
else -> StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult()
|
||||
}
|
||||
@Suppress("DEPRECATION")
|
||||
if (GradleVersion.current().baseVersion >= GradleVersion.version("7.4")
|
||||
|| !project.isConfigurationCacheRequested
|
||||
|| project.currentBuildId().name != "buildSrc"
|
||||
) {
|
||||
BuildEventsListenerRegistryHolder.getInstance(project).listenerRegistry.onTaskCompletion(buildService)
|
||||
}
|
||||
if (GradleVersion.current().baseVersion >= GradleVersion.version("8.1")) {
|
||||
StatisticsBuildFlowManager.getInstance(project).subscribeForBuildResult()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onFinish(event: FinishEvent?) {
|
||||
parameters.fusStatisticsAvailable.get() //force to calculate configuration metrics before build finish
|
||||
if ((event is TaskFinishEvent) && (event.result is TaskFailureResult)) {
|
||||
buildFailed = true
|
||||
if (event is TaskFinishEvent) {
|
||||
if (event.result is TaskFailureResult) {
|
||||
buildFailed = true
|
||||
}
|
||||
|
||||
val taskExecutionResult = TaskExecutionResults[event.descriptor.taskPath]
|
||||
taskExecutionResult?.also { reportTaskMetrics(it, event) }
|
||||
}
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
if (parameters.fusStatisticsAvailable.get()) {
|
||||
if (!parameters.useBuildFinishFlowAction.get()) {
|
||||
recordBuildFinished(null, buildFailed)
|
||||
}
|
||||
KotlinBuildStatsService.closeServices()
|
||||
log.kotlinDebug("Close ${this.javaClass.simpleName}")
|
||||
}
|
||||
|
||||
internal fun recordBuildFinished(action: String?, buildFailed: Boolean) {
|
||||
fusMetricsConsumer?.let { metricsConsumer ->
|
||||
KotlinBuildStatHandler.reportGlobalMetrics(metricsConsumer)
|
||||
parameters.configurationMetrics.orElse(emptyList()).get().forEach { it.addToConsumer(metricsConsumer) }
|
||||
KotlinBuildStatsService.applyIfInitialised {
|
||||
it.recordBuildFinish(action, buildFailed, metricsConsumer)
|
||||
KotlinBuildStatHandler.reportGlobalMetrics(fusMetricsConsumer)
|
||||
parameters.configurationMetrics.orElse(emptyList()).get().forEach { it.addToConsumer(fusMetricsConsumer) }
|
||||
KotlinBuildStatsService.applyIfInitialised {
|
||||
it.recordBuildFinish(action, buildFailed, fusMetricsConsumer, buildId)
|
||||
}
|
||||
KotlinBuildStatsService.closeServices()
|
||||
}
|
||||
private fun reportTaskMetrics(taskExecutionResult: TaskExecutionResult, event: TaskFinishEvent) {
|
||||
val totalTimeMs = event.result.endTime - event.result.startTime
|
||||
val buildMetrics = taskExecutionResult.buildMetrics
|
||||
fusMetricsConsumer.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
|
||||
fusMetricsConsumer.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
|
||||
fusMetricsConsumer.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
|
||||
|
||||
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
|
||||
|
||||
val linesOfCode = metricsMap[GradleBuildPerformanceMetric.ANALYZED_LINES_NUMBER]
|
||||
if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) {
|
||||
fusMetricsConsumer.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode)
|
||||
fusMetricsConsumer.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode)
|
||||
metricsMap[GradleBuildPerformanceMetric.ANALYSIS_LPS]?.also { value ->
|
||||
fusMetricsConsumer.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, value, null, linesOfCode)
|
||||
}
|
||||
metricsMap[GradleBuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value ->
|
||||
fusMetricsConsumer.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
|
||||
}
|
||||
}
|
||||
|
||||
fusMetricsConsumer.report(
|
||||
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
|
||||
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+16
-13
@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.gradle.plugin.statistics
|
||||
|
||||
import org.gradle.api.Project
|
||||
import org.gradle.api.artifacts.DependencySet
|
||||
import org.gradle.api.artifacts.ProjectDependency
|
||||
import org.gradle.api.invocation.Gradle
|
||||
import org.gradle.api.logging.Logging
|
||||
import org.jetbrains.kotlin.gradle.plugin.statistics.plugins.ObservablePlugins
|
||||
import org.jetbrains.kotlin.gradle.report.BuildReportType
|
||||
@@ -49,10 +51,10 @@ class KotlinBuildStatHandler {
|
||||
* Collect general configuration metrics
|
||||
*/
|
||||
internal fun collectGeneralConfigurationTimeMetrics(
|
||||
project: Project,
|
||||
isProjectIsolationEnabled: Boolean,
|
||||
gradle: Gradle,
|
||||
buildReportOutputs: List<BuildReportType>,
|
||||
pluginVersion: String,
|
||||
isProjectIsolationEnabled: Boolean,
|
||||
): MetricContainer {
|
||||
val configurationTimeMetrics = MetricContainer()
|
||||
|
||||
@@ -68,15 +70,17 @@ class KotlinBuildStatHandler {
|
||||
BuildReportType.TRY_NEXT_CONSOLE -> {}//ignore
|
||||
}
|
||||
}
|
||||
val gradle = project.gradle
|
||||
configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath)
|
||||
configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion)
|
||||
|
||||
//will be updated with KT-58266
|
||||
if (!isProjectIsolationEnabled) {
|
||||
gradle.taskGraph.whenReady { taskExecutionGraph ->
|
||||
val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct()
|
||||
configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install"))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead)
|
||||
|
||||
@@ -89,14 +93,9 @@ class KotlinBuildStatHandler {
|
||||
*/
|
||||
internal fun collectProjectConfigurationTimeMetrics(
|
||||
project: Project,
|
||||
isProjectIsolationEnabled: Boolean,
|
||||
): MetricContainer {
|
||||
val configurationTimeMetrics = MetricContainer()
|
||||
|
||||
if (isProjectIsolationEnabled) { //support project isolation - KT-58768
|
||||
return configurationTimeMetrics
|
||||
}
|
||||
|
||||
val statisticOverhead = measureTimeMillis {
|
||||
collectAppliedPluginsStatistics(project, configurationTimeMetrics)
|
||||
|
||||
@@ -202,8 +201,11 @@ class KotlinBuildStatHandler {
|
||||
}
|
||||
}
|
||||
|
||||
private fun reportLibrariesVersions(configurationTimeMetrics: MetricContainer, dependencies: DependencySet?) {
|
||||
dependencies?.forEach { dependency ->
|
||||
private fun reportLibrariesVersions(
|
||||
configurationTimeMetrics: MetricContainer,
|
||||
dependencies: DependencySet?,
|
||||
) {
|
||||
dependencies?.filter { it !is ProjectDependency }?.forEach { dependency ->
|
||||
when {
|
||||
dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put(
|
||||
StringMetrics.LIBRARY_SPRING_VERSION,
|
||||
@@ -272,11 +274,12 @@ class KotlinBuildStatHandler {
|
||||
sessionLogger: BuildSessionLogger,
|
||||
action: String?,
|
||||
buildFailed: Boolean,
|
||||
metrics: NonSynchronizedMetricsContainer,
|
||||
buildId: String,
|
||||
metricsContainer: NonSynchronizedMetricsContainer,
|
||||
) {
|
||||
runSafe("${KotlinBuildStatHandler::class.java}.reportBuildFinish") {
|
||||
metrics.sendToConsumer(sessionLogger)
|
||||
sessionLogger.finishBuildSession(action, buildFailed)
|
||||
metricsContainer.sendToConsumer(sessionLogger)
|
||||
sessionLogger.finishBuildSession(action, buildFailed, buildId)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+9
-6
@@ -187,9 +187,12 @@ internal abstract class KotlinBuildStatsService internal constructor() : Statist
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
open fun recordBuildFinish(action: String?, buildFailed: Boolean, metric: NonSynchronizedMetricsContainer) {}
|
||||
/**
|
||||
* Collects metrics at the end of a build
|
||||
*/
|
||||
open fun recordBuildFinish(action: String?, buildFailed: Boolean, metricsContainer: NonSynchronizedMetricsContainer, buildId: String) {}
|
||||
|
||||
open fun recordProjectsEvaluated(gradle: Gradle) {}
|
||||
open fun recordProjectsEvaluated(project: Project) {}
|
||||
}
|
||||
|
||||
internal class JMXKotlinBuildStatsService(private val mbs: MBeanServer, private val beanName: ObjectName) :
|
||||
@@ -256,12 +259,12 @@ internal abstract class AbstractKotlinBuildStatsService(
|
||||
return (gradle as? DefaultGradle)?.services?.get(BuildRequestMetaData::class.java)?.startTime
|
||||
}
|
||||
|
||||
override fun recordProjectsEvaluated(gradle: Gradle) {
|
||||
override fun recordProjectsEvaluated(project: Project) {
|
||||
runSafe("${DefaultKotlinBuildStatsService::class.java}.projectEvaluated") {
|
||||
if (!sessionLogger.isBuildSessionStarted()) {
|
||||
sessionLogger.startBuildSession(
|
||||
DaemonReuseCounter.incrementAndGetOrdinal(),
|
||||
gradleBuildStartTime(gradle)
|
||||
gradleBuildStartTime(project.gradle),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -299,7 +302,7 @@ internal class DefaultKotlinBuildStatsService internal constructor(
|
||||
report(StringMetrics.valueOf(name), value, subprojectName, weight)
|
||||
|
||||
//only one jmx bean service should report global metrics
|
||||
override fun recordBuildFinish(action: String?, buildFailed: Boolean, metrics: NonSynchronizedMetricsContainer) {
|
||||
KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, metrics)
|
||||
override fun recordBuildFinish(action: String?, buildFailed: Boolean, metricsContainer: NonSynchronizedMetricsContainer, buildId: String) {
|
||||
KotlinBuildStatHandler().reportBuildFinished(sessionLogger, action, buildFailed, buildId, metricsContainer)
|
||||
}
|
||||
}
|
||||
|
||||
+2
-33
@@ -15,7 +15,6 @@ import org.gradle.api.provider.Provider
|
||||
import org.gradle.api.services.BuildService
|
||||
import org.gradle.api.services.BuildServiceParameters
|
||||
import org.gradle.api.tasks.Internal
|
||||
import org.gradle.tooling.events.FailureResult
|
||||
import org.gradle.tooling.events.FinishEvent
|
||||
import org.gradle.tooling.events.OperationCompletionListener
|
||||
import org.gradle.tooling.events.task.TaskExecutionResult
|
||||
@@ -35,14 +34,11 @@ import org.jetbrains.kotlin.gradle.report.BuildReportsService.Companion.getStart
|
||||
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
|
||||
import org.jetbrains.kotlin.gradle.tasks.withType
|
||||
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
|
||||
import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
|
||||
import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
|
||||
import org.jetbrains.kotlin.gradle.plugin.StatisticsBuildFlowManager
|
||||
import org.jetbrains.kotlin.gradle.plugin.internal.isConfigurationCacheRequested
|
||||
import org.jetbrains.kotlin.gradle.plugin.statistics.BuildFusService
|
||||
import java.lang.management.ManagementFactory
|
||||
|
||||
internal interface UsesBuildMetricsService : Task {
|
||||
@@ -63,7 +59,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
|
||||
val projectName: Property<String>
|
||||
val kotlinVersion: Property<String>
|
||||
val buildConfigurationTags: ListProperty<StatTag>
|
||||
val fusService: Property<BuildFusService>
|
||||
}
|
||||
|
||||
private val log = Logging.getLogger(this.javaClass)
|
||||
@@ -114,30 +109,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
|
||||
val taskExecutionResult = TaskExecutionResults[taskPath]
|
||||
taskExecutionResult?.buildMetrics?.also {
|
||||
buildMetrics.addAll(it)
|
||||
|
||||
parameters.fusService.orNull?.reportFusMetrics { collector ->
|
||||
collector.report(NumericalMetrics.COMPILATION_DURATION, totalTimeMs)
|
||||
collector.report(BooleanMetrics.KOTLIN_COMPILATION_FAILED, event.result is FailureResult)
|
||||
collector.report(NumericalMetrics.COMPILATIONS_COUNT, 1)
|
||||
|
||||
val metricsMap = buildMetrics.buildPerformanceMetrics.asMap()
|
||||
|
||||
val linesOfCode = metricsMap[GradleBuildPerformanceMetric.ANALYZED_LINES_NUMBER]
|
||||
if (linesOfCode != null && linesOfCode > 0 && totalTimeMs > 0) {
|
||||
collector.report(NumericalMetrics.COMPILED_LINES_OF_CODE, linesOfCode)
|
||||
collector.report(NumericalMetrics.COMPILATION_LINES_PER_SECOND, linesOfCode * 1000 / totalTimeMs, null, linesOfCode)
|
||||
metricsMap[GradleBuildPerformanceMetric.ANALYSIS_LPS]?.also { value ->
|
||||
collector.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, value, null, linesOfCode)
|
||||
}
|
||||
metricsMap[GradleBuildPerformanceMetric.CODE_GENERATION_LPS]?.also { value ->
|
||||
collector.report(NumericalMetrics.CODE_GENERATION_LINES_PER_SECOND, value, null, linesOfCode)
|
||||
}
|
||||
}
|
||||
collector.report(
|
||||
NumericalMetrics.INCREMENTAL_COMPILATIONS_COUNT,
|
||||
if (taskExecutionResult.buildMetrics.buildAttributes.asMap().isEmpty()) 1 else 0
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val buildOperation = TaskRecord(
|
||||
@@ -190,7 +161,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
|
||||
|
||||
private fun registerIfAbsentImpl(
|
||||
project: Project,
|
||||
fusService: Provider<BuildFusService>
|
||||
): Provider<BuildMetricsService>? {
|
||||
// Return early if the service was already registered to avoid the overhead of reading the reporting settings below
|
||||
project.gradle.sharedServices.registrations.findByName(serviceName)?.let {
|
||||
@@ -224,7 +194,6 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
|
||||
it.parameters.projectDir.set(project.rootProject.layout.projectDirectory)
|
||||
//init gradle tags for build scan and http reports
|
||||
it.parameters.buildConfigurationTags.value(setupTags(project))
|
||||
it.parameters.fusService.set(fusService)
|
||||
}.also {
|
||||
subscribeForTaskEvents(project, it)
|
||||
}
|
||||
@@ -291,8 +260,8 @@ abstract class BuildMetricsService : BuildService<BuildMetricsService.Parameters
|
||||
}
|
||||
}
|
||||
|
||||
fun registerIfAbsent(project: Project, fusService: Provider<BuildFusService>) =
|
||||
registerIfAbsentImpl(project, fusService)?.also { serviceProvider ->
|
||||
fun registerIfAbsent(project: Project) =
|
||||
registerIfAbsentImpl(project)?.also { serviceProvider ->
|
||||
SingleActionPerProject.run(project, UsesBuildMetricsService::class.java.name) {
|
||||
project.tasks.withType<UsesBuildMetricsService>().configureEach { task ->
|
||||
task.buildMetricsService.value(serviceProvider).disallowChanges()
|
||||
|
||||
+1
-1
@@ -205,7 +205,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
|
||||
classLoadersCachingService,
|
||||
buildFinishedListenerService,
|
||||
buildIdService,
|
||||
buildFusService.orNull?.fusMetricsConsumer
|
||||
buildFusService.orNull?.getFusMetricsConsumer()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+32
-61
@@ -13,8 +13,9 @@ import org.jetbrains.kotlin.statistics.metrics.StringMetrics
|
||||
import org.junit.After
|
||||
import org.junit.Before
|
||||
import java.io.File
|
||||
import java.lang.IllegalStateException
|
||||
import java.nio.file.Files
|
||||
import java.util.*
|
||||
import kotlin.collections.ArrayList
|
||||
import kotlin.test.*
|
||||
|
||||
class BuildSessionLoggerTest {
|
||||
@@ -44,57 +45,32 @@ class BuildSessionLoggerTest {
|
||||
assertEquals(false, logger1.isBuildSessionStarted())
|
||||
logger1.startBuildSession(0, 0)
|
||||
|
||||
// created single folder with kotlin statistics
|
||||
assertEquals(1, statFilesCount())
|
||||
|
||||
// check that file locks work properly and do not use the same file
|
||||
logger2.startBuildSession(0, 0)
|
||||
assertEquals(2, statFilesCount())
|
||||
|
||||
assertEquals(true, logger1.isBuildSessionStarted())
|
||||
|
||||
logger1.finishBuildSession("", false)
|
||||
logger2.finishBuildSession("", false)
|
||||
logger1.finishBuildSession("", false, buildId = "1")
|
||||
// created single folder with kotlin statistics
|
||||
assertEquals(1, statFilesCount())
|
||||
|
||||
logger2.finishBuildSession("", false, buildId = "2")
|
||||
assertEquals(2, statFilesCount())
|
||||
|
||||
rootFolder.listFiles()?.first()?.listFiles()?.forEach { file ->
|
||||
assertTrue(
|
||||
file.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()),
|
||||
"Check that file name ${file.name} matches pattern ${BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN}"
|
||||
file.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN),
|
||||
"Check that file name \'${file.name}\' matches pattern \'${BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN}\'"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testSizeLimitation() {
|
||||
val maxFileSize = 10_000
|
||||
val logger = BuildSessionLogger(rootFolder, 100, maxFileSize.toLong())
|
||||
logger.startBuildSession(0, null)
|
||||
logger.finishBuildSession("", false)
|
||||
|
||||
assertEquals(1, statFilesCount())
|
||||
|
||||
val file2edit = rootFolder.listFiles()?.first()?.listFiles()?.first() ?: fail("Could not find single stat file")
|
||||
|
||||
logger.startBuildSession(0, 0)
|
||||
logger.finishBuildSession("", false)
|
||||
//new file should not be created
|
||||
assertEquals(1, statFilesCount())
|
||||
|
||||
file2edit.appendBytes(ByteArray(maxFileSize))
|
||||
|
||||
logger.startBuildSession(0, 0)
|
||||
logger.finishBuildSession("", false)
|
||||
|
||||
// a new file should be created as maximal file size
|
||||
assertEquals(2, statFilesCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testDeleteExtraFiles() {
|
||||
val maxFiles = 100
|
||||
val logger = BuildSessionLogger(rootFolder, maxFiles)
|
||||
logger.startBuildSession(0, null)
|
||||
logger.finishBuildSession("", false)
|
||||
logger.finishBuildSession("", false, buildId = "1")
|
||||
assertEquals(1, statFilesCount())
|
||||
|
||||
val statsFolder = rootFolder.listFiles()?.single() ?: fail("${rootFolder.absolutePath} was not created")
|
||||
@@ -102,12 +78,11 @@ class BuildSessionLoggerTest {
|
||||
|
||||
for (i in 1..200) {
|
||||
File(statsFolder, "$i").createNewFile()
|
||||
val formattedIndex = "%04d".format(i)
|
||||
File(statsFolder, "$formattedIndex-12-30-12-59-59-123.profile").createNewFile()
|
||||
File(statsFolder, "${UUID.randomUUID()}.profile").createNewFile()
|
||||
}
|
||||
|
||||
logger.startBuildSession(0, 0)
|
||||
logger.finishBuildSession("", false)
|
||||
logger.finishBuildSession("", false, buildId = "1")
|
||||
|
||||
assertTrue(
|
||||
statsFolder.listFiles()?.count { it.name == singleStatFile.name } == 1,
|
||||
@@ -117,48 +92,39 @@ class BuildSessionLoggerTest {
|
||||
// files not matching the pattern should not be affected
|
||||
assertEquals(
|
||||
200,
|
||||
statsFolder.listFiles()?.count { !it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()) },
|
||||
statsFolder.listFiles()?.count { !it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN) },
|
||||
"Some files which should not be affected, were removed"
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
maxFiles,
|
||||
statsFolder.listFiles()?.count { it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()) },
|
||||
statsFolder.listFiles()?.count { it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN) },
|
||||
"Some files which should not be affected, were removed"
|
||||
)
|
||||
assertEquals(
|
||||
statsFolder.listFiles()?.filter { it.name.matches(BuildSessionLogger.STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted(),
|
||||
BuildSessionLogger.listProfileFiles(statsFolder)
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testReadWriteMetrics() {
|
||||
val buildId = "test"
|
||||
val logger = BuildSessionLogger(rootFolder)
|
||||
logger.report(StringMetrics.KOTLIN_COMPILER_VERSION, "1.2.3.4-snapshot")
|
||||
|
||||
val startTime = System.currentTimeMillis() - 1001
|
||||
logger.startBuildSession(1, startTime)
|
||||
logger.finishBuildSession("Build", false)
|
||||
val reportFile = rootFolder.resolve(BuildSessionLogger.Companion.STATISTICS_FOLDER_NAME)
|
||||
.resolve(buildId + BuildSessionLogger.Companion.PROFILE_FILE_NAME_SUFFIX)
|
||||
|
||||
reportFile.createNewFile()
|
||||
reportFile.appendText("${StringMetrics.USE_FIR.name}=true\n")
|
||||
|
||||
logger.finishBuildSession("Build", false, buildId)
|
||||
assertEquals(1, statFilesCount())
|
||||
|
||||
val statFile = rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")
|
||||
statFile.appendBytes("break format of the file".toByteArray())
|
||||
statFile.appendBytes("break format of the file".toByteArray()) //this line should be filtered by MetricsContainer.readFromFile
|
||||
|
||||
logger.startBuildSession(1, startTime)
|
||||
|
||||
// the file should be locked
|
||||
try {
|
||||
MetricsContainer.readFromFile(statFile) {
|
||||
fail("No metrics should be read due to locked file")
|
||||
}
|
||||
fail("Method should have failed")
|
||||
} catch (e: IllegalStateException) {
|
||||
//all right
|
||||
}
|
||||
|
||||
|
||||
logger.finishBuildSession("", false)
|
||||
logger.finishBuildSession("", false, buildId)
|
||||
|
||||
val metrics = ArrayList<MetricsContainer>()
|
||||
MetricsContainer.readFromFile(statFile) {
|
||||
@@ -166,6 +132,11 @@ class BuildSessionLoggerTest {
|
||||
}
|
||||
|
||||
assertEquals(2, metrics.size, "Invalid number of MerticContainers was read")
|
||||
assertEquals(
|
||||
"true",
|
||||
metrics[0].getMetric(StringMetrics.USE_FIR)?.getValue()
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
"1.2.3",
|
||||
metrics[0].getMetric(StringMetrics.KOTLIN_COMPILER_VERSION)?.getValue()
|
||||
@@ -194,7 +165,7 @@ class BuildSessionLoggerTest {
|
||||
for (metric in NumericalMetrics.values()) {
|
||||
logger.report(metric, System.currentTimeMillis())
|
||||
}
|
||||
logger.finishBuildSession("Build", false)
|
||||
logger.finishBuildSession("Build", false, buildId = "test")
|
||||
|
||||
MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) {
|
||||
for (metric in StringMetrics.values()) {
|
||||
@@ -217,7 +188,7 @@ class BuildSessionLoggerTest {
|
||||
logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 10, null, 9)
|
||||
logger.report(NumericalMetrics.ANALYSIS_LINES_PER_SECOND, 100, null, 1)
|
||||
|
||||
logger.finishBuildSession("Build", false)
|
||||
logger.finishBuildSession("Build", false, buildId = "1")
|
||||
MetricsContainer.readFromFile(rootFolder.listFiles()?.single()?.listFiles()?.single() ?: fail("Could not find stat file")) {
|
||||
assertEquals(19L, it.getMetric(NumericalMetrics.ANALYSIS_LINES_PER_SECOND)?.getValue())
|
||||
}
|
||||
|
||||
+36
-76
@@ -5,85 +5,79 @@
|
||||
|
||||
package org.jetbrains.kotlin.statistics
|
||||
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
|
||||
import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger
|
||||
import org.jetbrains.kotlin.statistics.metrics.*
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.io.IOException
|
||||
import java.time.LocalDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.nio.file.Files
|
||||
|
||||
class BuildSessionLogger(
|
||||
rootPath: File,
|
||||
private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
|
||||
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
|
||||
private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
|
||||
private val forceValuesValidation: Boolean = false,
|
||||
forceValuesValidation: Boolean = false,
|
||||
) : StatisticsValuesConsumer {
|
||||
|
||||
companion object {
|
||||
const val PROFILE_FILE_NAME_SUFFIX = ".profile"
|
||||
const val STATISTICS_FOLDER_NAME = "kotlin-profile"
|
||||
const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}(.\\d+)?.profile"
|
||||
val STATISTICS_FILE_NAME_PATTERN = "[\\w-]*$PROFILE_FILE_NAME_SUFFIX".toRegex()
|
||||
|
||||
private const val DEFAULT_MAX_PROFILE_FILES = 1_000
|
||||
private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
|
||||
private const val DEFAULT_MAX_FILE_AGE = 30 * 24 * 3600 * 1000L //30 days
|
||||
|
||||
fun listProfileFiles(statisticsFolder: File): List<File>? {
|
||||
return statisticsFolder.listFiles()?.filterTo(ArrayList()) { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted()
|
||||
fun listProfileFiles(statisticsFolder: File): List<File> {
|
||||
return Files.newDirectoryStream(statisticsFolder.toPath()).use { dirStream ->
|
||||
dirStream.map { it.toFile() }
|
||||
.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN) }
|
||||
.sortedBy { it.lastModified() }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS")
|
||||
private val profileFileNameSuffix = ".profile"
|
||||
|
||||
private val statisticsFolder: File = File(
|
||||
rootPath,
|
||||
STATISTICS_FOLDER_NAME
|
||||
).also { it.mkdirs() }
|
||||
|
||||
private var buildSession: BuildSession? = null
|
||||
private var trackingFile: IRecordLogger? = null
|
||||
|
||||
private val metricsContainer = MetricsContainer(forceValuesValidation)
|
||||
|
||||
@Synchronized
|
||||
fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
|
||||
buildSession = BuildSession(buildStartedTime)
|
||||
report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart)
|
||||
|
||||
buildSession = BuildSession(buildStartedTime)
|
||||
initTrackingFile()
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun isBuildSessionStarted() = buildSession != null
|
||||
|
||||
@Synchronized
|
||||
private fun closeTrackingFile() {
|
||||
trackingFile?.let {
|
||||
metricsContainer.flush(it)
|
||||
it.close()
|
||||
trackingFile = null
|
||||
/**
|
||||
* Initializes a new build report file
|
||||
* The following contracts are implemented:
|
||||
* - each file contains metrics for one build
|
||||
* - any other process can add metrics to the file during build
|
||||
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
|
||||
*/
|
||||
private fun storeMetricsIntoFile(buildId: String) {
|
||||
try {
|
||||
statisticsFolder.mkdirs()
|
||||
val file = File(statisticsFolder, buildId + PROFILE_FILE_NAME_SUFFIX)
|
||||
|
||||
FileOutputStream(file, true).bufferedWriter().use {
|
||||
metricsContainer.flush(it)
|
||||
}
|
||||
} catch (_: IOException) {
|
||||
//ignore io exception
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes a new tracking file
|
||||
* The following contracts are implemented:
|
||||
* - number of tracking files should not be more than maxProfileFiles (the earlier file created the earlier deleted)
|
||||
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
|
||||
* - files are ordered on the basis of name (creation timestamp)
|
||||
* - if the last file has size less then maxFileSize, the next record will be append to it (new file created otherwise)
|
||||
* -
|
||||
*/
|
||||
@Synchronized
|
||||
private fun initTrackingFile() {
|
||||
closeTrackingFile()
|
||||
|
||||
private fun clearOldFiles() {
|
||||
// Get list of existing files. Try to create folder if possible, return from function if failed to create folder
|
||||
val fileCandidates = listProfileFiles(statisticsFolder) ?: if (statisticsFolder.mkdirs()) emptyList() else return
|
||||
val fileCandidates = listProfileFiles(statisticsFolder)
|
||||
|
||||
for ((index, file) in fileCandidates.withIndex()) {
|
||||
val toDelete = if (index < fileCandidates.size - maxProfileFiles)
|
||||
@@ -96,44 +90,14 @@ class BuildSessionLogger(
|
||||
file.delete()
|
||||
}
|
||||
}
|
||||
|
||||
// emergency check. What if a lot of files are locked due to some reason
|
||||
if ((listProfileFiles(statisticsFolder)?.size ?: 0) > maxProfileFiles * 2) {
|
||||
trackingFile = NullRecordLogger()
|
||||
return
|
||||
}
|
||||
|
||||
fun newFile(): File {
|
||||
val timestamp = profileFileNameFormatter.format(LocalDateTime.now())
|
||||
var result = File(statisticsFolder, timestamp + profileFileNameSuffix)
|
||||
var suffixIndex = 0
|
||||
while (result.exists()) {
|
||||
result = File(statisticsFolder, "${timestamp}.${suffixIndex++}$profileFileNameSuffix")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
val lastFile = fileCandidates.lastOrNull() ?: newFile()
|
||||
|
||||
trackingFile = try {
|
||||
if (lastFile.length() < maxFileSize) {
|
||||
FileRecordLogger(lastFile)
|
||||
} else {
|
||||
FileRecordLogger(newFile())
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
try {
|
||||
FileRecordLogger(newFile())
|
||||
} catch (e: IOException) {
|
||||
NullRecordLogger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Synchronized
|
||||
fun finishBuildSession(
|
||||
@Suppress("UNUSED_PARAMETER") action: String?,
|
||||
buildFailed: Boolean,
|
||||
buildId: String,
|
||||
) {
|
||||
try {
|
||||
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
|
||||
@@ -147,17 +111,13 @@ class BuildSessionLogger(
|
||||
report(NumericalMetrics.BUILD_FINISH_TIME, finishTime)
|
||||
report(BooleanMetrics.BUILD_FAILED, buildFailed)
|
||||
}
|
||||
buildSession = null
|
||||
} finally {
|
||||
unlockJournalFile()
|
||||
buildSession = null
|
||||
storeMetricsIntoFile(buildId)
|
||||
clearOldFiles()
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
private fun unlockJournalFile() {
|
||||
closeTrackingFile()
|
||||
}
|
||||
|
||||
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) =
|
||||
metricsContainer.report(metric, value, subprojectName, weight)
|
||||
|
||||
|
||||
-47
@@ -1,47 +0,0 @@
|
||||
/*
|
||||
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.fileloggers
|
||||
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.io.OutputStream
|
||||
import java.nio.channels.Channels
|
||||
import java.nio.channels.FileChannel
|
||||
import java.nio.channels.FileLock
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.StandardOpenOption
|
||||
|
||||
class FileRecordLogger(file: File) : IRecordLogger {
|
||||
private val channel: FileChannel =
|
||||
FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
|
||||
?: throw IOException("Could not open file $file")
|
||||
|
||||
private val outputStream: OutputStream
|
||||
private val lock: FileLock
|
||||
|
||||
init {
|
||||
lock = try {
|
||||
channel.tryLock() ?: throw IOException("Could not acquire an exclusive lock of file ${file.name}")
|
||||
} catch (e: Exception) {
|
||||
channel.close()
|
||||
// wrap in order to unify with FileOverlappingException
|
||||
throw IOException(e.message, e)
|
||||
}
|
||||
outputStream = Channels.newOutputStream(channel)
|
||||
}
|
||||
|
||||
override fun append(s: String) {
|
||||
outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING))
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
channel.use {
|
||||
outputStream.use {
|
||||
lock.release()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-4
@@ -143,8 +143,7 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta
|
||||
return true
|
||||
}
|
||||
|
||||
fun flush(trackingFile: IRecordLogger?) {
|
||||
if (trackingFile == null) return
|
||||
fun flush(writer: BufferedWriter) {
|
||||
val allMetrics = TreeMap<MetricDescriptor, IMetricContainer<out Any>>()
|
||||
synchronized(metricsLock) {
|
||||
allMetrics.putAll(numericalMetrics)
|
||||
@@ -153,10 +152,10 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta
|
||||
}
|
||||
for (entry in allMetrics.entries) {
|
||||
val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}"
|
||||
trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
|
||||
writer.appendLine("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
|
||||
}
|
||||
|
||||
trackingFile.append(BUILD_SESSION_SEPARATOR)
|
||||
writer.appendLine(BUILD_SESSION_SEPARATOR)
|
||||
|
||||
synchronized(metricsLock) {
|
||||
stringMetrics.clear()
|
||||
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
/*
|
||||
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.statistics.fileloggers
|
||||
|
||||
class NullRecordLogger : IRecordLogger {
|
||||
override fun append(s: String) {}
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user