azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use unique buildId for file name to store FUS metrics

Browse Source 
#KT-58768 In Progress
This commit is contained in:
Nataliya.Valtman
2023-08-31 13:43:03 +02:00
committed by Space Team
parent 30aad31ece
commit b19c496c44
13 changed files with 260 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/BuildSessionLogger.kt
+36 -76
View File
@@ -5,85 +5,79 @@
package org.jetbrains.kotlin.statistics
import org.jetbrains.kotlin.statistics.fileloggers.FileRecordLogger
import org.jetbrains.kotlin.statistics.fileloggers.IRecordLogger
import org.jetbrains.kotlin.statistics.fileloggers.MetricsContainer
import org.jetbrains.kotlin.statistics.fileloggers.NullRecordLogger
import org.jetbrains.kotlin.statistics.metrics.*
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.nio.file.Files
class BuildSessionLogger(
rootPath: File,
private val maxProfileFiles: Int = DEFAULT_MAX_PROFILE_FILES,
private val maxFileSize: Long = DEFAULT_MAX_PROFILE_FILE_SIZE,
private val maxFileAge: Long = DEFAULT_MAX_FILE_AGE,
private val forceValuesValidation: Boolean = false,
forceValuesValidation: Boolean = false,
) : StatisticsValuesConsumer {
companion object {
const val PROFILE_FILE_NAME_SUFFIX = ".profile"
const val STATISTICS_FOLDER_NAME = "kotlin-profile"
const val STATISTICS_FILE_NAME_PATTERN = "\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{3}(.\\d+)?.profile"
val STATISTICS_FILE_NAME_PATTERN = "[\\w-]*$PROFILE_FILE_NAME_SUFFIX".toRegex()
private const val DEFAULT_MAX_PROFILE_FILES = 1_000
private const val DEFAULT_MAX_PROFILE_FILE_SIZE = 100_000L
private const val DEFAULT_MAX_FILE_AGE = 30 * 24 * 3600 * 1000L //30 days
fun listProfileFiles(statisticsFolder: File): List<File>? {
return statisticsFolder.listFiles()?.filterTo(ArrayList()) { it.name.matches(STATISTICS_FILE_NAME_PATTERN.toRegex()) }?.sorted()
fun listProfileFiles(statisticsFolder: File): List<File> {
return Files.newDirectoryStream(statisticsFolder.toPath()).use { dirStream ->
dirStream.map { it.toFile() }
.filter { it.name.matches(STATISTICS_FILE_NAME_PATTERN) }
.sortedBy { it.lastModified() }
}
}
}
private val profileFileNameFormatter = DateTimeFormatter.ofPattern("YYYY-MM-dd-HH-mm-ss-SSS")
private val profileFileNameSuffix = ".profile"
private val statisticsFolder: File = File(
rootPath,
STATISTICS_FOLDER_NAME
).also { it.mkdirs() }
private var buildSession: BuildSession? = null
private var trackingFile: IRecordLogger? = null
private val metricsContainer = MetricsContainer(forceValuesValidation)
@Synchronized
fun startBuildSession(buildSinceDaemonStart: Long, buildStartedTime: Long?) {
buildSession = BuildSession(buildStartedTime)
report(NumericalMetrics.GRADLE_BUILD_NUMBER_IN_CURRENT_DAEMON, buildSinceDaemonStart)
buildSession = BuildSession(buildStartedTime)
initTrackingFile()
}
@Synchronized
fun isBuildSessionStarted() = buildSession != null
@Synchronized
private fun closeTrackingFile() {
trackingFile?.let {
metricsContainer.flush(it)
it.close()
trackingFile = null
/**
* Initializes a new build report file
* The following contracts are implemented:
* - each file contains metrics for one build
* - any other process can add metrics to the file during build
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
*/
private fun storeMetricsIntoFile(buildId: String) {
try {
statisticsFolder.mkdirs()
val file = File(statisticsFolder, buildId + PROFILE_FILE_NAME_SUFFIX)
FileOutputStream(file, true).bufferedWriter().use {
metricsContainer.flush(it)
}
} catch (_: IOException) {
//ignore io exception
}
}
/**
* Initializes a new tracking file
* The following contracts are implemented:
* - number of tracking files should not be more than maxProfileFiles (the earlier file created the earlier deleted)
* - files with age (current time - last modified) more than maxFileAge should be deleted (if we trust lastModified returned by FS)
* - files are ordered on the basis of name (creation timestamp)
* - if the last file has size less then maxFileSize, the next record will be append to it (new file created otherwise)
* -
*/
@Synchronized
private fun initTrackingFile() {
closeTrackingFile()
private fun clearOldFiles() {
// Get list of existing files. Try to create folder if possible, return from function if failed to create folder
val fileCandidates = listProfileFiles(statisticsFolder) ?: if (statisticsFolder.mkdirs()) emptyList() else return
val fileCandidates = listProfileFiles(statisticsFolder)
for ((index, file) in fileCandidates.withIndex()) {
val toDelete = if (index < fileCandidates.size - maxProfileFiles)
@@ -96,44 +90,14 @@ class BuildSessionLogger(
file.delete()
}
}
// emergency check. What if a lot of files are locked due to some reason
if ((listProfileFiles(statisticsFolder)?.size ?: 0) > maxProfileFiles * 2) {
trackingFile = NullRecordLogger()
return
}
fun newFile(): File {
val timestamp = profileFileNameFormatter.format(LocalDateTime.now())
var result = File(statisticsFolder, timestamp + profileFileNameSuffix)
var suffixIndex = 0
while (result.exists()) {
result = File(statisticsFolder, "${timestamp}.${suffixIndex++}$profileFileNameSuffix")
}
return result
}
val lastFile = fileCandidates.lastOrNull() ?: newFile()
trackingFile = try {
if (lastFile.length() < maxFileSize) {
FileRecordLogger(lastFile)
} else {
FileRecordLogger(newFile())
}
} catch (e: IOException) {
try {
FileRecordLogger(newFile())
} catch (e: IOException) {
NullRecordLogger()
}
}
}
@Synchronized
fun finishBuildSession(
@Suppress("UNUSED_PARAMETER") action: String?,
buildFailed: Boolean,
buildId: String,
) {
try {
// nanotime could not be used as build start time in nanotime is unknown. As result, the measured duration
@@ -147,17 +111,13 @@ class BuildSessionLogger(
report(NumericalMetrics.BUILD_FINISH_TIME, finishTime)
report(BooleanMetrics.BUILD_FAILED, buildFailed)
}
buildSession = null
} finally {
unlockJournalFile()
buildSession = null
storeMetricsIntoFile(buildId)
clearOldFiles()
}
}
@Synchronized
private fun unlockJournalFile() {
closeTrackingFile()
}
override fun report(metric: BooleanMetrics, value: Boolean, subprojectName: String?, weight: Long?) =
metricsContainer.report(metric, value, subprojectName, weight)
libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/FileRecordLogger.kt
-47
View File
@@ -1,47 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
import java.io.File
import java.io.IOException
import java.io.OutputStream
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.channels.FileLock
import java.nio.file.Paths
import java.nio.file.StandardOpenOption
class FileRecordLogger(file: File) : IRecordLogger {
private val channel: FileChannel =
FileChannel.open(Paths.get(file.toURI()), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND)
?: throw IOException("Could not open file $file")
private val outputStream: OutputStream
private val lock: FileLock
init {
lock = try {
channel.tryLock() ?: throw IOException("Could not acquire an exclusive lock of file ${file.name}")
} catch (e: Exception) {
channel.close()
// wrap in order to unify with FileOverlappingException
throw IOException(e.message, e)
}
outputStream = Channels.newOutputStream(channel)
}
override fun append(s: String) {
outputStream.write("$s\n".toByteArray(MetricsContainer.ENCODING))
}
override fun close() {
channel.use {
outputStream.use {
lock.release()
}
}
}
}
libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/MetricsContainer.kt
+3 -4
View File
@@ -143,8 +143,7 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta
return true
}
fun flush(trackingFile: IRecordLogger?) {
if (trackingFile == null) return
fun flush(writer: BufferedWriter) {
val allMetrics = TreeMap<MetricDescriptor, IMetricContainer<out Any>>()
synchronized(metricsLock) {
allMetrics.putAll(numericalMetrics)
@@ -153,10 +152,10 @@ class MetricsContainer(private val forceValuesValidation: Boolean = false) : Sta
}
for (entry in allMetrics.entries) {
val suffix = if (entry.key.projectHash == null) "" else ".${entry.key.projectHash}"
trackingFile.append("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
writer.appendLine("${entry.key.name}$suffix=${entry.value.toStringRepresentation()}")
}
trackingFile.append(BUILD_SESSION_SEPARATOR)
writer.appendLine(BUILD_SESSION_SEPARATOR)
synchronized(metricsLock) {
stringMetrics.clear()
libraries/tools/kotlin-gradle-statistics/src/main/kotlin/org/jetbrains/kotlin/statistics/fileloggers/NullRecordLogger.kt
-13
View File
@@ -1,13 +0,0 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.statistics.fileloggers
class NullRecordLogger : IRecordLogger {
override fun append(s: String) {}
override fun close() {
}
}