azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add compiler metrics to JPS build report

Browse Source 
#KT-63549: Fixed
This commit is contained in:
Nataliya.Valtman
2023-11-16 12:06:24 +01:00
committed by Space Team
parent 6cff71e7d1
commit 6bbf5b83c8
25 changed files with 616 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTestIncremental.kt
+80
View File
@@ -26,6 +26,8 @@ import org.jetbrains.kotlin.jps.build.fixtures.EnableICFixture
import org.jetbrains.kotlin.jps.model.kotlinCommonCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
import kotlin.reflect.KMutableProperty1
class KotlinJpsBuildTestIncremental : KotlinJpsBuildTest() {
@@ -43,6 +45,84 @@ class KotlinJpsBuildTestIncremental : KotlinJpsBuildTest() {
).run()
}
fun testJpsBuildReportIC() {
val reportDir = workDir.resolve("buildReport")
@Suppress("UNREACHABLE_CODE")
fun getReportFile(): File {
return Files.list(reportDir.toPath()).let {
val files = it.toArray()
val singleFile = (files.singleOrNull() as Path?).also {
it ?: fail("The directory must contain a single file, but got: $files")
}
return singleFile?.toFile()!!
}
}
fun assertFileContains(
file: File,
vararg expectedText: String,
) {
val text = file.readText()
val textNotInTheFile = expectedText.filterNot { text.contains(it) }
assert(textNotInTheFile.isEmpty()) {
"""
|$file does not contain:
|${textNotInTheFile.joinToString(separator = "\n")}
|
|actual file content:
|"$text"
|
""".trimMargin()
}
}
fun validateAndDeleteReportFile(vararg expectedText: String) {
assertTrue(reportDir.exists())
val reportFile = getReportFile()
assertFileContains(reportFile, *expectedText)
reportFile.delete()
}
val reportMetricsList = arrayOf(
"Task 'kotlinProject' finished in",
"Task info:",
"Kotlin language version: 2.0",
"Time metrics:",
"Jps iteration:",
"Compiler code analysis:",
"Compiler code generation:"
)
fun testImpl() {
assertTrue("Daemon was not enabled!", isDaemonEnabled())
doTest()
validateAndDeleteReportFile(
*reportMetricsList,
"Changed files: [${workDir.resolve("src/Foo.kt").path}, ${workDir.resolve("src/main.kt").path}]"
)
val mainKt = File(workDir, "src/main.kt")
change(mainKt.path, "fun main() {}")
buildAllModules().assertSuccessful()
validateAndDeleteReportFile(
*reportMetricsList,
"Changed files: [${workDir.resolve("src/main.kt").path}]"
)
}
withDaemon {
withSystemProperty("kotlin.build.report.file.output_dir", reportDir.path) {
testImpl()
}
}
}
fun testJpsDaemonIC() {
fun testImpl() {
assertTrue("Daemon was not enabled!", isDaemonEnabled())
jps/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/JpsCompilationResult.kt
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import java.io.Serializable
import java.rmi.RemoteException
import java.rmi.server.UnicastRemoteObject
class JpsCompilationResult : CompilationResults,
UnicastRemoteObject(
SOCKET_ANY_FREE_PORT,
LoopbackNetworkInterface.clientLoopbackSocketFactory,
LoopbackNetworkInterface.serverLoopbackSocketFactory
) {
var icLogLines: List<String> = emptyList()
val compiledFiles = ArrayList<String>()
private val buildMetricsReporter = BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>()
val buildMetrics: BuildMetrics<JpsBuildTime, JpsBuildPerformanceMetric>
get() = buildMetricsReporter.getMetrics()
private val log = JpsKotlinLogger(KotlinBuilder.LOG)
@Throws(RemoteException::class)
override fun add(compilationResultCategory: Int, value: Serializable) {
when (compilationResultCategory) {
CompilationResultCategory.IC_COMPILE_ITERATION.code -> {
@Suppress("UNCHECKED_CAST")
val compileIterationResult = value as? CompileIterationResult
if (compileIterationResult != null) {
val sourceFiles = compileIterationResult.sourceFiles
buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.IC_COMPILE_ITERATION)
compiledFiles.addAll(sourceFiles.map { it.path })
}
}
CompilationResultCategory.BUILD_REPORT_LINES.code,
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES.code -> {
@Suppress("UNCHECKED_CAST")
(value as? List<String>)?.let { icLogLines = it }
}
CompilationResultCategory.BUILD_METRICS.code -> {
(value as BuildMetricsValue).let {
when (it.key) {
CompilationPerformanceMetrics.CODE_GENERATION -> buildMetrics.buildTimes.addTimeMs(JpsBuildTime.CODE_GENERATION, it.value)
CompilationPerformanceMetrics.CODE_ANALYSIS -> buildMetrics.buildTimes.addTimeMs(JpsBuildTime.CODE_ANALYSIS, it.value)
CompilationPerformanceMetrics.COMPILER_INITIALIZATION -> buildMetrics.buildTimes.addTimeMs(JpsBuildTime.COMPILER_INITIALIZATION, it.value)
CompilationPerformanceMetrics.ANALYZED_LINES_NUMBER -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.ANALYZED_LINES_NUMBER, it.value)
CompilationPerformanceMetrics.ANALYSIS_LPS -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.ANALYSIS_LPS, it.value)
CompilationPerformanceMetrics.CODE_GENERATED_LINES_NUMBER -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.CODE_GENERATED_LINES_NUMBER, it.value)
CompilationPerformanceMetrics.CODE_GENERATION_LPS -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.CODE_GENERATION_LPS, it.value)
}
}
}
}
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/JpsKotlinCompilerRunner.kt
+31 -16
View File
@@ -31,6 +31,7 @@ import org.jetbrains.kotlin.daemon.client.CompileServiceSession
import org.jetbrains.kotlin.daemon.client.KotlinCompilerClient
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.PrintStream
@@ -91,7 +92,7 @@ class JpsKotlinCompilerRunner {
fun classesFqNamesByFiles(
environment: JpsCompilerEnvironment,
files: Set<File>
files: Set<File>,
): Set<String> = withDaemonOrFallback(
withDaemon = {
doWithDaemon(environment) { sessionId, daemon ->
@@ -113,7 +114,8 @@ class JpsKotlinCompilerRunner {
environment: JpsCompilerEnvironment,
destination: String,
classpath: Collection<String>,
sourceFiles: Collection<File>
sourceFiles: Collection<File>,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2MetadataArguments))
@@ -124,7 +126,7 @@ class JpsKotlinCompilerRunner {
arguments.destination = arguments.destination ?: destination
withCompilerSettings(compilerSettings) {
runCompiler(KotlinCompilerClass.METADATA, arguments, environment)
runCompiler(KotlinCompilerClass.METADATA, arguments, environment, buildMetricReporter)
}
}
@@ -133,12 +135,13 @@ class JpsKotlinCompilerRunner {
k2jvmArguments: K2JVMCompilerArguments,
compilerSettings: CompilerSettings,
environment: JpsCompilerEnvironment,
moduleFile: File
moduleFile: File,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2jvmArguments))
setupK2JvmArguments(moduleFile, arguments)
withCompilerSettings(compilerSettings) {
runCompiler(KotlinCompilerClass.JVM, arguments, environment)
runCompiler(KotlinCompilerClass.JVM, arguments, environment, buildMetricReporter)
}
}
@@ -152,7 +155,8 @@ class JpsKotlinCompilerRunner {
sourceMapRoots: Collection<File>,
libraries: List<String>,
friendModules: List<String>,
outputFile: File
outputFile: File,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
log.debug("K2JS: common arguments: " + ArgumentUtils.convertArgumentsToStringList(commonArguments))
log.debug("K2JS: JS arguments: " + ArgumentUtils.convertArgumentsToStringList(k2jsArguments))
@@ -168,26 +172,32 @@ class JpsKotlinCompilerRunner {
log.debug("K2JS: arguments after setup" + ArgumentUtils.convertArgumentsToStringList(arguments))
withCompilerSettings(compilerSettings) {
runCompiler(KotlinCompilerClass.JS, arguments, environment)
runCompiler(KotlinCompilerClass.JS, arguments, environment, buildMetricReporter)
}
}
private fun compileWithDaemonOrFallback(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
log.debug("Using kotlin-home = " + environment.kotlinPaths.homePath)
withDaemonOrFallback(
withDaemon = { compileWithDaemon(compilerClassName, compilerArgs, environment) },
withDaemon = { compileWithDaemon(compilerClassName, compilerArgs, environment, buildMetricReporter) },
fallback = { fallbackCompileStrategy(compilerArgs, compilerClassName, environment) }
)
}
private fun runCompiler(compilerClassName: String, compilerArgs: CommonCompilerArguments, environment: JpsCompilerEnvironment) {
private fun runCompiler(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
try {
compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment)
compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment, buildMetricReporter)
} catch (e: Throwable) {
MessageCollectorUtil.reportException(environment.messageCollector, e)
reportInternalCompilerError(environment.messageCollector)
@@ -197,7 +207,8 @@ class JpsKotlinCompilerRunner {
private fun compileWithDaemon(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?,
): Int? {
val targetPlatform = when (compilerClassName) {
KotlinCompilerClass.JVM -> CompileService.TargetPlatform.JVM
@@ -214,6 +225,8 @@ class JpsKotlinCompilerRunner {
reportSeverity(verbose),
requestedCompilationResults = emptyArray()
)
val compilationResult = JpsCompilationResult()
return doWithDaemon(environment) { sessionId, daemon ->
environment.withProgressReporter { progress ->
progress.compilationStarted()
@@ -222,8 +235,10 @@ class JpsKotlinCompilerRunner {
withAdditionalCompilerArgs(compilerArgs),
options,
JpsCompilerServicesFacadeImpl(environment),
null
compilationResult
)
}.also {
buildMetricReporter?.addCompilerMetrics(compilationResult)
}
}
}
@@ -237,7 +252,7 @@ class JpsKotlinCompilerRunner {
private fun <T> doWithDaemon(
environment: JpsCompilerEnvironment,
fn: (sessionId: Int, daemon: CompileService) -> CompileService.CallResult<T>
fn: (sessionId: Int, daemon: CompileService) -> CompileService.CallResult<T>,
): T? {
log.debug("Try to connect to daemon")
val connection = getDaemonConnection(environment)
@@ -280,7 +295,7 @@ class JpsKotlinCompilerRunner {
private fun fallbackCompileStrategy(
compilerArgs: CommonCompilerArguments,
compilerClassName: String,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
) {
if ("true" == System.getProperty("kotlin.jps.tests") && "true" == System.getProperty(FAIL_ON_FALLBACK_PROPERTY)) {
error("Cannot compile with Daemon, see logs bellow. Fallback strategy is disabled in tests")
@@ -325,7 +340,7 @@ class JpsKotlinCompilerRunner {
_commonSources: Collection<File>,
_libraries: List<String>,
_friendModules: List<String>,
settings: K2JSCompilerArguments
settings: K2JSCompilerArguments,
) {
with(settings) {
noStdlib = true
jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+29 -7
View File
@@ -24,6 +24,7 @@ import org.jetbrains.kotlin.build.report.ICReporter.ReportSeverity
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.debug
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
@@ -40,7 +41,9 @@ import org.jetbrains.kotlin.jps.KotlinJpsBundle
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
import org.jetbrains.kotlin.jps.model.kotlinKind
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.jps.statistic.JpsStatisticsReportService
import org.jetbrains.kotlin.jps.statistic.statisticsReportServiceKey
import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget
import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
@@ -66,7 +69,6 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
System.getProperty("kotlin.jps.classesToLoadByParent")?.split(',')?.map { it.trim() } ?: emptyList()
private val classPrefixesToLoadByParentFromRegistry =
System.getProperty("kotlin.jps.classPrefixesToLoadByParent")?.split(',')?.map { it.trim() } ?: emptyList()
private val reportService = JpsStatisticsReportService.create()
val classesToLoadByParent: ClassCondition
get() = ClassCondition { className ->
@@ -100,6 +102,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
override fun buildStarted(context: CompileContext) {
logSettings(context)
val reportService = JpsStatisticsReportService.create()
context.putUserData(statisticsReportServiceKey, reportService)
reportService.buildStarted(context)
}
@@ -161,6 +165,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
override fun buildFinished(context: CompileContext) {
ensureKotlinContextDisposed(context)
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.buildFinish(context)
}
@@ -295,7 +300,19 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
outputConsumer: OutputConsumer
): ExitCode {
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.moduleBuildStarted(chunk)
return doBuild(context, chunk, dirtyFilesHolder, outputConsumer).also { result ->
reportService.moduleBuildFinished(chunk, context, result)
}
}
private fun doBuild(
context: CompileContext,
chunk: ModuleChunk,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
outputConsumer: OutputConsumer
): ExitCode {
if (chunk.isDummy(context))
return NOTHING_DONE
@@ -320,6 +337,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val fsOperations = FSOperationsHelper(context, chunk, kotlinDirtyFilesHolder, LOG)
try {
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.reportDirtyFiles(kotlinDirtyFilesHolder)
return reportService.reportMetrics(chunk, JpsBuildTime.JPS_ITERATION) {
val proposedExitCode =
doBuild(chunk, kotlinTarget, context, kotlinDirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
@@ -342,8 +361,6 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
LOG.info("Caught exception: $e")
MessageCollectorUtil.reportException(messageCollector, e)
return ABORT
} finally {
reportService.moduleBuildFinished(chunk, context)
}
}
@@ -354,7 +371,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder,
messageCollector: MessageCollectorAdapter,
outputConsumer: OutputConsumer,
fsOperations: FSOperationsHelper
fsOperations: FSOperationsHelper,
): ExitCode {
// Workaround for Android Studio
if (representativeTarget is KotlinJvmModuleBuildTarget && !JavaBuilder.IS_ENABLED[context, true]) {
@@ -448,6 +465,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
LOG.debug("Compiling files: ${kotlinDirtyFilesHolder.allDirtyFiles}")
}
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.reportCompilerArguments(chunk, kotlinChunk)
val start = System.nanoTime()
val outputItemCollector = doCompileModuleChunk(
kotlinChunk,
@@ -457,7 +476,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
kotlinDirtyFilesHolder,
fsOperations,
environment,
incrementalCaches
incrementalCaches,
reportService.getMetricReporter(chunk)
)
statisticsLogger.registerStatistic(chunk, System.nanoTime() - start)
@@ -591,13 +611,15 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
fsOperations: FSOperationsHelper,
environment: JpsCompilerEnvironment,
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
buildMetricReporter: JpsBuilderMetricReporter?,
): OutputItemsCollector? {
kotlinChunk.targets.forEach {
it.nextRound(context)
}
if (representativeTarget.isIncrementalCompilationEnabled) {
buildMetricReporter?.addTag(StatTag.INCREMENTAL)
for (target in kotlinChunk.targets) {
val cache = incrementalCaches[target]
val jpsTarget = target.jpsModuleBuildTarget
@@ -613,7 +635,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
}
}
val isDoneSomething = representativeTarget.compileModuleChunk(commonArguments, dirtyFilesHolder, environment)
val isDoneSomething = representativeTarget.compileModuleChunk(commonArguments, dirtyFilesHolder, environment, buildMetricReporter)
return if (isDoneSomething) environment.outputItemsCollector else null
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsBuilderMetricReporter.kt
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.statistic
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.compilerRunner.JpsCompilationResult
interface JpsBuilderMetricReporter : BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> {
fun flush(context: CompileContext): JpsCompileStatisticsData
fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext, exitCode: String)
fun addChangedFiles(files: List<String>)
fun addCompilerArguments(arguments: List<String>)
fun setKotlinLanguageVersion(languageVersion: String?)
fun addTag(tag: StatTag)
fun addCompilerMetrics(jpsCompilationResult: JpsCompilationResult)
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsBuilderMetricReporterImpl.kt
+104
View File
@@ -0,0 +1,104 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.statistic
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.BuildDataType
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.compilerRunner.JpsCompilationResult
import java.net.InetAddress
import java.util.*
import kotlin.collections.ArrayList
class JpsBuilderMetricReporterImpl(
chunk: ModuleChunk,
private val reporter: BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>,
private val label: String? = null,
private val kotlinVersion: String = "kotlin_version"
) :
JpsBuilderMetricReporter, BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> by reporter {
companion object {
private val hostName: String? = try {
InetAddress.getLocalHost().hostName
} catch (_: Exception) {
//do nothing
null
}
private val uuid = UUID.randomUUID()
}
private val startTime = System.currentTimeMillis()
private var finishTime: Long = 0L
private val tags = HashSet<StatTag>()
private val changedFiles = ArrayList<String>()
private val compilerArguments = ArrayList<String>()
private val moduleString = chunk.name
private var exitCode = "Unknown"
private var kotlinLanguageVersion: String? = null
override fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext, exitCode: String) {
finishTime = System.currentTimeMillis()
this.exitCode = exitCode
}
override fun addChangedFiles(files: List<String>) {
changedFiles.addAll(files)
}
override fun addCompilerArguments(arguments: List<String>) {
compilerArguments.addAll(arguments)
}
override fun setKotlinLanguageVersion(languageVersion: String?) {
kotlinLanguageVersion = languageVersion
}
override fun addTag(tag: StatTag) {
tags.add(tag)
}
override fun addCompilerMetrics(jpsCompilationResult: JpsCompilationResult) {
reporter.addMetrics(jpsCompilationResult.buildMetrics)
}
override fun flush(context: CompileContext): JpsCompileStatisticsData {
val buildMetrics = reporter.getMetrics()
return JpsCompileStatisticsData(
projectName = context.projectDescriptor.project.name,
label = label,
taskName = moduleString,
taskResult = exitCode,
startTimeMs = startTime,
durationMs = finishTime - startTime,
tags = tags,
buildUuid = uuid.toString(),
changes = changedFiles,
kotlinVersion = kotlinVersion,
hostName = hostName,
finishTime = finishTime,
buildTimesMetrics = buildMetrics.buildTimes.asMapMs(),
performanceMetrics = buildMetrics.buildPerformanceMetrics.asMap(),
compilerArguments = compilerArguments,
nonIncrementalAttributes = emptySet(),
type = BuildDataType.JPS_DATA.name,
fromKotlinPlugin = true,
compiledSources = emptyList(),
skipMessage = null,
icLogLines = emptyList(),
gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(),
gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(),
kotlinLanguageVersion = kotlinLanguageVersion
)
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsFileReportService.kt
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.statistic
import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.CompileStatisticsData
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.compilerRunner.JpsKotlinLogger
import java.io.File
internal class JpsFileReportService(
buildReportDir: File,
projectName: String,
printMetrics: Boolean,
logger: JpsKotlinLogger,
) : FileReportService<JpsBuildTime, JpsBuildPerformanceMetric>(buildReportDir, projectName, printMetrics, logger) {
override fun printCustomTaskMetrics(statisticsData: CompileStatisticsData<JpsBuildTime, JpsBuildPerformanceMetric>) {
p.print("Changed files: ${statisticsData.getChanges().sorted()}")
p.print("Execution result: ${statisticsData.getTaskResult()}")
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt
+44 -81
View File
@@ -8,92 +8,29 @@ package org.jetbrains.kotlin.jps.statistic
import com.intellij.openapi.diagnostic.Logger
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.GlobalContextKey
import org.jetbrains.jps.incremental.ModuleLevelBuilder
import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode
import org.jetbrains.kotlin.build.report.FileReportSettings
import org.jetbrains.kotlin.build.report.HttpReportSettings
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.build.report.statistics.BuildDataType
import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters
import org.jetbrains.kotlin.build.report.statistics.HttpReportService
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.build.report.statistics.*
import org.jetbrains.kotlin.compilerRunner.JpsKotlinLogger
import org.jetbrains.kotlin.jps.build.KotlinChunk
import org.jetbrains.kotlin.jps.build.KotlinCompileContext
import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import java.io.File
import java.net.InetAddress
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
interface JpsBuilderMetricReporter : BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> {
fun flush(context: CompileContext): JpsCompileStatisticsData
fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext)
}
private const val jpsBuildTaskName = "JPS build"
class JpsBuilderMetricReporterImpl(
chunk: ModuleChunk,
private val reporter: BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>,
private val label: String? = null,
private val kotlinVersion: String = "kotlin_version",
) :
JpsBuilderMetricReporter, BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> by reporter {
companion object {
private val hostName: String? = try {
InetAddress.getLocalHost().hostName
} catch (_: Exception) {
//do nothing
null
}
private val uuid = UUID.randomUUID()
}
private val startTime = System.currentTimeMillis()
private var finishTime: Long = 0L
private val tags = HashSet<StatTag>()
private val moduleString = chunk.name
override fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext) {
finishTime = System.currentTimeMillis()
}
override fun flush(context: CompileContext): JpsCompileStatisticsData {
val buildMetrics = reporter.getMetrics()
return JpsCompileStatisticsData(
projectName = context.projectDescriptor.project.name,
label = label,
taskName = moduleString,
taskResult = "Unknown",//TODO will be updated in KT-58026
startTimeMs = startTime,
durationMs = finishTime - startTime,
tags = tags,
buildUuid = uuid.toString(),
changes = emptyList(), //TODO will be updated in KT-58026
kotlinVersion = kotlinVersion,
hostName = hostName,
finishTime = finishTime,
buildTimesMetrics = buildMetrics.buildTimes.asMapMs(),
performanceMetrics = buildMetrics.buildPerformanceMetrics.asMap(),
compilerArguments = emptyList(), //TODO will be updated in KT-58026
nonIncrementalAttributes = emptySet(),
type = BuildDataType.JPS_DATA.name,
fromKotlinPlugin = true,
compiledSources = emptyList(),
skipMessage = null,
icLogLines = emptyList(),
gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(),
gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(),
kotlinLanguageVersion = null
)
}
}
internal val statisticsReportServiceKey = GlobalContextKey<JpsStatisticsReportService>("jpsStatistics")
sealed class JpsStatisticsReportService {
companion object {
fun create(): JpsStatisticsReportService {
internal fun create(): JpsStatisticsReportService {
val fileReportSettings = initFileReportSettings()
val httpReportSettings = initHttpReportSettings()
@@ -102,9 +39,11 @@ sealed class JpsStatisticsReportService {
} else {
JpsStatisticsReportServiceImpl(fileReportSettings, httpReportSettings)
}
}
internal fun getFromContext(context: CompileContext): JpsStatisticsReportService =
context.getUserData(statisticsReportServiceKey) ?: DummyJpsStatisticsReportService
private fun initFileReportSettings(): FileReportSettings? {
return System.getProperty("kotlin.build.report.file.output_dir")?.let { FileReportSettings(File(it)) }
}
@@ -123,8 +62,12 @@ sealed class JpsStatisticsReportService {
abstract fun buildStarted(context: CompileContext)
abstract fun buildFinish(context: CompileContext)
abstract fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext)
abstract fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext, exitCode: ModuleLevelBuilder.ExitCode)
abstract fun moduleBuildStarted(chunk: ModuleChunk)
abstract fun reportDirtyFiles(kotlinDirtySourceFilesHolder: KotlinDirtySourceFilesHolder)
abstract fun reportCompilerArguments(chunk: ModuleChunk, kotlinChunk: KotlinChunk)
abstract fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter?
}
@@ -135,9 +78,13 @@ object DummyJpsStatisticsReportService : JpsStatisticsReportService() {
override fun buildStarted(context: CompileContext) {}
override fun buildFinish(context: CompileContext) {}
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext) {}
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext, exitCode: ExitCode) {}
override fun moduleBuildStarted(chunk: ModuleChunk) {}
override fun reportDirtyFiles(kotlinDirtySourceFilesHolder: KotlinDirtySourceFilesHolder) {}
override fun reportCompilerArguments(chunk: ModuleChunk, kotlinChunk: KotlinChunk) {}
override fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter? = null
}
class JpsStatisticsReportServiceImpl(
@@ -161,7 +108,7 @@ class JpsStatisticsReportServiceImpl(
log.debug("JpsStatisticsReportService: Build started for $moduleName module")
}
private fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter? {
override fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter? {
val moduleName = chunk.name
return getMetricReporter(moduleName)
}
@@ -176,7 +123,7 @@ class JpsStatisticsReportServiceImpl(
return metricReporter
}
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext) {
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext, exitCode: ExitCode) {
val moduleName = chunk.name
val metrics = buildMetrics.remove(moduleName)
if (metrics == null) {
@@ -184,7 +131,7 @@ class JpsStatisticsReportServiceImpl(
return
}
log.debug("JpsStatisticsReportService: Build started for $moduleName module")
metrics.buildFinish(chunk, context)
metrics.buildFinish(chunk, context, exitCode.name)
finishedModuleBuildMetrics.add(metrics)
}
@@ -192,9 +139,11 @@ class JpsStatisticsReportServiceImpl(
val compileStatisticsData = finishedModuleBuildMetrics.map { it.flush(context) }
httpService?.sendData(compileStatisticsData, loggerAdapter)
fileReportSettings?.also {
FileReportService.reportBuildStatInFile(
it.buildReportDir, context.projectDescriptor.project.name, true, compileStatisticsData,
BuildStartParameters(tasks = listOf(jpsBuildTaskName)), emptyList(), loggerAdapter
JpsFileReportService(
it.buildReportDir, context.projectDescriptor.project.name, true, loggerAdapter
).process(
compileStatisticsData,
BuildStartParameters(tasks = listOf(jpsBuildTaskName)), emptyList(),
)
}
}
@@ -203,6 +152,20 @@ class JpsStatisticsReportServiceImpl(
return getMetricReporter(chunk)?.measure(metric, action) ?: action.invoke()
}
override fun reportDirtyFiles(kotlinDirtySourceFilesHolder: KotlinDirtySourceFilesHolder) {
getMetricReporter(kotlinDirtySourceFilesHolder.chunk)?.let {
it.addChangedFiles(kotlinDirtySourceFilesHolder.allRemovedFilesFiles.map { it.path })
it.addChangedFiles(kotlinDirtySourceFilesHolder.allDirtyFiles.map { it.path })
}
}
override fun reportCompilerArguments(chunk: ModuleChunk, kotlinChunk: KotlinChunk) {
getMetricReporter(chunk)?.let {
it.addCompilerArguments(kotlinChunk.compilerArguments.freeArgs)
it.setKotlinLanguageVersion(kotlinChunk.compilerArguments.languageVersion)
}
}
override fun buildStarted(context: CompileContext) {
loggerAdapter.info("Build started for $context with enabled build metric reports.")
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinCommonModuleBuildTarget.kt
+5 -2
View File
@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.build.ModuleBuildTarget
import org.jetbrains.kotlin.jps.model.k2MetadataCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
private const val COMMON_BUILD_META_INFO_FILE_NAME = "common-build-meta-info.txt"
@@ -47,7 +48,8 @@ class KotlinCommonModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModu
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
require(chunk.representativeTarget == this)
@@ -60,7 +62,8 @@ class KotlinCommonModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModu
environment,
destination,
dependenciesOutputDirs + libraryFiles,
sourceFiles // incremental K2MetadataCompiler not supported yet
sourceFiles, // incremental K2MetadataCompiler not supported yet
buildMetricReporter
)
return true
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt
+5 -2
View File
@@ -35,6 +35,7 @@ import org.jetbrains.kotlin.jps.model.k2JsCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
import org.jetbrains.kotlin.jps.model.testOutputFilePath
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.utils.JsLibraryUtils
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils.JS_EXT
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils.META_JS_SUFFIX
@@ -92,7 +93,8 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
require(chunk.representativeTarget == this)
@@ -116,7 +118,8 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
sourceMapRoots,
libraries,
friendBuildTargetsMetaFiles,
outputFile
outputFile,
buildMetricReporter
)
return true
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt
+5 -2
View File
@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache
import org.jetbrains.kotlin.jps.model.k2JvmCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.modules.KotlinModuleXmlBuilder
@@ -99,7 +100,8 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
require(chunk.representativeTarget == this)
@@ -144,7 +146,8 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB
module.k2JvmCompilerArguments,
module.kotlinCompilerSettings,
environment,
moduleFile
moduleFile,
buildMetricReporter
)
} finally {
if (System.getProperty(DELETE_MODULE_FILE_PROPERTY) != "false") {
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt
+3 -1
View File
@@ -31,6 +31,7 @@ import org.jetbrains.kotlin.jps.incremental.loadDiff
import org.jetbrains.kotlin.jps.incremental.localCacheVersionManager
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
import org.jetbrains.kotlin.jps.model.testOutputFilePath
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.progress.CompilationCanceledException
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
@@ -202,7 +203,8 @@ abstract class KotlinModuleBuildTarget<BuildMetaInfoType : BuildMetaInfo> intern
abstract fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean
open fun registerOutputItems(outputConsumer: ModuleLevelBuilder.OutputConsumer, outputItems: List<GeneratedFile>) {
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinUnsupportedModuleBuildTarget.kt
+3 -1
View File
@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.jps.build.KotlinCompileContext
import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.model.platform
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.platform.idePlatformKind
class KotlinUnsupportedModuleBuildTarget(
@@ -40,7 +41,8 @@ class KotlinUnsupportedModuleBuildTarget(
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
shouldNotBeCalled()
}
jps/jps-plugin/testData/general/JpsBuildReportIC/kotlinProject.iml
Vendored
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="IDEA_JDK" jdkType="JavaSDK" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="KotlinRuntime" level="project" />
</component>
</module>
jps/jps-plugin/testData/general/JpsBuildReportIC/kotlinProject.ipr
Vendored
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<option name="DEFAULT_COMPILER" value="Javac" />
</component>
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/kotlinProject.iml" filepath="$PROJECT_DIR$/kotlinProject.iml" />
</modules>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" assert-keyword="true" jdk-15="true" project-jdk-name="IDEA_JDK" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
jps/jps-plugin/testData/general/JpsBuildReportIC/src/Foo.kt
Vendored
+1
View File
@@ -0,0 +1 @@
class Foo()
jps/jps-plugin/testData/general/JpsBuildReportIC/src/main.kt
Vendored
+3
View File
@@ -0,0 +1,3 @@
fun main() {
Foo()
}