azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add compiler metrics to JPS build report

Browse Source 
#KT-63549: Fixed
This commit is contained in:
Nataliya.Valtman
2023-11-16 12:06:24 +01:00
committed by Space Team
parent 6cff71e7d1
commit 6bbf5b83c8
25 changed files with 616 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt
+1
View File
@@ -26,6 +26,7 @@ enum class JpsBuildPerformanceMetric(
DAEMON_GC_COUNT(readableString = "Count of GC", type = ValueType.NUMBER),
COMPILE_ITERATION(parent = null, "Total compiler iteration", type = ValueType.NUMBER),
IC_COMPILE_ITERATION(parent = COMPILE_ITERATION, "Total kotlin compiler iteration", type = ValueType.NUMBER),
ANALYZED_LINES_NUMBER(parent = COMPILE_ITERATION, "Number of lines analyzed", type = ValueType.NUMBER),
CODE_GENERATED_LINES_NUMBER(parent = COMPILE_ITERATION, "Number of lines for code generation", type = ValueType.NUMBER),
ANALYSIS_LPS(parent = COMPILE_ITERATION, "Analysis lines per second", type = ValueType.NUMBER),
compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt
+7 -1
View File
@@ -20,9 +20,15 @@ interface BuildTime : Serializable {
}
@Suppress("Reformat")
enum class JpsBuildTime(private val parent: JpsBuildTime? = null, private val readableString: String) : BuildTime {
JPS_ITERATION(readableString = "Jps iteration")
JPS_ITERATION(readableString = "Jps iteration"),
COMPILATION_ROUND(JPS_ITERATION, "Sources compilation round"),
COMPILER_PERFORMANCE(COMPILATION_ROUND, readableString = "Compiler time"),
COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"),
CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analysis"),
CODE_GENERATION(COMPILER_PERFORMANCE, "Compiler code generation"),
;
override fun getReadableString(): String = readableString
compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/FileReportService.kt
+15 -23
View File
@@ -15,34 +15,21 @@ import java.io.Serializable
import java.text.SimpleDateFormat
import java.util.*
class FileReportService<B : BuildTime, P : BuildPerformanceMetric>(
private val outputFile: File,
open class FileReportService<B : BuildTime, P : BuildPerformanceMetric>(
buildReportDir: File,
projectName: String,
private val printMetrics: Boolean,
private val logger: KotlinLogger,
) : Serializable {
companion object {
private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC") }
fun <B : BuildTime, P : BuildPerformanceMetric> reportBuildStatInFile(
buildReportDir: File,
projectName: String,
includeMetricsInReport: Boolean,
buildData: List<CompileStatisticsData<B, P>>,
startParameters: BuildStartParameters,
failureMessages: List<String>,
logger: KotlinLogger,
) {
val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time)
val reportFile = buildReportDir.resolve("$projectName-build-$ts.txt")
FileReportService<B, P>(
outputFile = reportFile,
printMetrics = includeMetricsInReport,
logger = logger
).process(buildData, startParameters, failureMessages)
}
}
private val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time)
private val outputFile = buildReportDir.resolve("$projectName-build-$ts.txt")
private lateinit var p: Printer
protected lateinit var p: Printer
open fun printCustomTaskMetrics(statisticsData: CompileStatisticsData<B, P>) {}
fun process(
statisticsData: List<CompileStatisticsData<B, P>>,
@@ -263,14 +250,18 @@ class FileReportService<B : BuildTime, P : BuildPerformanceMetric>(
p.println()
}
private fun printTasksLog(statisticsData: List<CompileStatisticsData<B, P>>) {
private fun printTasksLog(
statisticsData: List<CompileStatisticsData<B, P>>,
) {
for (task in statisticsData.sortedWith(compareBy({ -it.getDurationMs() }, { it.getStartTimeMs() }))) {
printTaskLog(task)
p.println()
}
}
private fun <B : BuildTime, P : BuildPerformanceMetric> printTaskLog(statisticsData: CompileStatisticsData<B, P>) {
private fun printTaskLog(
statisticsData: CompileStatisticsData<B, P>,
) {
val skipMessage = statisticsData.getSkipMessage()
if (skipMessage != null) {
p.println("Task '${statisticsData.getTaskName()}' was skipped: $skipMessage")
@@ -295,6 +286,7 @@ class FileReportService<B : BuildTime, P : BuildPerformanceMetric>(
statisticsData.getBuildTimesMetrics(), statisticsData.getPerformanceMetrics(), statisticsData.getNonIncrementalAttributes(),
statisticsData.getGcTimeMetrics(), statisticsData.getGcCountMetrics()
)
printCustomTaskMetrics(statisticsData)
}
}
}
compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/Printer.kt
+1 -1
View File
@@ -9,7 +9,7 @@ import java.io.IOException
private val LINE_SEPARATOR = System.getProperty("line.separator")
internal class Printer(
class Printer(
private val out: Appendable,
private val indentUnit: String = " ",
private var indent: String = ""
compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt
+17
View File
@@ -31,3 +31,20 @@ enum class CompilationResultCategory(val code: Int) {
VERBOSE_BUILD_REPORT_LINES(2),
BUILD_METRICS(3)
}
data class BuildMetricsValue(
val key: CompilationPerformanceMetrics,
val value: Long
): Serializable
enum class CompilationPerformanceMetrics {
COMPILER_INITIALIZATION,
CODE_ANALYSIS,
ANALYZED_LINES_NUMBER,
ANALYSIS_LPS,
CODE_GENERATION,
CODE_GENERATED_LINES_NUMBER,
CODE_GENERATION_LPS,
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt
+85 -33
View File
@@ -27,9 +27,7 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
import org.jetbrains.kotlin.build.report.metrics.endMeasureGc
import org.jetbrains.kotlin.build.report.metrics.startMeasureGc
import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.*
import org.jetbrains.kotlin.cli.common.arguments.*
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -106,7 +104,7 @@ abstract class CompileServiceImplBase(
val daemonOptions: DaemonOptions,
val compilerId: CompilerId,
val port: Int,
val timer: Timer
val timer: Timer,
) {
protected val log by lazy { Logger.getLogger("compiler") }
@@ -118,7 +116,7 @@ abstract class CompileServiceImplBase(
protected class ClientOrSessionProxy<out T : Any>(
val aliveFlagPath: String?,
val data: T? = null,
private var disposable: Disposable? = null
private var disposable: Disposable? = null,
) {
val isAlive: Boolean
get() = aliveFlagPath?.let { File(it).exists() } ?: true // assuming that if no file was given, the client is alive
@@ -211,7 +209,7 @@ abstract class CompileServiceImplBase(
private inline fun <T> Iterable<T>.cleanMatching(
lock: ReentrantReadWriteLock,
crossinline pred: (T) -> Boolean,
crossinline clean: (T) -> Unit
crossinline clean: (T) -> Unit,
): Boolean {
var anyDead = false
lock.read {
@@ -288,6 +286,48 @@ abstract class CompileServiceImplBase(
}
}
protected fun getPerformanceMetrics(compiler: CLICompiler<CommonCompilerArguments>): List<BuildMetricsValue> {
val performanceMetrics = ArrayList<BuildMetricsValue>()
compiler.defaultPerformanceManager.getMeasurementResults().forEach {
when (it) {
is CompilerInitializationMeasurement -> {
performanceMetrics.add(BuildMetricsValue(CompilationPerformanceMetrics.COMPILER_INITIALIZATION, it.milliseconds))
}
is CodeAnalysisMeasurement -> {
performanceMetrics.add(BuildMetricsValue(CompilationPerformanceMetrics.CODE_ANALYSIS, it.milliseconds))
it.lines?.apply {
performanceMetrics.add(BuildMetricsValue(CompilationPerformanceMetrics.ANALYZED_LINES_NUMBER, this.toLong()))
if (it.milliseconds > 0) {
performanceMetrics.add(
BuildMetricsValue(
CompilationPerformanceMetrics.ANALYSIS_LPS,
this * 1000 / it.milliseconds
)
)
}
}
}
is CodeGenerationMeasurement -> {
performanceMetrics.add(
BuildMetricsValue(CompilationPerformanceMetrics.CODE_GENERATION, it.milliseconds)
)
it.lines?.apply {
performanceMetrics.add(BuildMetricsValue(CompilationPerformanceMetrics.CODE_GENERATED_LINES_NUMBER, this.toLong()))
if (it.milliseconds > 0) {
performanceMetrics.add(
BuildMetricsValue(
CompilationPerformanceMetrics.CODE_GENERATION_LPS,
this * 1000 / it.milliseconds
)
)
}
}
}
}
}
return performanceMetrics
}
protected inline fun <ServicesFacadeT, JpsServicesFacadeT, CompilationResultsT> compileImpl(
sessionId: Int,
compilerArguments: Array<out String>,
@@ -298,7 +338,7 @@ abstract class CompileServiceImplBase(
createMessageCollector: (ServicesFacadeT, CompilationOptions) -> MessageCollector,
createReporter: (ServicesFacadeT, CompilationOptions) -> DaemonMessageReporter,
createServices: (JpsServicesFacadeT, EventManager, Profiler) -> Services,
getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>
getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>,
) = kotlin.run {
val messageCollector = createMessageCollector(servicesFacade, compilationOptions)
val daemonReporter = createReporter(servicesFacade, compilationOptions)
@@ -326,7 +366,16 @@ abstract class CompileServiceImplBase(
withIncrementalCompilation(k2PlatformArgs, enabled = servicesFacade.hasIncrementalCaches()) {
doCompile(sessionId, daemonReporter, tracer = null) { eventManger, profiler ->
val services = createServices(servicesFacade, eventManger, profiler)
compiler.exec(messageCollector, services, k2PlatformArgs)
val exitCode = compiler.exec(messageCollector, services, k2PlatformArgs)
compilationResults.also {
val compilationResult = it as CompilationResults
getPerformanceMetrics(compiler).forEach {
compilationResult.add(CompilationResultCategory.BUILD_METRICS.code, it)
}
}
exitCode
}
}
}
@@ -391,7 +440,7 @@ abstract class CompileServiceImplBase(
sessionId: Int,
daemonMessageReporter: DaemonMessageReporter,
tracer: RemoteOperationsTracer?,
body: (EventManager, Profiler) -> ExitCode
body: (EventManager, Profiler) -> ExitCode,
): CompileService.CallResult<Int> = run {
log.fine("alive!")
withValidClientOrSessionProxy(sessionId) {
@@ -417,7 +466,7 @@ abstract class CompileServiceImplBase(
protected inline fun <R> checkedCompile(
daemonMessageReporter: DaemonMessageReporter,
rpcProfiler: Profiler,
body: () -> R
body: () -> R,
): R {
try {
val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler()
@@ -432,9 +481,11 @@ abstract class CompileServiceImplBase(
val pc = profiler.getTotalCounters()
val rpc = rpcProfiler.getTotalCounters()
"PERF: Compile on daemon: ${pc.time.ms()} ms; thread: user ${pc.threadUserTime.ms()} ms, sys ${(pc.threadTime - pc.threadUserTime).ms()} ms; rpc: ${rpc.count} calls, ${rpc.time.ms()} ms, thread ${rpc.threadTime.ms()} ms; memory: ${endMem.kb()} kb (${"%+d".format(
pc.memory.kb()
)} kb)".let {
"PERF: Compile on daemon: ${pc.time.ms()} ms; thread: user ${pc.threadUserTime.ms()} ms, sys ${(pc.threadTime - pc.threadUserTime).ms()} ms; rpc: ${rpc.count} calls, ${rpc.time.ms()} ms, thread ${rpc.threadTime.ms()} ms; memory: ${endMem.kb()} kb (${
"%+d".format(
pc.memory.kb()
)
} kb)".let {
daemonMessageReporter.report(ReportSeverity.INFO, it)
log.info(it)
}
@@ -454,9 +505,9 @@ abstract class CompileServiceImplBase(
log.log(
Level.SEVERE,
"Exception: $e\n ${e.stackTrace.joinToString("\n ")}${
if (e.cause != null && e.cause != e) {
"\nCaused by: ${e.cause}\n ${e.cause!!.stackTrace.joinToString("\n ")}"
} else ""
if (e.cause != null && e.cause != e) {
"\nCaused by: ${e.cause}\n ${e.cause!!.stackTrace.joinToString("\n ")}"
} else ""
}"
)
throw e
@@ -498,7 +549,7 @@ abstract class CompileServiceImplBase(
protected inline fun <R> ifAliveChecksImpl(
minAliveness: Aliveness = Aliveness.LastSession,
body: () -> CompileService.CallResult<R>
body: () -> CompileService.CallResult<R>,
): CompileService.CallResult<R> {
val curState = state.alive.get()
return when {
@@ -519,7 +570,7 @@ abstract class CompileServiceImplBase(
protected inline fun <R> withValidClientOrSessionProxy(
sessionId: Int,
body: (ClientOrSessionProxy<Any>?) -> CompileService.CallResult<R>
body: (ClientOrSessionProxy<Any>?) -> CompileService.CallResult<R>,
): CompileService.CallResult<R> {
val session: ClientOrSessionProxy<Any>? =
if (sessionId == CompileService.NO_SESSION) null
@@ -536,7 +587,7 @@ abstract class CompileServiceImplBase(
args: K2JSCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>
reporter: RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>,
): ExitCode {
reporter.startMeasureGc()
@Suppress("DEPRECATION") // TODO: get rid of that parsing KT-62759
@@ -579,7 +630,7 @@ abstract class CompileServiceImplBase(
k2jvmArgs: K2JVMCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>
reporter: RemoteBuildReporter<GradleBuildTime, GradleBuildPerformanceMetric>,
): ExitCode {
reporter.startMeasureGc()
val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS +
@@ -651,7 +702,7 @@ abstract class CompileServiceImplBase(
protected inline fun <R, KotlinJvmReplServiceT> withValidReplImpl(
sessionId: Int,
body: KotlinJvmReplServiceT.() -> CompileService.CallResult<R>
body: KotlinJvmReplServiceT.() -> CompileService.CallResult<R>,
): CompileService.CallResult<R> =
withValidClientOrSessionProxy(sessionId) { session ->
@Suppress("UNCHECKED_CAST")
@@ -668,16 +719,17 @@ class CompileServiceImpl(
val daemonJVMOptions: DaemonJVMOptions,
port: Int,
timer: Timer,
val onShutdown: () -> Unit
val onShutdown: () -> Unit,
) : CompileService, CompileServiceImplBase(daemonOptions, compilerId, port, timer) {
private inline fun <R> withValidRepl(
sessionId: Int,
body: KotlinJvmReplService.() -> CompileService.CallResult<R>
body: KotlinJvmReplService.() -> CompileService.CallResult<R>,
) = withValidReplImpl(sessionId, body)
override val lastUsedSeconds: Long get() =
if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds
override val lastUsedSeconds: Long
get() =
if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds
private val rwlock = ReentrantReadWriteLock()
@@ -758,7 +810,7 @@ class CompileServiceImpl(
}
override fun classesFqNamesByFiles(
sessionId: Int, sourceFiles: Set<File>
sessionId: Int, sourceFiles: Set<File>,
): CompileService.CallResult<Set<String>> =
ifAlive {
withValidClientOrSessionProxy(sessionId) {
@@ -771,7 +823,7 @@ class CompileServiceImpl(
compilerArguments: Array<out String>,
compilationOptions: CompilationOptions,
servicesFacade: CompilerServicesFacadeBase,
compilationResults: CompilationResults?
compilationResults: CompilationResults?,
) = ifAlive {
compileImpl(
sessionId,
@@ -783,7 +835,7 @@ class CompileServiceImpl(
createMessageCollector = ::CompileServicesFacadeMessageCollector,
createReporter = ::DaemonMessageReporter,
createServices = this::createCompileServices,
getICReporter = { a, b, c -> getBuildReporter(a, b!!, c)}
getICReporter = { a, b, c -> getBuildReporter(a, b!!, c) }
)
}
@@ -794,7 +846,7 @@ class CompileServiceImpl(
private fun createCompileServices(
@Suppress("DEPRECATION") facade: CompilerCallbackServicesFacade,
eventManager: EventManager,
rpcProfiler: Profiler
rpcProfiler: Profiler,
): Services {
val builder = Services.Builder()
if (facade.hasIncrementalCaches()) {
@@ -834,7 +886,7 @@ class CompileServiceImpl(
compilationOptions: CompilationOptions,
servicesFacade: CompilerServicesFacadeBase,
templateClasspath: List<File>,
templateClassName: String
templateClassName: String,
): CompileService.CallResult<Int> = ifAlive(minAliveness = Aliveness.Alive) {
if (compilationOptions.targetPlatform != CompileService.TargetPlatform.JVM)
CompileService.CallResult.Error("Sorry, only JVM target platform is supported now")
@@ -1047,7 +1099,7 @@ class CompileServiceImpl(
state.sessions.isEmpty() -> shutdownWithDelay()
else -> {
daemonOptions.autoshutdownIdleSeconds =
TimeUnit.MILLISECONDS.toSeconds(daemonOptions.forceShutdownTimeoutMilliseconds).toInt()
TimeUnit.MILLISECONDS.toSeconds(daemonOptions.forceShutdownTimeoutMilliseconds).toInt()
daemonOptions.autoshutdownUnusedSeconds = daemonOptions.autoshutdownIdleSeconds
log.info("Some sessions are active, waiting for them to finish")
log.info("Unused/idle timeouts are set to ${daemonOptions.autoshutdownUnusedSeconds}/${daemonOptions.autoshutdownIdleSeconds}s")
@@ -1101,7 +1153,7 @@ class CompileServiceImpl(
private inline fun <R> ifAlive(
minAliveness: Aliveness = Aliveness.LastSession,
body: () -> CompileService.CallResult<R>
body: () -> CompileService.CallResult<R>,
): CompileService.CallResult<R> = rwlock.read {
ifAliveChecksImpl(minAliveness, body)
}
@@ -1115,7 +1167,7 @@ class CompileServiceImpl(
private inline fun <R> ifAliveExclusive(
minAliveness: Aliveness = Aliveness.LastSession,
body: () -> CompileService.CallResult<R>
body: () -> CompileService.CallResult<R>,
): CompileService.CallResult<R> = rwlock.write {
ifAliveChecksImpl(minAliveness, body)
}
jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTestIncremental.kt
+80
View File
@@ -26,6 +26,8 @@ import org.jetbrains.kotlin.jps.build.fixtures.EnableICFixture
import org.jetbrains.kotlin.jps.model.kotlinCommonCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
import kotlin.reflect.KMutableProperty1
class KotlinJpsBuildTestIncremental : KotlinJpsBuildTest() {
@@ -43,6 +45,84 @@ class KotlinJpsBuildTestIncremental : KotlinJpsBuildTest() {
).run()
}
fun testJpsBuildReportIC() {
val reportDir = workDir.resolve("buildReport")
@Suppress("UNREACHABLE_CODE")
fun getReportFile(): File {
return Files.list(reportDir.toPath()).let {
val files = it.toArray()
val singleFile = (files.singleOrNull() as Path?).also {
it ?: fail("The directory must contain a single file, but got: $files")
}
return singleFile?.toFile()!!
}
}
fun assertFileContains(
file: File,
vararg expectedText: String,
) {
val text = file.readText()
val textNotInTheFile = expectedText.filterNot { text.contains(it) }
assert(textNotInTheFile.isEmpty()) {
"""
|$file does not contain:
|${textNotInTheFile.joinToString(separator = "\n")}
|
|actual file content:
|"$text"
|
""".trimMargin()
}
}
fun validateAndDeleteReportFile(vararg expectedText: String) {
assertTrue(reportDir.exists())
val reportFile = getReportFile()
assertFileContains(reportFile, *expectedText)
reportFile.delete()
}
val reportMetricsList = arrayOf(
"Task 'kotlinProject' finished in",
"Task info:",
"Kotlin language version: 2.0",
"Time metrics:",
"Jps iteration:",
"Compiler code analysis:",
"Compiler code generation:"
)
fun testImpl() {
assertTrue("Daemon was not enabled!", isDaemonEnabled())
doTest()
validateAndDeleteReportFile(
*reportMetricsList,
"Changed files: [${workDir.resolve("src/Foo.kt").path}, ${workDir.resolve("src/main.kt").path}]"
)
val mainKt = File(workDir, "src/main.kt")
change(mainKt.path, "fun main() {}")
buildAllModules().assertSuccessful()
validateAndDeleteReportFile(
*reportMetricsList,
"Changed files: [${workDir.resolve("src/main.kt").path}]"
)
}
withDaemon {
withSystemProperty("kotlin.build.report.file.output_dir", reportDir.path) {
testImpl()
}
}
}
fun testJpsDaemonIC() {
fun testImpl() {
assertTrue("Daemon was not enabled!", isDaemonEnabled())
jps/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/JpsCompilationResult.kt
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import java.io.Serializable
import java.rmi.RemoteException
import java.rmi.server.UnicastRemoteObject
class JpsCompilationResult : CompilationResults,
UnicastRemoteObject(
SOCKET_ANY_FREE_PORT,
LoopbackNetworkInterface.clientLoopbackSocketFactory,
LoopbackNetworkInterface.serverLoopbackSocketFactory
) {
var icLogLines: List<String> = emptyList()
val compiledFiles = ArrayList<String>()
private val buildMetricsReporter = BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>()
val buildMetrics: BuildMetrics<JpsBuildTime, JpsBuildPerformanceMetric>
get() = buildMetricsReporter.getMetrics()
private val log = JpsKotlinLogger(KotlinBuilder.LOG)
@Throws(RemoteException::class)
override fun add(compilationResultCategory: Int, value: Serializable) {
when (compilationResultCategory) {
CompilationResultCategory.IC_COMPILE_ITERATION.code -> {
@Suppress("UNCHECKED_CAST")
val compileIterationResult = value as? CompileIterationResult
if (compileIterationResult != null) {
val sourceFiles = compileIterationResult.sourceFiles
buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.IC_COMPILE_ITERATION)
compiledFiles.addAll(sourceFiles.map { it.path })
}
}
CompilationResultCategory.BUILD_REPORT_LINES.code,
CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES.code -> {
@Suppress("UNCHECKED_CAST")
(value as? List<String>)?.let { icLogLines = it }
}
CompilationResultCategory.BUILD_METRICS.code -> {
(value as BuildMetricsValue).let {
when (it.key) {
CompilationPerformanceMetrics.CODE_GENERATION -> buildMetrics.buildTimes.addTimeMs(JpsBuildTime.CODE_GENERATION, it.value)
CompilationPerformanceMetrics.CODE_ANALYSIS -> buildMetrics.buildTimes.addTimeMs(JpsBuildTime.CODE_ANALYSIS, it.value)
CompilationPerformanceMetrics.COMPILER_INITIALIZATION -> buildMetrics.buildTimes.addTimeMs(JpsBuildTime.COMPILER_INITIALIZATION, it.value)
CompilationPerformanceMetrics.ANALYZED_LINES_NUMBER -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.ANALYZED_LINES_NUMBER, it.value)
CompilationPerformanceMetrics.ANALYSIS_LPS -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.ANALYSIS_LPS, it.value)
CompilationPerformanceMetrics.CODE_GENERATED_LINES_NUMBER -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.CODE_GENERATED_LINES_NUMBER, it.value)
CompilationPerformanceMetrics.CODE_GENERATION_LPS -> buildMetrics.buildPerformanceMetrics.add(JpsBuildPerformanceMetric.CODE_GENERATION_LPS, it.value)
}
}
}
}
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/JpsKotlinCompilerRunner.kt
+31 -16
View File
@@ -31,6 +31,7 @@ import org.jetbrains.kotlin.daemon.client.CompileServiceSession
import org.jetbrains.kotlin.daemon.client.KotlinCompilerClient
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.PrintStream
@@ -91,7 +92,7 @@ class JpsKotlinCompilerRunner {
fun classesFqNamesByFiles(
environment: JpsCompilerEnvironment,
files: Set<File>
files: Set<File>,
): Set<String> = withDaemonOrFallback(
withDaemon = {
doWithDaemon(environment) { sessionId, daemon ->
@@ -113,7 +114,8 @@ class JpsKotlinCompilerRunner {
environment: JpsCompilerEnvironment,
destination: String,
classpath: Collection<String>,
sourceFiles: Collection<File>
sourceFiles: Collection<File>,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2MetadataArguments))
@@ -124,7 +126,7 @@ class JpsKotlinCompilerRunner {
arguments.destination = arguments.destination ?: destination
withCompilerSettings(compilerSettings) {
runCompiler(KotlinCompilerClass.METADATA, arguments, environment)
runCompiler(KotlinCompilerClass.METADATA, arguments, environment, buildMetricReporter)
}
}
@@ -133,12 +135,13 @@ class JpsKotlinCompilerRunner {
k2jvmArguments: K2JVMCompilerArguments,
compilerSettings: CompilerSettings,
environment: JpsCompilerEnvironment,
moduleFile: File
moduleFile: File,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2jvmArguments))
setupK2JvmArguments(moduleFile, arguments)
withCompilerSettings(compilerSettings) {
runCompiler(KotlinCompilerClass.JVM, arguments, environment)
runCompiler(KotlinCompilerClass.JVM, arguments, environment, buildMetricReporter)
}
}
@@ -152,7 +155,8 @@ class JpsKotlinCompilerRunner {
sourceMapRoots: Collection<File>,
libraries: List<String>,
friendModules: List<String>,
outputFile: File
outputFile: File,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
log.debug("K2JS: common arguments: " + ArgumentUtils.convertArgumentsToStringList(commonArguments))
log.debug("K2JS: JS arguments: " + ArgumentUtils.convertArgumentsToStringList(k2jsArguments))
@@ -168,26 +172,32 @@ class JpsKotlinCompilerRunner {
log.debug("K2JS: arguments after setup" + ArgumentUtils.convertArgumentsToStringList(arguments))
withCompilerSettings(compilerSettings) {
runCompiler(KotlinCompilerClass.JS, arguments, environment)
runCompiler(KotlinCompilerClass.JS, arguments, environment, buildMetricReporter)
}
}
private fun compileWithDaemonOrFallback(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
log.debug("Using kotlin-home = " + environment.kotlinPaths.homePath)
withDaemonOrFallback(
withDaemon = { compileWithDaemon(compilerClassName, compilerArgs, environment) },
withDaemon = { compileWithDaemon(compilerClassName, compilerArgs, environment, buildMetricReporter) },
fallback = { fallbackCompileStrategy(compilerArgs, compilerClassName, environment) }
)
}
private fun runCompiler(compilerClassName: String, compilerArgs: CommonCompilerArguments, environment: JpsCompilerEnvironment) {
private fun runCompiler(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?,
) {
try {
compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment)
compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment, buildMetricReporter)
} catch (e: Throwable) {
MessageCollectorUtil.reportException(environment.messageCollector, e)
reportInternalCompilerError(environment.messageCollector)
@@ -197,7 +207,8 @@ class JpsKotlinCompilerRunner {
private fun compileWithDaemon(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?,
): Int? {
val targetPlatform = when (compilerClassName) {
KotlinCompilerClass.JVM -> CompileService.TargetPlatform.JVM
@@ -214,6 +225,8 @@ class JpsKotlinCompilerRunner {
reportSeverity(verbose),
requestedCompilationResults = emptyArray()
)
val compilationResult = JpsCompilationResult()
return doWithDaemon(environment) { sessionId, daemon ->
environment.withProgressReporter { progress ->
progress.compilationStarted()
@@ -222,8 +235,10 @@ class JpsKotlinCompilerRunner {
withAdditionalCompilerArgs(compilerArgs),
options,
JpsCompilerServicesFacadeImpl(environment),
null
compilationResult
)
}.also {
buildMetricReporter?.addCompilerMetrics(compilationResult)
}
}
}
@@ -237,7 +252,7 @@ class JpsKotlinCompilerRunner {
private fun <T> doWithDaemon(
environment: JpsCompilerEnvironment,
fn: (sessionId: Int, daemon: CompileService) -> CompileService.CallResult<T>
fn: (sessionId: Int, daemon: CompileService) -> CompileService.CallResult<T>,
): T? {
log.debug("Try to connect to daemon")
val connection = getDaemonConnection(environment)
@@ -280,7 +295,7 @@ class JpsKotlinCompilerRunner {
private fun fallbackCompileStrategy(
compilerArgs: CommonCompilerArguments,
compilerClassName: String,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
) {
if ("true" == System.getProperty("kotlin.jps.tests") && "true" == System.getProperty(FAIL_ON_FALLBACK_PROPERTY)) {
error("Cannot compile with Daemon, see logs bellow. Fallback strategy is disabled in tests")
@@ -325,7 +340,7 @@ class JpsKotlinCompilerRunner {
_commonSources: Collection<File>,
_libraries: List<String>,
_friendModules: List<String>,
settings: K2JSCompilerArguments
settings: K2JSCompilerArguments,
) {
with(settings) {
noStdlib = true
jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+29 -7
View File
@@ -24,6 +24,7 @@ import org.jetbrains.kotlin.build.report.ICReporter.ReportSeverity
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.debug
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
@@ -40,7 +41,9 @@ import org.jetbrains.kotlin.jps.KotlinJpsBundle
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
import org.jetbrains.kotlin.jps.model.kotlinKind
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.jps.statistic.JpsStatisticsReportService
import org.jetbrains.kotlin.jps.statistic.statisticsReportServiceKey
import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget
import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
@@ -66,7 +69,6 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
System.getProperty("kotlin.jps.classesToLoadByParent")?.split(',')?.map { it.trim() } ?: emptyList()
private val classPrefixesToLoadByParentFromRegistry =
System.getProperty("kotlin.jps.classPrefixesToLoadByParent")?.split(',')?.map { it.trim() } ?: emptyList()
private val reportService = JpsStatisticsReportService.create()
val classesToLoadByParent: ClassCondition
get() = ClassCondition { className ->
@@ -100,6 +102,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
override fun buildStarted(context: CompileContext) {
logSettings(context)
val reportService = JpsStatisticsReportService.create()
context.putUserData(statisticsReportServiceKey, reportService)
reportService.buildStarted(context)
}
@@ -161,6 +165,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
override fun buildFinished(context: CompileContext) {
ensureKotlinContextDisposed(context)
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.buildFinish(context)
}
@@ -295,7 +300,19 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
outputConsumer: OutputConsumer
): ExitCode {
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.moduleBuildStarted(chunk)
return doBuild(context, chunk, dirtyFilesHolder, outputConsumer).also { result ->
reportService.moduleBuildFinished(chunk, context, result)
}
}
private fun doBuild(
context: CompileContext,
chunk: ModuleChunk,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
outputConsumer: OutputConsumer
): ExitCode {
if (chunk.isDummy(context))
return NOTHING_DONE
@@ -320,6 +337,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val fsOperations = FSOperationsHelper(context, chunk, kotlinDirtyFilesHolder, LOG)
try {
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.reportDirtyFiles(kotlinDirtyFilesHolder)
return reportService.reportMetrics(chunk, JpsBuildTime.JPS_ITERATION) {
val proposedExitCode =
doBuild(chunk, kotlinTarget, context, kotlinDirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
@@ -342,8 +361,6 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
LOG.info("Caught exception: $e")
MessageCollectorUtil.reportException(messageCollector, e)
return ABORT
} finally {
reportService.moduleBuildFinished(chunk, context)
}
}
@@ -354,7 +371,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder,
messageCollector: MessageCollectorAdapter,
outputConsumer: OutputConsumer,
fsOperations: FSOperationsHelper
fsOperations: FSOperationsHelper,
): ExitCode {
// Workaround for Android Studio
if (representativeTarget is KotlinJvmModuleBuildTarget && !JavaBuilder.IS_ENABLED[context, true]) {
@@ -448,6 +465,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
LOG.debug("Compiling files: ${kotlinDirtyFilesHolder.allDirtyFiles}")
}
val reportService = JpsStatisticsReportService.getFromContext(context)
reportService.reportCompilerArguments(chunk, kotlinChunk)
val start = System.nanoTime()
val outputItemCollector = doCompileModuleChunk(
kotlinChunk,
@@ -457,7 +476,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
kotlinDirtyFilesHolder,
fsOperations,
environment,
incrementalCaches
incrementalCaches,
reportService.getMetricReporter(chunk)
)
statisticsLogger.registerStatistic(chunk, System.nanoTime() - start)
@@ -591,13 +611,15 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
fsOperations: FSOperationsHelper,
environment: JpsCompilerEnvironment,
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
buildMetricReporter: JpsBuilderMetricReporter?,
): OutputItemsCollector? {
kotlinChunk.targets.forEach {
it.nextRound(context)
}
if (representativeTarget.isIncrementalCompilationEnabled) {
buildMetricReporter?.addTag(StatTag.INCREMENTAL)
for (target in kotlinChunk.targets) {
val cache = incrementalCaches[target]
val jpsTarget = target.jpsModuleBuildTarget
@@ -613,7 +635,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
}
}
val isDoneSomething = representativeTarget.compileModuleChunk(commonArguments, dirtyFilesHolder, environment)
val isDoneSomething = representativeTarget.compileModuleChunk(commonArguments, dirtyFilesHolder, environment, buildMetricReporter)
return if (isDoneSomething) environment.outputItemsCollector else null
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsBuilderMetricReporter.kt
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.statistic
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.compilerRunner.JpsCompilationResult
interface JpsBuilderMetricReporter : BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> {
fun flush(context: CompileContext): JpsCompileStatisticsData
fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext, exitCode: String)
fun addChangedFiles(files: List<String>)
fun addCompilerArguments(arguments: List<String>)
fun setKotlinLanguageVersion(languageVersion: String?)
fun addTag(tag: StatTag)
fun addCompilerMetrics(jpsCompilationResult: JpsCompilationResult)
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsBuilderMetricReporterImpl.kt
+104
View File
@@ -0,0 +1,104 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.statistic
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporterImpl
import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.BuildDataType
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.compilerRunner.JpsCompilationResult
import java.net.InetAddress
import java.util.*
import kotlin.collections.ArrayList
class JpsBuilderMetricReporterImpl(
chunk: ModuleChunk,
private val reporter: BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>,
private val label: String? = null,
private val kotlinVersion: String = "kotlin_version"
) :
JpsBuilderMetricReporter, BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> by reporter {
companion object {
private val hostName: String? = try {
InetAddress.getLocalHost().hostName
} catch (_: Exception) {
//do nothing
null
}
private val uuid = UUID.randomUUID()
}
private val startTime = System.currentTimeMillis()
private var finishTime: Long = 0L
private val tags = HashSet<StatTag>()
private val changedFiles = ArrayList<String>()
private val compilerArguments = ArrayList<String>()
private val moduleString = chunk.name
private var exitCode = "Unknown"
private var kotlinLanguageVersion: String? = null
override fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext, exitCode: String) {
finishTime = System.currentTimeMillis()
this.exitCode = exitCode
}
override fun addChangedFiles(files: List<String>) {
changedFiles.addAll(files)
}
override fun addCompilerArguments(arguments: List<String>) {
compilerArguments.addAll(arguments)
}
override fun setKotlinLanguageVersion(languageVersion: String?) {
kotlinLanguageVersion = languageVersion
}
override fun addTag(tag: StatTag) {
tags.add(tag)
}
override fun addCompilerMetrics(jpsCompilationResult: JpsCompilationResult) {
reporter.addMetrics(jpsCompilationResult.buildMetrics)
}
override fun flush(context: CompileContext): JpsCompileStatisticsData {
val buildMetrics = reporter.getMetrics()
return JpsCompileStatisticsData(
projectName = context.projectDescriptor.project.name,
label = label,
taskName = moduleString,
taskResult = exitCode,
startTimeMs = startTime,
durationMs = finishTime - startTime,
tags = tags,
buildUuid = uuid.toString(),
changes = changedFiles,
kotlinVersion = kotlinVersion,
hostName = hostName,
finishTime = finishTime,
buildTimesMetrics = buildMetrics.buildTimes.asMapMs(),
performanceMetrics = buildMetrics.buildPerformanceMetrics.asMap(),
compilerArguments = compilerArguments,
nonIncrementalAttributes = emptySet(),
type = BuildDataType.JPS_DATA.name,
fromKotlinPlugin = true,
compiledSources = emptyList(),
skipMessage = null,
icLogLines = emptyList(),
gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(),
gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(),
kotlinLanguageVersion = kotlinLanguageVersion
)
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsFileReportService.kt
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.statistic
import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
import org.jetbrains.kotlin.build.report.statistics.CompileStatisticsData
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.compilerRunner.JpsKotlinLogger
import java.io.File
internal class JpsFileReportService(
buildReportDir: File,
projectName: String,
printMetrics: Boolean,
logger: JpsKotlinLogger,
) : FileReportService<JpsBuildTime, JpsBuildPerformanceMetric>(buildReportDir, projectName, printMetrics, logger) {
override fun printCustomTaskMetrics(statisticsData: CompileStatisticsData<JpsBuildTime, JpsBuildPerformanceMetric>) {
p.print("Changed files: ${statisticsData.getChanges().sorted()}")
p.print("Execution result: ${statisticsData.getTaskResult()}")
}
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt
+44 -81
View File
@@ -8,92 +8,29 @@ package org.jetbrains.kotlin.jps.statistic
import com.intellij.openapi.diagnostic.Logger
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.GlobalContextKey
import org.jetbrains.jps.incremental.ModuleLevelBuilder
import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode
import org.jetbrains.kotlin.build.report.FileReportSettings
import org.jetbrains.kotlin.build.report.HttpReportSettings
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.build.report.statistics.BuildDataType
import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters
import org.jetbrains.kotlin.build.report.statistics.HttpReportService
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.build.report.statistics.*
import org.jetbrains.kotlin.compilerRunner.JpsKotlinLogger
import org.jetbrains.kotlin.jps.build.KotlinChunk
import org.jetbrains.kotlin.jps.build.KotlinCompileContext
import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import java.io.File
import java.net.InetAddress
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
interface JpsBuilderMetricReporter : BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> {
fun flush(context: CompileContext): JpsCompileStatisticsData
fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext)
}
private const val jpsBuildTaskName = "JPS build"
class JpsBuilderMetricReporterImpl(
chunk: ModuleChunk,
private val reporter: BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>,
private val label: String? = null,
private val kotlinVersion: String = "kotlin_version",
) :
JpsBuilderMetricReporter, BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> by reporter {
companion object {
private val hostName: String? = try {
InetAddress.getLocalHost().hostName
} catch (_: Exception) {
//do nothing
null
}
private val uuid = UUID.randomUUID()
}
private val startTime = System.currentTimeMillis()
private var finishTime: Long = 0L
private val tags = HashSet<StatTag>()
private val moduleString = chunk.name
override fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext) {
finishTime = System.currentTimeMillis()
}
override fun flush(context: CompileContext): JpsCompileStatisticsData {
val buildMetrics = reporter.getMetrics()
return JpsCompileStatisticsData(
projectName = context.projectDescriptor.project.name,
label = label,
taskName = moduleString,
taskResult = "Unknown",//TODO will be updated in KT-58026
startTimeMs = startTime,
durationMs = finishTime - startTime,
tags = tags,
buildUuid = uuid.toString(),
changes = emptyList(), //TODO will be updated in KT-58026
kotlinVersion = kotlinVersion,
hostName = hostName,
finishTime = finishTime,
buildTimesMetrics = buildMetrics.buildTimes.asMapMs(),
performanceMetrics = buildMetrics.buildPerformanceMetrics.asMap(),
compilerArguments = emptyList(), //TODO will be updated in KT-58026
nonIncrementalAttributes = emptySet(),
type = BuildDataType.JPS_DATA.name,
fromKotlinPlugin = true,
compiledSources = emptyList(),
skipMessage = null,
icLogLines = emptyList(),
gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(),
gcCountMetrics = buildMetrics.gcMetrics.asGcCountMap(),
kotlinLanguageVersion = null
)
}
}
internal val statisticsReportServiceKey = GlobalContextKey<JpsStatisticsReportService>("jpsStatistics")
sealed class JpsStatisticsReportService {
companion object {
fun create(): JpsStatisticsReportService {
internal fun create(): JpsStatisticsReportService {
val fileReportSettings = initFileReportSettings()
val httpReportSettings = initHttpReportSettings()
@@ -102,9 +39,11 @@ sealed class JpsStatisticsReportService {
} else {
JpsStatisticsReportServiceImpl(fileReportSettings, httpReportSettings)
}
}
internal fun getFromContext(context: CompileContext): JpsStatisticsReportService =
context.getUserData(statisticsReportServiceKey) ?: DummyJpsStatisticsReportService
private fun initFileReportSettings(): FileReportSettings? {
return System.getProperty("kotlin.build.report.file.output_dir")?.let { FileReportSettings(File(it)) }
}
@@ -123,8 +62,12 @@ sealed class JpsStatisticsReportService {
abstract fun buildStarted(context: CompileContext)
abstract fun buildFinish(context: CompileContext)
abstract fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext)
abstract fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext, exitCode: ModuleLevelBuilder.ExitCode)
abstract fun moduleBuildStarted(chunk: ModuleChunk)
abstract fun reportDirtyFiles(kotlinDirtySourceFilesHolder: KotlinDirtySourceFilesHolder)
abstract fun reportCompilerArguments(chunk: ModuleChunk, kotlinChunk: KotlinChunk)
abstract fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter?
}
@@ -135,9 +78,13 @@ object DummyJpsStatisticsReportService : JpsStatisticsReportService() {
override fun buildStarted(context: CompileContext) {}
override fun buildFinish(context: CompileContext) {}
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext) {}
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext, exitCode: ExitCode) {}
override fun moduleBuildStarted(chunk: ModuleChunk) {}
override fun reportDirtyFiles(kotlinDirtySourceFilesHolder: KotlinDirtySourceFilesHolder) {}
override fun reportCompilerArguments(chunk: ModuleChunk, kotlinChunk: KotlinChunk) {}
override fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter? = null
}
class JpsStatisticsReportServiceImpl(
@@ -161,7 +108,7 @@ class JpsStatisticsReportServiceImpl(
log.debug("JpsStatisticsReportService: Build started for $moduleName module")
}
private fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter? {
override fun getMetricReporter(chunk: ModuleChunk): JpsBuilderMetricReporter? {
val moduleName = chunk.name
return getMetricReporter(moduleName)
}
@@ -176,7 +123,7 @@ class JpsStatisticsReportServiceImpl(
return metricReporter
}
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext) {
override fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext, exitCode: ExitCode) {
val moduleName = chunk.name
val metrics = buildMetrics.remove(moduleName)
if (metrics == null) {
@@ -184,7 +131,7 @@ class JpsStatisticsReportServiceImpl(
return
}
log.debug("JpsStatisticsReportService: Build started for $moduleName module")
metrics.buildFinish(chunk, context)
metrics.buildFinish(chunk, context, exitCode.name)
finishedModuleBuildMetrics.add(metrics)
}
@@ -192,9 +139,11 @@ class JpsStatisticsReportServiceImpl(
val compileStatisticsData = finishedModuleBuildMetrics.map { it.flush(context) }
httpService?.sendData(compileStatisticsData, loggerAdapter)
fileReportSettings?.also {
FileReportService.reportBuildStatInFile(
it.buildReportDir, context.projectDescriptor.project.name, true, compileStatisticsData,
BuildStartParameters(tasks = listOf(jpsBuildTaskName)), emptyList(), loggerAdapter
JpsFileReportService(
it.buildReportDir, context.projectDescriptor.project.name, true, loggerAdapter
).process(
compileStatisticsData,
BuildStartParameters(tasks = listOf(jpsBuildTaskName)), emptyList(),
)
}
}
@@ -203,6 +152,20 @@ class JpsStatisticsReportServiceImpl(
return getMetricReporter(chunk)?.measure(metric, action) ?: action.invoke()
}
override fun reportDirtyFiles(kotlinDirtySourceFilesHolder: KotlinDirtySourceFilesHolder) {
getMetricReporter(kotlinDirtySourceFilesHolder.chunk)?.let {
it.addChangedFiles(kotlinDirtySourceFilesHolder.allRemovedFilesFiles.map { it.path })
it.addChangedFiles(kotlinDirtySourceFilesHolder.allDirtyFiles.map { it.path })
}
}
override fun reportCompilerArguments(chunk: ModuleChunk, kotlinChunk: KotlinChunk) {
getMetricReporter(chunk)?.let {
it.addCompilerArguments(kotlinChunk.compilerArguments.freeArgs)
it.setKotlinLanguageVersion(kotlinChunk.compilerArguments.languageVersion)
}
}
override fun buildStarted(context: CompileContext) {
loggerAdapter.info("Build started for $context with enabled build metric reports.")
}
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinCommonModuleBuildTarget.kt
+5 -2
View File
@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.build.ModuleBuildTarget
import org.jetbrains.kotlin.jps.model.k2MetadataCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
private const val COMMON_BUILD_META_INFO_FILE_NAME = "common-build-meta-info.txt"
@@ -47,7 +48,8 @@ class KotlinCommonModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModu
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
require(chunk.representativeTarget == this)
@@ -60,7 +62,8 @@ class KotlinCommonModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModu
environment,
destination,
dependenciesOutputDirs + libraryFiles,
sourceFiles // incremental K2MetadataCompiler not supported yet
sourceFiles, // incremental K2MetadataCompiler not supported yet
buildMetricReporter
)
return true
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt
+5 -2
View File
@@ -35,6 +35,7 @@ import org.jetbrains.kotlin.jps.model.k2JsCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
import org.jetbrains.kotlin.jps.model.testOutputFilePath
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.utils.JsLibraryUtils
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils.JS_EXT
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils.META_JS_SUFFIX
@@ -92,7 +93,8 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
require(chunk.representativeTarget == this)
@@ -116,7 +118,8 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
sourceMapRoots,
libraries,
friendBuildTargetsMetaFiles,
outputFile
outputFile,
buildMetricReporter
)
return true
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt
+5 -2
View File
@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache
import org.jetbrains.kotlin.jps.model.k2JvmCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.modules.KotlinModuleXmlBuilder
@@ -99,7 +100,8 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
require(chunk.representativeTarget == this)
@@ -144,7 +146,8 @@ class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleB
module.k2JvmCompilerArguments,
module.kotlinCompilerSettings,
environment,
moduleFile
moduleFile,
buildMetricReporter
)
} finally {
if (System.getProperty(DELETE_MODULE_FILE_PROPERTY) != "false") {
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt
+3 -1
View File
@@ -31,6 +31,7 @@ import org.jetbrains.kotlin.jps.incremental.loadDiff
import org.jetbrains.kotlin.jps.incremental.localCacheVersionManager
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
import org.jetbrains.kotlin.jps.model.testOutputFilePath
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.progress.CompilationCanceledException
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
@@ -202,7 +203,8 @@ abstract class KotlinModuleBuildTarget<BuildMetaInfoType : BuildMetaInfo> intern
abstract fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean
open fun registerOutputItems(outputConsumer: ModuleLevelBuilder.OutputConsumer, outputItems: List<GeneratedFile>) {
jps/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinUnsupportedModuleBuildTarget.kt
+3 -1
View File
@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.jps.build.KotlinCompileContext
import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.model.platform
import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.platform.idePlatformKind
class KotlinUnsupportedModuleBuildTarget(
@@ -40,7 +41,8 @@ class KotlinUnsupportedModuleBuildTarget(
override fun compileModuleChunk(
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
environment: JpsCompilerEnvironment,
buildMetricReporter: JpsBuilderMetricReporter?
): Boolean {
shouldNotBeCalled()
}
jps/jps-plugin/testData/general/JpsBuildReportIC/kotlinProject.iml
Vendored
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="IDEA_JDK" jdkType="JavaSDK" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="KotlinRuntime" level="project" />
</component>
</module>
jps/jps-plugin/testData/general/JpsBuildReportIC/kotlinProject.ipr
Vendored
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<option name="DEFAULT_COMPILER" value="Javac" />
</component>
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/kotlinProject.iml" filepath="$PROJECT_DIR$/kotlinProject.iml" />
</modules>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" assert-keyword="true" jdk-15="true" project-jdk-name="IDEA_JDK" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
jps/jps-plugin/testData/general/JpsBuildReportIC/src/Foo.kt
Vendored
+1
View File
@@ -0,0 +1 @@
class Foo()
jps/jps-plugin/testData/general/JpsBuildReportIC/src/main.kt
Vendored
+3
View File
@@ -0,0 +1,3 @@
fun main() {
Foo()
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/GradleFileReportService.kt
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.plugin.statistics
import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.buildtools.api.KotlinLogger
import java.io.File
class GradleFileReportService(
buildReportDir: File,
projectName: String,
printMetrics: Boolean,
logger: KotlinLogger,
) : FileReportService<GradleBuildTime, GradleBuildPerformanceMetric>(buildReportDir, projectName, printMetrics, logger) {}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt
+15 -13
View File
@@ -10,12 +10,12 @@ import org.gradle.api.logging.Logging
import org.gradle.tooling.events.task.TaskFinishEvent
import org.jetbrains.kotlin.build.report.metrics.ValueType
import org.jetbrains.kotlin.build.report.statistics.HttpReportService
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.build.report.statistics.formatSize
import org.jetbrains.kotlin.build.report.statistics.BuildFinishStatisticsData
import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.gradle.plugin.statistics.GradleFileReportService
import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData
import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord
import org.jetbrains.kotlin.gradle.report.data.GradleCompileStatisticsData
@@ -51,7 +51,7 @@ class BuildReportsService {
fun close(
buildOperationRecords: Collection<BuildOperationRecord>,
failureMessages: List<String>,
parameters: BuildReportParameters
parameters: BuildReportParameters,
) {
val buildData = BuildExecutionData(
startParameters = parameters.startParameters,
@@ -65,14 +65,15 @@ class BuildReportsService {
executorService.submit { reportBuildFinish(parameters) }
}
reportingSettings.fileReportSettings?.also {
FileReportService.reportBuildStatInFile(
GradleFileReportService(
it.buildReportDir,
parameters.projectName,
it.includeMetricsInReport,
loggerAdapter
).process(
transformOperationRecordsToCompileStatisticsData(buildOperationRecords, parameters, onlyKotlinTask = false),
parameters.startParameters,
failureMessages.filter { it.isNotEmpty() },
loggerAdapter
)
}
@@ -92,7 +93,7 @@ class BuildReportsService {
buildOperationRecords: Collection<BuildOperationRecord>,
parameters: BuildReportParameters,
onlyKotlinTask: Boolean,
metricsToShow: Set<String>? = null
metricsToShow: Set<String>? = null,
) = buildOperationRecords.mapNotNull {
prepareData(
taskResult = null,
@@ -112,7 +113,7 @@ class BuildReportsService {
fun onFinish(
event: TaskFinishEvent, buildOperation: BuildOperationRecord,
parameters: BuildReportParameters
parameters: BuildReportParameters,
) {
addHttpReport(event, buildOperation, parameters)
}
@@ -162,7 +163,7 @@ class BuildReportsService {
private fun addHttpReport(
event: TaskFinishEvent,
buildOperationRecord: BuildOperationRecord,
parameters: BuildReportParameters
parameters: BuildReportParameters,
) {
parameters.httpService?.also { httpService ->
val data =
@@ -189,7 +190,7 @@ class BuildReportsService {
event: TaskFinishEvent,
buildOperationRecord: BuildOperationRecord,
parameters: BuildReportParameters,
buildScanExtension: BuildScanExtensionHolder
buildScanExtension: BuildScanExtensionHolder,
) {
val buildScanSettings = parameters.reportingSettings.buildScanReportSettings ?: return
@@ -212,7 +213,7 @@ class BuildReportsService {
internal fun addBuildScanReport(
buildOperationRecords: Collection<BuildOperationRecord>,
parameters: BuildReportParameters,
buildScanExtension: BuildScanExtensionHolder
buildScanExtension: BuildScanExtensionHolder,
) {
val buildScanSettings = parameters.reportingSettings.buildScanReportSettings ?: return
@@ -256,14 +257,14 @@ class BuildReportsService {
private fun addBuildScanValue(
buildScan: BuildScanExtensionHolder,
data: GradleCompileStatisticsData,
customValue: String
customValue: String,
) {
buildScan.buildScan.value(data.getTaskName(), customValue)
customValues++
}
private fun reportTryNextToConsole(
data: BuildExecutionData
data: BuildExecutionData,
) {
val tasksData = data.buildOperationRecord
.filterIsInstance<TaskRecord>()
@@ -309,7 +310,8 @@ class BuildReportsService {
}
val timeData =
data.getBuildTimesMetrics().map { (key, value) -> "${key.getReadableString()}: ${value}ms" } //sometimes it is better to have separate variable to be able debug
data.getBuildTimesMetrics()
.map { (key, value) -> "${key.getReadableString()}: ${value}ms" } //sometimes it is better to have separate variable to be able debug
val perfData = data.getPerformanceMetrics().map { (key, value) ->
when (key.getType()) {
ValueType.BYTES -> "${key.getReadableString()}: ${formatSize(value)}"
@@ -419,5 +421,5 @@ data class BuildReportParameters(
val label: String?,
val projectName: String,
val kotlinVersion: String,
val additionalTags: Set<StatTag>
val additionalTags: Set<StatTag>,
)