Don't use global state for keeping incremental compilation state

Previously IC state was stored in System properties. As result parallel
compilation might cause incorrect state of IC, what led to corruption
of kotlin_module files. Now IC state is stored via CompilerArguments
and CompilerConfiguration
#KT-46038 Fixed
This commit is contained in:
Andrey Uskov
2021-09-20 21:17:18 +03:00
committed by teamcity
parent 99300bd885
commit 2adc851f1b
21 changed files with 98 additions and 32 deletions
@@ -47,7 +47,8 @@ fun makeModuleFile(
commonSources: Iterable<File>,
javaSourceRoots: Iterable<JvmSourceRoot>,
classpath: Iterable<File>,
friendDirs: Iterable<File>
friendDirs: Iterable<File>,
isIncrementalMode: Boolean = true
): File {
val builder = KotlinModuleXmlBuilder()
builder.addModule(
@@ -65,7 +66,8 @@ fun makeModuleFile(
isTest,
// this excludes the output directories from the class path, to be removed for true incremental compilation
setOf(outputDir),
friendDirs
friendDirs,
isIncrementalMode
)
val scriptFile = Files.createTempFile("kjps", sanitizeJavaIdentifier(name) + ".script.xml").toFile()
@@ -31,6 +31,7 @@ class KotlinModuleXmlBuilder {
openTag(p, MODULES)
}
@Deprecated(message = "State of IC should be set explicitly")
fun addModule(
moduleName: String,
outputDir: String,
@@ -43,6 +44,34 @@ class KotlinModuleXmlBuilder {
isTests: Boolean,
directoriesToFilterOut: Set<File>,
friendDirs: Iterable<File>
) = addModule(
moduleName,
outputDir,
sourceFiles,
javaSourceRoots,
classpathRoots,
commonSourceFiles,
modularJdkRoot,
targetTypeId,
isTests,
directoriesToFilterOut,
friendDirs,
IncrementalCompilation.isEnabledForJvm()
)
fun addModule(
moduleName: String,
outputDir: String,
sourceFiles: Iterable<File>,
javaSourceRoots: Iterable<JvmSourceRoot>,
classpathRoots: Iterable<File>,
commonSourceFiles: Iterable<File>,
modularJdkRoot: File?,
targetTypeId: String,
isTests: Boolean,
directoriesToFilterOut: Set<File>,
friendDirs: Iterable<File>,
isIncrementalCompilation: Boolean
): KotlinModuleXmlBuilder {
assert(!done) { "Already done" }
@@ -67,7 +96,7 @@ class KotlinModuleXmlBuilder {
}
processJavaSourceRoots(javaSourceRoots)
processClasspath(classpathRoots, directoriesToFilterOut)
processClasspath(classpathRoots, directoriesToFilterOut, isIncrementalCompilation)
if (modularJdkRoot != null) {
p.println("<", MODULAR_JDK_ROOT, " ", PATH, "=\"", getEscapedPath(modularJdkRoot), "\"/>")
@@ -79,10 +108,11 @@ class KotlinModuleXmlBuilder {
private fun processClasspath(
files: Iterable<File>,
directoriesToFilterOut: Set<File>) {
directoriesToFilterOut: Set<File>,
isIncrementalCompilation: Boolean) {
p.println("<!-- Classpath -->")
for (file in files) {
val isOutput = directoriesToFilterOut.contains(file) && !IncrementalCompilation.isEnabledForJvm()
val isOutput = directoriesToFilterOut.contains(file) && !isIncrementalCompilation
if (isOutput) {
// For IDEA's make (incremental compilation) purposes, output directories of the current module and its dependencies
// appear on the class path, so we are at risk of seeing the results of the previous build, i.e. if some class was
@@ -372,7 +372,7 @@ class MultifileClassCodegenImpl(
private fun getCompiledPackageFragment(
facadeFqName: FqName, state: GenerationState
): IncrementalPackageFragmentProvider.IncrementalMultifileClassPackageFragment? {
if (!IncrementalCompilation.isEnabledForJvm()) return null
if (!state.isIncrementalCompilation) return null
val packageFqName = facadeFqName.parent()
@@ -72,6 +72,7 @@ class GenerationState private constructor(
val isIrBackend: Boolean,
val ignoreErrors: Boolean,
val diagnosticReporter: DiagnosticReporter,
val isIncrementalCompilation: Boolean
) {
class Builder(
private val project: Project,
@@ -129,13 +130,16 @@ class GenerationState private constructor(
fun diagnosticReporter(v: DiagnosticReporter) =
apply { diagnosticReporter = v }
val isIncrementalCompilation: Boolean = configuration.getBoolean(CommonConfigurationKeys.INCREMENTAL_COMPILATION)
fun build() =
GenerationState(
project, builderFactory, module, bindingContext, files, configuration,
generateDeclaredClassFilter, codegenFactory, targetId,
moduleName, outDirectory, onIndependentPartCompilationEnd, wantsDiagnostics,
jvmBackendClassResolver, isIrBackend, ignoreErrors,
diagnosticReporter ?: DiagnosticReporterFactory.createReporter()
diagnosticReporter ?: DiagnosticReporterFactory.createReporter(),
isIncrementalCompilation
)
}
@@ -394,6 +394,9 @@ abstract class CommonCompilerArguments : CommonToolArguments() {
)
var normalizeAbsolutePath: Boolean by FreezableVar(false)
@Argument(value = "-Xenable-incremental-compilation", description = "Enable incremental compilation")
var incrementalCompilation: Boolean? by FreezableVar(null)
open fun configureAnalysisFlags(collector: MessageCollector, languageVersion: LanguageVersion): MutableMap<AnalysisFlag<*>, Any> {
return HashMap<AnalysisFlag<*>, Any>().apply {
put(AnalysisFlags.skipMetadataVersionCheck, skipMetadataVersionCheck)
@@ -66,6 +66,7 @@ import java.util.stream.Collectors;
import static org.jetbrains.kotlin.cli.common.ExitCode.COMPILATION_ERROR;
import static org.jetbrains.kotlin.cli.common.ExitCode.OK;
import static org.jetbrains.kotlin.cli.common.UtilsKt.getLibraryFromHome;
import static org.jetbrains.kotlin.cli.common.UtilsKt.incrementalCompilationIsEnabledForJs;
import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*;
public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
@@ -181,7 +182,7 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
return exitCode;
}
if (arguments.getFreeArgs().isEmpty() && !IncrementalCompilation.isEnabledForJs()) {
if (arguments.getFreeArgs().isEmpty() && (!incrementalCompilationIsEnabledForJs(arguments))) {
if (arguments.getVersion()) {
return OK;
}
@@ -219,7 +220,7 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
return ExitCode.COMPILATION_ERROR;
}
if (sourcesFiles.isEmpty() && !IncrementalCompilation.isEnabledForJs()) {
if (sourcesFiles.isEmpty() && !incrementalCompilationIsEnabledForJs(arguments)) {
messageCollector.report(ERROR, "No source files", null);
return COMPILATION_ERROR;
}
@@ -114,7 +114,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
return scriptingEvaluator.eval(arguments, configuration, projectEnv)
}
if (arguments.freeArgs.isEmpty() && !IncrementalCompilation.isEnabledForJs()) {
if (arguments.freeArgs.isEmpty() && !(incrementalCompilationIsEnabledForJs(arguments))) {
if (arguments.version) {
return OK
}
@@ -166,7 +166,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
return ExitCode.COMPILATION_ERROR
}
if (sourcesFiles.isEmpty() && !IncrementalCompilation.isEnabledForJs() && arguments.includes.isNullOrEmpty()) {
if (sourcesFiles.isEmpty() && (!incrementalCompilationIsEnabledForJs(arguments)) && arguments.includes.isNullOrEmpty()) {
messageCollector.report(ERROR, "No source files", null)
return COMPILATION_ERROR
}
@@ -27,6 +27,7 @@ fun <A : CommonCompilerArguments> CompilerConfiguration.setupCommonArguments(
put(CommonConfigurationKeys.EXPECT_ACTUAL_LINKER, arguments.expectActualLinker)
putIfNotNull(CLIConfigurationKeys.INTELLIJ_PLUGIN_ROOT, arguments.intellijPluginRoot)
put(CommonConfigurationKeys.REPORT_OUTPUT_FILES, arguments.reportOutputFiles)
put(CommonConfigurationKeys.INCREMENTAL_COMPILATION, incrementalCompilationIsEnabled(arguments))
val metadataVersionString = arguments.metadataVersion
if (metadataVersionString != null) {
@@ -16,10 +16,12 @@
package org.jetbrains.kotlin.cli.common
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.messages.MessageUtil
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.isSubpackageOf
import org.jetbrains.kotlin.psi.KtFile
@@ -28,6 +30,14 @@ import org.jetbrains.kotlin.utils.KotlinPaths
import java.io.File
import kotlin.system.exitProcess
fun incrementalCompilationIsEnabled(arguments: CommonCompilerArguments): Boolean {
return arguments.incrementalCompilation ?: IncrementalCompilation.isEnabledForJvm()
}
fun incrementalCompilationIsEnabledForJs(arguments: CommonCompilerArguments): Boolean {
return arguments.incrementalCompilation ?: IncrementalCompilation.isEnabledForJs()
}
fun checkKotlinPackageUsage(configuration: CompilerConfiguration, files: Collection<KtFile>): Boolean {
if (configuration.getBoolean(CLIConfigurationKeys.ALLOW_KOTLIN_PACKAGE)) {
return true
@@ -240,7 +240,7 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
configuration: CompilerConfiguration, arguments: K2JVMCompilerArguments, services: Services
) {
with(configuration) {
if (IncrementalCompilation.isEnabledForJvm()) {
if (incrementalCompilationIsEnabled(arguments)) {
putIfNotNull(CommonConfigurationKeys.LOOKUP_TRACKER, services[LookupTracker::class.java])
putIfNotNull(CommonConfigurationKeys.EXPECT_ACTUAL_TRACKER, services[ExpectActualTracker::class.java])
@@ -66,6 +66,10 @@ object CommonConfigurationKeys {
@JvmField
val KLIB_NORMALIZE_ABSOLUTE_PATH =
CompilerConfigurationKey.create<Boolean>("Normalize absolute paths in klib (replace file separator with '/')")
@JvmField
val INCREMENTAL_COMPILATION =
CompilerConfigurationKey.create<Boolean>("Enable incremental compilation")
}
var CompilerConfiguration.languageVersionSettings: LanguageVersionSettings
@@ -319,7 +319,7 @@ abstract class CompileServiceImplBase(
CompilerMode.JPS_COMPILER -> {
@Suppress("UNCHECKED_CAST")
servicesFacade as JpsServicesFacadeT
withIC(enabled = servicesFacade.hasIncrementalCaches()) {
withIC(k2PlatformArgs, enabled = servicesFacade.hasIncrementalCaches()) {
doCompile(sessionId, daemonReporter, tracer = null) { eventManger, profiler ->
val services = createServices(servicesFacade, eventManger, profiler)
compiler.exec(messageCollector, services, k2PlatformArgs)
@@ -346,7 +346,7 @@ abstract class CompileServiceImplBase(
val gradleIncrementalServicesFacade = servicesFacade
when (targetPlatform) {
CompileService.TargetPlatform.JVM -> withIC {
CompileService.TargetPlatform.JVM -> withIC(k2PlatformArgs) {
doCompile(sessionId, daemonReporter, tracer = null) { _, _ ->
execIncrementalCompiler(
k2PlatformArgs as K2JVMCompilerArguments,
@@ -360,7 +360,7 @@ abstract class CompileServiceImplBase(
)
}
}
CompileService.TargetPlatform.JS -> withJsIC {
CompileService.TargetPlatform.JS -> withJsIC(k2PlatformArgs) {
doCompile(sessionId, daemonReporter, tracer = null) { _, _ ->
execJsIncrementalCompiler(
k2PlatformArgs as K2JSCompilerArguments,
@@ -159,7 +159,8 @@ abstract class AbstractFullPipelineModularizedTest : AbstractModularizedTest() {
"java-production",
isTests = false,
emptySet(),
friendDirs = moduleData.friendDirs
friendDirs = moduleData.friendDirs,
isIncrementalCompilation = true
)
val modulesFile = tmp.toFile().resolve("modules.xml")
modulesFile.writeText(builder.asText().toString())
@@ -61,7 +61,6 @@ abstract class IncrementalCompilerRunner<
//TODO(valtman) temporal measure to ensure quick disable, should be deleted after successful release
protected val withSnapshot: Boolean = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false
protected abstract fun isICEnabled(): Boolean
protected abstract fun createCacheManager(args: Args, projectDir: File?): CacheManager
protected abstract fun destinationDir(args: Args): File
@@ -84,7 +83,6 @@ abstract class IncrementalCompilerRunner<
providedChangedFiles: ChangedFiles?,
projectDir: File? = null
): ExitCode {
assert(isICEnabled()) { "Incremental compilation is not enabled" }
var caches = createCacheManager(args, projectDir)
if (withSnapshot) {
@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.isIrBackendEnabled
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -52,7 +53,7 @@ fun makeJsIncrementally(
.filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList()
val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
withJsIC {
withJsIC(args) {
val compiler = IncrementalJsCompilerRunner(
cachesDir, buildReporter,
buildHistoryFile = buildHistoryFile,
@@ -63,11 +64,15 @@ fun makeJsIncrementally(
}
}
inline fun <R> withJsIC(fn: () -> R): R {
@Suppress("DEPRECATION")
inline fun <R> withJsIC(args: CommonCompilerArguments, enabled: Boolean = true, fn: () -> R): R {
val isJsEnabledBackup = IncrementalCompilation.isEnabledForJs()
IncrementalCompilation.setIsEnabledForJs(true)
try {
if (args.incrementalCompilation == null) {
args.incrementalCompilation = enabled
}
return fn()
} finally {
IncrementalCompilation.setIsEnabledForJs(isJsEnabledBackup)
@@ -86,8 +91,6 @@ class IncrementalJsCompilerRunner(
reporter,
buildHistoryFile = buildHistoryFile
) {
override fun isICEnabled(): Boolean =
IncrementalCompilation.isEnabledForJs()
override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
@@ -35,6 +35,7 @@ import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.FilteringMessageCollector
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -83,7 +84,7 @@ fun makeIncrementally(
args.javaSourceRoots = sourceRoots.map { it.absolutePath }.toTypedArray()
val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
withIC {
withIC(args) {
val compiler = IncrementalJvmCompilerRunner(
cachesDir,
buildReporter,
@@ -109,11 +110,15 @@ object EmptyICReporter : ICReporterBase() {
override fun reportMarkDirty(affectedFiles: Iterable<File>, reason: String) {}
}
inline fun <R> withIC(enabled: Boolean = true, fn: () -> R): R {
@Suppress("DEPRECATION")
inline fun <R> withIC(args: CommonCompilerArguments, enabled: Boolean = true, fn: () -> R): R {
val isEnabledBackup = IncrementalCompilation.isEnabledForJvm()
IncrementalCompilation.setIsEnabledForJvm(enabled)
try {
if (args.incrementalCompilation == null) {
args.incrementalCompilation = enabled
}
return fn()
} finally {
IncrementalCompilation.setIsEnabledForJvm(isEnabledBackup)
@@ -136,9 +141,6 @@ class IncrementalJvmCompilerRunner(
additionalOutputFiles = outputFiles,
buildHistoryFile = buildHistoryFile
) {
override fun isICEnabled(): Boolean =
IncrementalCompilation.isEnabledForJvm()
override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
IncrementalJvmCachesManager(
cacheDirectory,
@@ -618,7 +618,7 @@ class ModulesStructure(
ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
val analysisResult = analyzer.analysisResult
if (IncrementalCompilation.isEnabledForJs()) {
if (compilerConfiguration.getBoolean(CommonConfigurationKeys.INCREMENTAL_COMPILATION)) {
/** can throw [IncrementalNextRoundException] */
compareMetadataAndGoToNextICRoundIfNeeded(analysisResult, compilerConfiguration, project, files, errorPolicy.allowErrors)
}
+2
View File
@@ -65,6 +65,8 @@ where advanced options include:
Use 'warning' level to issue warnings instead of errors.
-Xextended-compiler-checks Enable additional compiler checks that might provide verbose diagnostic information for certain errors.
Warning: this mode is not backward-compatible and might cause compilation errors in previously compiled code.
-Xenable-incremental-compilation
Enable incremental compilation
-Xinference-compatibility Enable compatibility changes for generic type inference algorithm
-Xinline-classes Enable experimental inline classes
-Xintellij-plugin-root=<path> Path to the kotlin-compiler.jar or directory where IntelliJ configuration files can be found
+2
View File
@@ -170,6 +170,8 @@ where advanced options include:
Use 'warning' level to issue warnings instead of errors.
-Xextended-compiler-checks Enable additional compiler checks that might provide verbose diagnostic information for certain errors.
Warning: this mode is not backward-compatible and might cause compilation errors in previously compiled code.
-Xenable-incremental-compilation
Enable incremental compilation
-Xinference-compatibility Enable compatibility changes for generic type inference algorithm
-Xinline-classes Enable experimental inline classes
-Xintellij-plugin-root=<path> Path to the kotlin-compiler.jar or directory where IntelliJ configuration files can be found
@@ -25,23 +25,26 @@ public class IncrementalCompilation {
public static final String INCREMENTAL_COMPILATION_JS_PROPERTY = "kotlin.incremental.compilation.js";
public static boolean isEnabledForJvm() {
return "true".equals(System.getProperty(INCREMENTAL_COMPILATION_JVM_PROPERTY));
return Boolean.valueOf(System.getProperty(INCREMENTAL_COMPILATION_JVM_PROPERTY));
}
public static boolean isEnabledForJs() {
return "true".equals(System.getProperty(INCREMENTAL_COMPILATION_JS_PROPERTY));
return Boolean.valueOf(System.getProperty(INCREMENTAL_COMPILATION_JS_PROPERTY));
}
@Deprecated
@TestOnly
public static void setIsEnabledForJvm(boolean value) {
System.setProperty(INCREMENTAL_COMPILATION_JVM_PROPERTY, String.valueOf(value));
}
@Deprecated
@TestOnly
public static void setIsEnabledForJs(boolean value) {
System.setProperty(INCREMENTAL_COMPILATION_JS_PROPERTY, String.valueOf(value));
}
@Deprecated
public static void toJvmArgs(List<String> jvmArgs) {
if (isEnabledForJvm()) addJvmSystemFlag(jvmArgs, INCREMENTAL_COMPILATION_JVM_PROPERTY);
if (isEnabledForJs()) addJvmSystemFlag(jvmArgs, INCREMENTAL_COMPILATION_JS_PROPERTY);
@@ -319,7 +319,7 @@ class GenerateIrRuntime {
val cleanBuildStart = System.nanoTime()
withJsIC {
withJsIC(args) {
val buildHistoryFile = File(cachesDir, "build-history.bin")
val compiler = IncrementalJsCompilerRunner(
cachesDir, BuildReporter(EmptyICReporter, DoNothingBuildMetricsReporter),
@@ -363,7 +363,7 @@ class GenerateIrRuntime {
done,
update
) {
withJsIC {
withJsIC(args) {
val buildHistoryFile = File(cachesDir, "build-history.bin")
val compiler = IncrementalJsCompilerRunner(
cachesDir, BuildReporter(EmptyICReporter, DoNothingBuildMetricsReporter),