From 52b3cb362bbe3173515857aae17e82714c19cad1 Mon Sep 17 00:00:00 2001 From: Georgy Bronnikov Date: Thu, 5 Nov 2020 09:59:38 +0300 Subject: [PATCH] IR: thread pool in PerformByIrFilePhase --- .../arguments/K2JVMCompilerArguments.kt | 7 ++-- .../jetbrains/kotlin/cli/jvm/jvmArguments.kt | 2 +- .../kotlin/config/CommonConfigurationKeys.kt | 4 +-- .../backend/common/phaser/PhaseBuilders.kt | 36 +++++++++++-------- compiler/testData/cli/jvm/extraHelp.out | 4 ++- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JVMCompilerArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JVMCompilerArguments.kt index bb910106471..926e80406d6 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JVMCompilerArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JVMCompilerArguments.kt @@ -115,10 +115,11 @@ class K2JVMCompilerArguments : CommonCompilerArguments() { var doNotClearBindingContext: Boolean by FreezableVar(false) @Argument( - value = "-Xir-run-lowerings-in-parallel", - description = "When using the IR backend, run lowerings for each file in parallel" + value = "-Xir-threads-for-file-lowerings", + description = "When using the IR backend, run lowerings by file in N parallel threads.\n" + + "Default value is 1" ) - var runLoweringsInParallel: Boolean by FreezableVar(false) + var threadsForFileLowerings: String by FreezableVar("1") @Argument(value = "-Xmodule-path", valueDescription = "", description = "Paths where to find Java 9+ modules") var javaModulePath: String? by NullableStringFreezableVar(null) diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/jvmArguments.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/jvmArguments.kt index 196e5915734..932a6482b19 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/jvmArguments.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/jvmArguments.kt @@ -292,7 +292,7 @@ fun CompilerConfiguration.configureAdvancedJvmOptions(arguments: K2JVMCompilerAr arguments.declarationsOutputPath?.let { put(JVMConfigurationKeys.DECLARATIONS_JSON_PATH, it) } - put(CommonConfigurationKeys.RUN_LOWERINGS_IN_PARALLEL, arguments.runLoweringsInParallel) + put(CommonConfigurationKeys.THREADS_FOR_FILE_LOWERINGS, arguments.threadsForFileLowerings.toIntOrNull() ?: 1) } fun CompilerConfiguration.configureKlibPaths(arguments: K2JVMCompilerArguments) { diff --git a/compiler/config/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.kt b/compiler/config/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.kt index f3eddf8e8ae..9e77280ee7a 100644 --- a/compiler/config/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.kt +++ b/compiler/config/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.kt @@ -52,8 +52,8 @@ object CommonConfigurationKeys { val USE_FIR_EXTENDED_CHECKERS = CompilerConfigurationKey.create("fir extended checkers") @JvmField - val RUN_LOWERINGS_IN_PARALLEL = - CompilerConfigurationKey.create("When using the IR backend, run lowerings for each file in parallel") + val THREADS_FOR_FILE_LOWERINGS = + CompilerConfigurationKey.create("When using the IR backend, run lowerings by file in N parallel threads") } var CompilerConfiguration.languageVersionSettings: LanguageVersionSettings diff --git a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseBuilders.kt b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseBuilders.kt index a9385ff7f24..89e0b3b50b2 100644 --- a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseBuilders.kt +++ b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/phaser/PhaseBuilders.kt @@ -13,8 +13,9 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrModuleFragment -import java.lang.IllegalStateException -import kotlin.concurrent.thread +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference // Phase composition. private class CompositePhase( @@ -120,10 +121,13 @@ private class PerformByIrFilePhase( phaserState: PhaserState, context: Context, input: IrModuleFragment - ): IrModuleFragment = if (context.configuration.getBoolean(CommonConfigurationKeys.RUN_LOWERINGS_IN_PARALLEL)) - invokeParallel(phaseConfig, phaserState, context, input) - else - invokeSequential(phaseConfig, phaserState, context, input) + ): IrModuleFragment { + val nThreads = context.configuration.get(CommonConfigurationKeys.THREADS_FOR_FILE_LOWERINGS) ?: 1 + return if (nThreads > 1) + invokeParallel(phaseConfig, phaserState, context, input, nThreads) + else + invokeSequential(phaseConfig, phaserState, context, input) + } private fun invokeSequential( phaseConfig: PhaseConfig, phaserState: PhaserState, context: Context, input: IrModuleFragment @@ -144,31 +148,35 @@ private class PerformByIrFilePhase( } private fun invokeParallel( - phaseConfig: PhaseConfig, phaserState: PhaserState, context: Context, input: IrModuleFragment + phaseConfig: PhaseConfig, phaserState: PhaserState, context: Context, input: IrModuleFragment, nThreads: Int ): IrModuleFragment { if (input.files.isEmpty()) return input // We can only report one exception through ISE - var thrownFromThread: Throwable? = null + val thrownFromThread = AtomicReference?>(null) // Each thread needs its own copy of phaserState.alreadyDone val filesAndStates = input.files.map { it to phaserState.clone() } - val threads = filesAndStates.map { (irFile, state) -> - thread { + + val executor = Executors.newFixedThreadPool(nThreads) + for ((irFile, state) in filesAndStates) { + executor.execute { try { val filePhaserState = state.changeType() for (phase in lower) { phase.invoke(phaseConfig, filePhaserState, context, irFile) } } catch (e: Throwable) { - thrownFromThread = e + thrownFromThread.set(Pair(e, irFile)) } } } + executor.shutdown() + executor.awaitTermination(1, TimeUnit.DAYS) // Wait long enough - threads.forEach { it.join() } - - if (thrownFromThread != null) throw IllegalStateException("Exception in file lowering", thrownFromThread) + thrownFromThread.get()?.let { (e, irFile) -> + CodegenUtil.reportBackendException(e, "IrLowering", irFile.fileEntry.name) + } // Presumably each thread has run through the same list of phases. phaserState.alreadyDone.addAll(filesAndStates[0].second.alreadyDone) diff --git a/compiler/testData/cli/jvm/extraHelp.out b/compiler/testData/cli/jvm/extraHelp.out index 836d1d04734..374d825bcbd 100644 --- a/compiler/testData/cli/jvm/extraHelp.out +++ b/compiler/testData/cli/jvm/extraHelp.out @@ -99,7 +99,6 @@ where advanced options include: -Xsam-conversions={class|indy} Select code generation scheme for SAM conversions. -Xsam-conversions=indy Generate SAM conversions using `invokedynamic` with `LambdaMetafactory.metafactory`. Requires `-jvm-target 1.8` or greater. -Xsam-conversions=class Generate SAM conversions as explicit classes - -Xir-run-lowerings-in-parallel When using the IR backend, run lowerings for each file in parallel -Xsanitize-parentheses Transform '(' and ')' in method names to some other character sequence. This mode can BREAK BINARY COMPATIBILITY and is only supposed to be used to workaround problems with parentheses in identifiers on certain platforms @@ -123,6 +122,9 @@ where advanced options include: Suppress deprecation warning about deprecated JVM target versions -Xsuppress-missing-builtins-error Suppress the "cannot access built-in declaration" error (useful with -no-stdlib) + -Xir-threads-for-file-lowerings + When using the IR backend, run lowerings by file in N parallel threads. + Default value is 1 -Xuse-ir Use the IR backend -Xuse-javac Use javac for Java source and class files analysis -Xuse-old-backend Use the old JVM backend