azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Build per-file caches in parallel

Browse Source
This commit is contained in:
Igor Chevdar
2023-02-20 15:07:06 +02:00
committed by Space Team
parent 164cbc9968
commit f442936320
3 changed files with 101 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt
+9
View File
@@ -247,6 +247,15 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
@Argument(value = "-Xmake-per-file-cache", description = "Force compiler to produce per-file cache")
var makePerFileCache: Boolean = false
@Argument(
value = "-Xbackend-threads",
valueDescription = "<N>",
description = "Run codegen by file in N parallel threads.\n" +
"0 means use a thread per processor core.\n" +
"Default value is 1"
)
var backendThreads: String = "1"
@Argument(value = "-Xexport-kdoc", description = "Export KDoc in framework header")
var exportKDoc: Boolean = false
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt
+18
View File
@@ -180,6 +180,16 @@ fun CompilerConfiguration.setupFromArguments(arguments: K2NativeCompilerArgument
arguments.autoCacheDir?.let { put(AUTO_CACHE_DIR, it) }
arguments.filesToCache?.let { put(FILES_TO_CACHE, it.toList()) }
put(MAKE_PER_FILE_CACHE, arguments.makePerFileCache)
val nThreadsRaw = parseBackendThreads(arguments.backendThreads)
val availableProcessors = Runtime.getRuntime().availableProcessors()
val nThreads = if (nThreadsRaw == 0) availableProcessors else nThreadsRaw
if (nThreads > 1) {
report(LOGGING, "Running backend in parallel with $nThreads threads")
}
if (nThreads > availableProcessors) {
report(WARNING, "The number of threads $nThreads is more than the number of processors $availableProcessors")
}
put(CommonConfigurationKeys.PARALLEL_BACKEND_THREADS, nThreads)
parseShortModuleName(arguments, this@setupFromArguments, outputKind)?.let {
put(SHORT_MODULE_NAME, it)
@@ -405,6 +415,14 @@ private fun parseLibraryToAddToCache(
}
}
private fun parseBackendThreads(stringValue: String): Int {
val value = stringValue.toIntOrNull()
?: throw KonanCompilationException("Cannot parse -Xbackend-threads value: \"$stringValue\". Please use an integer number")
if (value < 0)
throw KonanCompilationException("-Xbackend-threads value cannot be negative")
return value
}
// TODO: Support short names for current module in ObjC export and lift this limitation.
private fun parseShortModuleName(
arguments: K2NativeCompilerArguments,
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt
+74 -18
View File
@@ -12,6 +12,7 @@ import org.jetbrains.kotlin.backend.konan.driver.utilities.CExportFiles
import org.jetbrains.kotlin.backend.konan.driver.utilities.createTempFiles
import org.jetbrains.kotlin.backend.konan.ir.konanLibrary
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
@@ -24,6 +25,10 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.Family
import org.jetbrains.kotlin.library.impl.javaFile
import org.jetbrains.kotlin.konan.file.File
import java.util.concurrent.Callable
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicReference
internal fun PhaseEngine<PhaseContext>.runFrontend(config: KonanConfig, environment: KotlinCoreEnvironment): FrontendPhaseOutput.Full? {
val frontendOutput = useContext(FrontendContextImpl(config)) { it.runPhase(FrontendPhase, environment) }
@@ -57,16 +62,36 @@ internal fun <C : PhaseContext> PhaseEngine<C>.runBackend(backendContext: Contex
val config = context.config
useContext(backendContext) { backendEngine ->
backendEngine.runPhase(functionsWithoutBoundCheck)
val fragments = backendEngine.splitIntoFragments(irModule)
fragments.forEach { fragment ->
val tempFiles = createTempFiles(config, fragment.cacheDeserializationStrategy)
fun createGenerationStateAndRunLowerings(fragment: BackendJobFragment): NativeGenerationState {
val outputPath = config.cacheSupport.tryGetImplicitOutput(fragment.cacheDeserializationStrategy) ?: config.outputPath
val outputFiles = OutputFiles(outputPath, config.target, config.produce)
val generationState = NativeGenerationState(context.config, backendContext,
fragment.cacheDeserializationStrategy, fragment.dependenciesTracker, fragment.llvmModuleSpecification, outputFiles,
llvmModuleName = "out" // TODO: Currently, all llvm modules are named as "out" which might lead to collisions.
)
try {
val module = fragment.irModule
newEngine(generationState) { generationStateEngine ->
if (context.config.produce.isCache) {
generationStateEngine.runPhase(BuildAdditionalCacheInfoPhase, module)
}
if (context.config.produce == CompilerOutputKind.PROGRAM) {
generationStateEngine.runPhase(EntryPointPhase, module)
}
generationStateEngine.lowerModuleWithDependencies(module)
}
return generationState
} catch (t: Throwable) {
generationState.dispose()
throw t
}
}
fun runAfterLowerings(fragment: BackendJobFragment, generationState: NativeGenerationState) {
val tempFiles = createTempFiles(config, fragment.cacheDeserializationStrategy)
val outputFiles = generationState.outputFiles
try {
val outputPath = config.cacheSupport.tryGetImplicitOutput(fragment.cacheDeserializationStrategy) ?: config.outputPath
val outputFiles = OutputFiles(outputPath, config.target, config.produce)
val generationState = NativeGenerationState(context.config, backendContext,
fragment.cacheDeserializationStrategy, fragment.dependenciesTracker, fragment.llvmModuleSpecification, outputFiles,
llvmModuleName = "out" // TODO: Currently, all llvm modules are named as "out" which might lead to collisions.
)
backendEngine.useContext(generationState) { generationStateEngine ->
val bitcodeFile = tempFiles.create(generationState.llvmModuleName, ".bc").javaFile()
val cExportFiles = if (config.produce.isNativeLibrary) {
@@ -92,6 +117,41 @@ internal fun <C : PhaseContext> PhaseEngine<C>.runBackend(backendContext: Contex
tempFiles.dispose()
}
}
val fragments = backendEngine.splitIntoFragments(irModule)
val nThreads = context.config.configuration.get(CommonConfigurationKeys.PARALLEL_BACKEND_THREADS) ?: 1
if (nThreads == 1) {
fragments.forEach { fragment ->
runAfterLowerings(fragment, createGenerationStateAndRunLowerings(fragment))
}
} else {
val fragmentsList = fragments.toList()
if (fragmentsList.size == 1) {
val fragment = fragmentsList[0]
runAfterLowerings(fragment, createGenerationStateAndRunLowerings(fragment))
} else {
// We'd love to run entire pipeline in parallel, but it's difficult (mainly because of the lowerings,
// which need cross-file access all the time and it's not easy to overcome this). So, for now,
// we split the pipeline into two parts - everything before lowerings (including them)
// which is run sequentially, and everything else which in run in parallel.
val generationStates = fragmentsList.map { fragment -> createGenerationStateAndRunLowerings(fragment) }
val executor = Executors.newFixedThreadPool(nThreads)
val thrownFromThread = AtomicReference<Throwable?>(null)
val tasks = fragmentsList.zip(generationStates).map { (fragment, generationState) ->
Callable {
try {
runAfterLowerings(fragment, generationState)
} catch (t: Throwable) {
thrownFromThread.set(t)
}
}
}
executor.invokeAll(tasks.toList())
executor.shutdown()
executor.awaitTermination(1, TimeUnit.DAYS)
thrownFromThread.get()?.let { throw it }
}
}
}
}
@@ -189,12 +249,6 @@ internal data class ModuleCompilationOutput(
* 5. Serializes it to a bitcode file.
*/
internal fun PhaseEngine<NativeGenerationState>.compileModule(module: IrModuleFragment, bitcodeFile: java.io.File, cExportFiles: CExportFiles?) {
if (context.config.produce.isCache) {
runPhase(BuildAdditionalCacheInfoPhase, module)
}
if (context.config.produce == CompilerOutputKind.PROGRAM) {
runPhase(EntryPointPhase, module)
}
runBackendCodegen(module, cExportFiles)
val checkExternalCalls = context.config.configuration.getBoolean(KonanConfigKeys.CHECK_EXTERNAL_CALLS)
if (checkExternalCalls) {
@@ -253,16 +307,18 @@ internal fun <C : PhaseContext> PhaseEngine<C>.compileAndLink(
}
}
internal fun PhaseEngine<NativeGenerationState>.runBackendCodegen(module: IrModuleFragment, cExportFiles: CExportFiles?) {
internal fun PhaseEngine<NativeGenerationState>.lowerModuleWithDependencies(module: IrModuleFragment) {
runAllLowerings(module)
val dependenciesToCompile = findDependenciesToCompile()
// TODO: KonanLibraryResolver.TopologicalLibraryOrder actually returns libraries in the reverse topological order.
// TODO: Does the order of files really matter with the new MM?
// TODO: Does the order of files really matter with the new MM? (and with lazy top-levels initialization?)
dependenciesToCompile.reversed().forEach { irModule ->
runAllLowerings(irModule)
}
mergeDependencies(module, dependenciesToCompile)
}
internal fun PhaseEngine<NativeGenerationState>.runBackendCodegen(module: IrModuleFragment, cExportFiles: CExportFiles?) {
runCodegen(module)
val generatedBitcodeFiles = if (context.config.produce.isNativeLibrary) {
require(cExportFiles != null)