From 2cad26f4cc4297cfa69a63d286f9da6f814ff04d Mon Sep 17 00:00:00 2001 From: Alexander Korepanov Date: Mon, 27 Feb 2023 17:36:48 +0100 Subject: [PATCH] [JS IR] Use cacheable ZIP file system accessor in JS IR BE Related to KT-51712 --- .../jetbrains/kotlin/cli/js/K2JsIrCompiler.kt | 17 +++++++++++ .../kotlin/ir/backend/js/ic/CacheUpdater.kt | 4 ++- .../backend/common/CommonJsKLibResolver.kt | 23 ++++++++++----- .../jetbrains/kotlin/ir/backend/js/klib.kt | 4 ++- js/js.frontend/build.gradle.kts | 1 + .../kotlin/js/config/JSConfigurationKeys.java | 5 ++++ .../incremental/AbstractInvalidationTest.kt | 11 +++++++ .../js/test/ir/AbstractJsKLibABITestCase.kt | 29 +++++++++++++------ .../AbstractJsKlibBinaryCompatibilityTest.kt | 6 ++-- 9 files changed, 80 insertions(+), 20 deletions(-) diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index be6cb8fd516..fe52a284b30 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.cli.js import com.intellij.openapi.Disposable +import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.text.StringUtil import org.jetbrains.kotlin.backend.common.CompilationException import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig @@ -55,6 +56,8 @@ import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult import org.jetbrains.kotlin.js.config.* +import org.jetbrains.kotlin.konan.file.ZipFileSystemAccessor +import org.jetbrains.kotlin.konan.file.ZipFileSystemCacheableAccessor import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion import org.jetbrains.kotlin.name.FqName @@ -74,6 +77,16 @@ private val K2JSCompilerArguments.granularity: JsGenerationGranularity else -> JsGenerationGranularity.WHOLE_PROGRAM } +private class DisposableZipFileSystemAccessor private constructor( + private val zipAccessor: ZipFileSystemCacheableAccessor +) : Disposable, ZipFileSystemAccessor by zipAccessor { + constructor(cacheLimit: Int) : this(ZipFileSystemCacheableAccessor(cacheLimit)) + + override fun dispose() { + zipAccessor.reset() + } +} + class K2JsIrCompiler : CLICompiler() { override val defaultPerformanceManager: CommonCompilerPerformanceManager = @@ -194,6 +207,10 @@ class K2JsIrCompiler : CLICompiler() { configurationJs.put(JSConfigurationKeys.GENERATE_DTS, arguments.generateDts) configurationJs.put(JSConfigurationKeys.GENERATE_INLINE_ANONYMOUS_FUNCTIONS, arguments.irGenerateInlineAnonymousFunctions) + val zipAccessor = DisposableZipFileSystemAccessor(64) + Disposer.register(rootDisposable, zipAccessor) + configurationJs.put(JSConfigurationKeys.ZIP_FILE_SYSTEM_ACCESSOR, zipAccessor) + if (!checkKotlinPackageUsage(environmentForJS.configuration, sourcesFiles)) return COMPILATION_ERROR val outputDirPath = arguments.outputDir diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index 5874a69827a..52cadb73ffb 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -13,6 +13,7 @@ import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.konan.properties.propertyList import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS import org.jetbrains.kotlin.library.KotlinLibrary @@ -87,7 +88,8 @@ class CacheUpdater( // libraries in topological order: [stdlib, ..., main] val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") { - val allResolvedDependencies = CommonJsKLibResolver.resolve(allModules, compilerConfiguration.resolverLogger) + val zipAccessor = compilerConfiguration.get(JSConfigurationKeys.ZIP_FILE_SYSTEM_ACCESSOR) + val allResolvedDependencies = CommonJsKLibResolver.resolve(allModules, compilerConfiguration.resolverLogger, zipAccessor) val libraries = allResolvedDependencies.getFullList(TopologicalLibraryOrder).let { resolvedLibraries -> val mainLibraryIndex = resolvedLibraries.indexOfLast { diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/CommonJsKLibResolver.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/CommonJsKLibResolver.kt index 4571ecbc457..cbe5d895424 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/CommonJsKLibResolver.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/CommonJsKLibResolver.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.backend.common import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.file.ZipFileSystemAccessor import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.KotlinLibraryProperResolverWithAttributes import org.jetbrains.kotlin.library.UnresolvedLibrary @@ -16,15 +17,21 @@ import org.jetbrains.kotlin.library.metadata.resolver.impl.libraryResolver import org.jetbrains.kotlin.util.Logger object CommonJsKLibResolver { - fun resolve(libraries: Collection, logger: Logger): KotlinLibraryResolveResult = + fun resolve( + libraries: Collection, + logger: Logger, + zipAccessor: ZipFileSystemAccessor? = null + ): KotlinLibraryResolveResult = resolveWithoutDependencies( libraries, - logger + logger, + zipAccessor ).resolveWithDependencies() fun resolveWithoutDependencies( libraries: Collection, - logger: Logger + logger: Logger, + zipAccessor: ZipFileSystemAccessor? ): KLibResolution { val unresolvedLibraries = libraries.map { UnresolvedLibrary(it, null) } val libraryAbsolutePaths = libraries.map { File(it).absolutePath } @@ -35,7 +42,8 @@ object CommonJsKLibResolver { distributionKlib = null, localKotlinDir = null, skipCurrentDir = false, - logger = logger + logger = logger, + zipAccessor = zipAccessor ).libraryResolver() return KLibResolution( @@ -67,7 +75,8 @@ private class KLibResolverHelper( distributionKlib: String?, localKotlinDir: String?, skipCurrentDir: Boolean, - logger: Logger + logger: Logger, + private val zipAccessor: ZipFileSystemAccessor? ) : KotlinLibraryProperResolverWithAttributes( repositories, directLibs, @@ -78,5 +87,5 @@ private class KLibResolverHelper( emptyList() ) { // Stick with the default KotlinLibrary for now. - override fun libraryComponentBuilder(file: File, isDefault: Boolean) = createKotlinLibraryComponents(file, isDefault) -} \ No newline at end of file + override fun libraryComponentBuilder(file: File, isDefault: Boolean) = createKotlinLibraryComponents(file, isDefault, zipAccessor) +} diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index 5a592f83dcf..a82bc258c61 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -47,6 +47,7 @@ import org.jetbrains.kotlin.js.analyze.AbstractTopDownAnalyzerFacadeForWeb import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy import org.jetbrains.kotlin.js.config.JSConfigurationKeys +import org.jetbrains.kotlin.konan.file.ZipFileSystemAccessor import org.jetbrains.kotlin.konan.properties.Properties import org.jetbrains.kotlin.konan.properties.propertyList import org.jetbrains.kotlin.library.* @@ -472,7 +473,8 @@ class ModulesStructure( val allDependenciesResolution = CommonJsKLibResolver.resolveWithoutDependencies( dependencies, - compilerConfiguration.resolverLogger + compilerConfiguration.resolverLogger, + compilerConfiguration.get(JSConfigurationKeys.ZIP_FILE_SYSTEM_ACCESSOR) ) val fullResolvedLibraries by lazy { diff --git a/js/js.frontend/build.gradle.kts b/js/js.frontend/build.gradle.kts index 28ea0d21eff..4b0c6359620 100644 --- a/js/js.frontend/build.gradle.kts +++ b/js/js.frontend/build.gradle.kts @@ -4,6 +4,7 @@ plugins { } dependencies { + implementation(project(":kotlin-util-io")) api(project(":compiler:util")) api(project(":compiler:frontend")) api(project(":core:compiler.common.js")) diff --git a/js/js.frontend/src/org/jetbrains/kotlin/js/config/JSConfigurationKeys.java b/js/js.frontend/src/org/jetbrains/kotlin/js/config/JSConfigurationKeys.java index 9c8afd137c0..5c5f2696de3 100644 --- a/js/js.frontend/src/org/jetbrains/kotlin/js/config/JSConfigurationKeys.java +++ b/js/js.frontend/src/org/jetbrains/kotlin/js/config/JSConfigurationKeys.java @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.js.config; +import org.jetbrains.kotlin.konan.file.ZipFileSystemAccessor; import org.jetbrains.kotlin.config.CompilerConfigurationKey; import org.jetbrains.kotlin.incremental.js.IncrementalNextRoundChecker; import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider; @@ -120,4 +121,8 @@ public class JSConfigurationKeys { public static final CompilerConfigurationKey WASM_GENERATE_WAT = CompilerConfigurationKey.create("generate wat file"); + + public static final CompilerConfigurationKey ZIP_FILE_SYSTEM_ACCESSOR = + CompilerConfigurationKey.create("zip file system accessor, used for klib reading"); + } diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index 22f6d98b7a1..76f27f02914 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -30,6 +30,7 @@ import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.js.test.converters.ClassicJsBackendFacade import org.jetbrains.kotlin.js.test.utils.MODULE_EMULATION_FILE import org.jetbrains.kotlin.js.testOld.V8IrJsTestChecker +import org.jetbrains.kotlin.konan.file.ZipFileSystemCacheableAccessor import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.serialization.js.ModuleKind @@ -61,10 +62,17 @@ abstract class AbstractInvalidationTest( private const val SOURCE_MAPPING_URL_PREFIX = "//# sourceMappingURL=" } + private val zipAccessor = ZipFileSystemCacheableAccessor(2) + override fun createEnvironment(): KotlinCoreEnvironment { return KotlinCoreEnvironment.createForTests(TestDisposable(), CompilerConfiguration(), EnvironmentConfigFiles.JS_CONFIG_FILES) } + override fun tearDown() { + zipAccessor.reset() + super.tearDown() + } + private fun parseProjectInfo(testName: String, infoFile: File): ProjectInfo { return ProjectInfoParser(infoFile).parse(testName) } @@ -127,6 +135,9 @@ abstract class AbstractInvalidationTest( } build() } + + zipAccessor.reset() + copy.put(JSConfigurationKeys.ZIP_FILE_SYSTEM_ACCESSOR, zipAccessor) return copy } diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsKLibABITestCase.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsKLibABITestCase.kt index 313bf598d3e..968d72ec5c3 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsKLibABITestCase.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsKLibABITestCase.kt @@ -33,6 +33,7 @@ import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.js.testOld.V8IrJsTestChecker import org.jetbrains.kotlin.klib.KlibABITestUtils import org.jetbrains.kotlin.klib.KlibABITestUtils.MAIN_MODULE_NAME +import org.jetbrains.kotlin.konan.file.ZipFileSystemCacheableAccessor import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.serialization.js.ModuleKind @@ -54,6 +55,8 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() { private lateinit var buildDir: File private lateinit var environment: KotlinCoreEnvironment + private val zipAccessor = ZipFileSystemCacheableAccessor(2) + override fun setUp() { super.setUp() buildDir = createTempDirectory().toFile().also { it.mkdirs() } @@ -66,10 +69,24 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() { } override fun tearDown() { + zipAccessor.reset() buildDir.deleteRecursively() super.tearDown() } + private fun createConfig(moduleName: String): CompilerConfiguration { + val config = environment.configuration.copy() + config.put(CommonConfigurationKeys.MODULE_NAME, moduleName) + config.put(JSConfigurationKeys.PARTIAL_LINKAGE, true) + config.put(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN) + config.put(JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION, true) + + zipAccessor.reset() + config.put(JSConfigurationKeys.ZIP_FILE_SYSTEM_ACCESSOR, zipAccessor) + + return config + } + private inner class JsTestConfiguration(testPath: String) : KlibABITestUtils.TestConfiguration { override val testDir: File = File(testPath).absoluteFile override val buildDir: File get() = this@AbstractJsKLibABITestCase.buildDir @@ -82,6 +99,7 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() { this@AbstractJsKLibABITestCase.buildBinaryAndRun(mainModuleKlibFile, dependencies) override fun onNonEmptyBuildDirectory(directory: File) { + zipAccessor.reset() directory.listFiles()?.forEach(File::deleteRecursively) } @@ -108,11 +126,9 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() { fun doTest(testPath: String) = KlibABITestUtils.runTest(JsTestConfiguration(testPath)) private fun buildKlib(moduleName: String, moduleSourceDir: File, dependencies: KlibABITestUtils.Dependencies, klibFile: File) { + val config = createConfig(moduleName) val ktFiles = environment.createPsiFiles(moduleSourceDir) - val config = environment.configuration.copy() - config.put(CommonConfigurationKeys.MODULE_NAME, moduleName) - val sourceModule = prepareAnalyzedSourceModule( environment.project, ktFiles, @@ -156,12 +172,7 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() { } private fun buildBinaryAndRun(mainModuleKlibFile: File, allDependencies: KlibABITestUtils.Dependencies) { - val configuration = environment.configuration.copy() - - configuration.put(JSConfigurationKeys.PARTIAL_LINKAGE, true) - configuration.put(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN) - configuration.put(JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION, true) - configuration.put(CommonConfigurationKeys.MODULE_NAME, MAIN_MODULE_NAME) + val configuration = createConfig(MAIN_MODULE_NAME) val compilationOutputs = if (useIncrementalCompiler) buildBinaryWithIC(configuration, mainModuleKlibFile, allDependencies) diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/testOld/compatibility/binary/AbstractJsKlibBinaryCompatibilityTest.kt b/js/js.tests/test/org/jetbrains/kotlin/js/testOld/compatibility/binary/AbstractJsKlibBinaryCompatibilityTest.kt index dd6974cb68b..852162ab4c5 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/testOld/compatibility/binary/AbstractJsKlibBinaryCompatibilityTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/testOld/compatibility/binary/AbstractJsKlibBinaryCompatibilityTest.kt @@ -50,7 +50,9 @@ abstract class AbstractJsKlibBinaryCompatibilityTest : AbstractKlibBinaryCompati return uniqueDependencies } - private val TestModule.jsPath get() = File(workingDir, "${this.name}.js").absolutePath + private val jsOutDir get() = workingDir.resolve("out") + + private val TestModule.jsPath get() = File(jsOutDir, "${this.name}.js").absolutePath private fun createFiles(files: List): List = files.map { @@ -83,7 +85,7 @@ abstract class AbstractJsKlibBinaryCompatibilityTest : AbstractKlibBinaryCompati val args = K2JSCompilerArguments().apply { freeArgs = createFiles(module.files) + runnerFunctionFile() libraries = module.dependenciesToLibrariesArg(version = 2) - outputDir = workingDir.normalize().absolutePath + outputDir = jsOutDir.normalize().absolutePath moduleName = module.name irProduceJs = true irOnly = true