From 4aea9b349c11e34b60b89921f88fa5e9069e66bb Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Tue, 11 Jul 2017 23:12:20 +0300 Subject: [PATCH] Refactor incremental services --- .../jetbrains/kotlin/cli/js/K2JSCompiler.java | 33 ++++- .../jps/build/AbstractLookupTrackerTest.kt | 124 ++++++++---------- .../AbstractJsProtoComparisonTest.kt | 4 +- .../incremental/js/IncrementalDataProvider.kt | 11 +- .../js/IncrementalResultsConsumer.kt | 8 +- .../incremental/js/TranslationResultValue.kt | 20 +++ .../js/analyze/TopDownAnalyzerFacadeForJS.kt | 2 +- .../jetbrains/kotlin/js/test/BasicBoxTest.kt | 75 +++++------ 8 files changed, 148 insertions(+), 129 deletions(-) create mode 100644 js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java b/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java index a3356032110..4ea118b349f 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java @@ -16,9 +16,11 @@ package org.jetbrains.kotlin.cli.js; +import com.google.common.collect.Lists; import com.intellij.openapi.Disposable; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.ExceptionUtil; @@ -47,6 +49,7 @@ import org.jetbrains.kotlin.config.CompilerConfiguration; import org.jetbrains.kotlin.config.ContentRootsKt; import org.jetbrains.kotlin.config.Services; import org.jetbrains.kotlin.incremental.components.LookupTracker; +import org.jetbrains.kotlin.incremental.js.TranslationResultValue; import org.jetbrains.kotlin.js.analyze.TopDownAnalyzerFacadeForJS; import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult; import org.jetbrains.kotlin.js.config.EcmaVersion; @@ -71,6 +74,7 @@ import org.jetbrains.kotlin.utils.StringsKt; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.*; import java.util.stream.Collectors; @@ -115,12 +119,33 @@ public class K2JSCompiler extends CLICompiler { K2JSTranslator translator = new K2JSTranslator(config); IncrementalDataProvider incrementalDataProvider = config.getConfiguration().get(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER); if (incrementalDataProvider != null) { - List translationUnits = new ArrayList<>(); + Map nonCompiledSources = new HashMap(allKotlinFiles.size()); for (KtFile ktFile : allKotlinFiles) { - translationUnits.add(new TranslationUnit.SourceFile(ktFile)); + nonCompiledSources.put(VfsUtilCore.virtualToIoFile(ktFile.getVirtualFile()), ktFile); } - for (byte[] binaryTree : incrementalDataProvider.getBinaryTrees()) { - translationUnits.add(new TranslationUnit.BinaryAst(binaryTree)); + + Map compiledParts = incrementalDataProvider.getCompiledPackageParts(); + + File[] allSources = new File[compiledParts.size() + allKotlinFiles.size()]; + int i = 0; + for (File file : compiledParts.keySet()) { + allSources[i++] = file; + } + for (File file : nonCompiledSources.keySet()) { + allSources[i++] = file; + } + Arrays.sort(allSources); + + List translationUnits = new ArrayList<>(); + for (i = 0; i < allSources.length; i++) { + KtFile nonCompiled = nonCompiledSources.get(allSources[i]); + if (nonCompiled != null) { + translationUnits.add(new TranslationUnit.SourceFile(nonCompiled)); + } + else { + TranslationResultValue translatedValue = compiledParts.get(allSources[i]); + translationUnits.add(new TranslationUnit.BinaryAst(translatedValue.getBinaryAst())); + } } return translator.translateUnits(reporter, translationUnits, mainCallParameters, jsAnalysisResult); } diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt index bd661cac9bb..da8ebc68ebb 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt @@ -21,7 +21,6 @@ import com.intellij.util.containers.HashMap import com.intellij.util.containers.StringInterner import org.jetbrains.kotlin.TestWithWorkingDir import org.jetbrains.kotlin.build.JvmSourceRoot -import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler @@ -33,14 +32,12 @@ import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.Position import org.jetbrains.kotlin.incremental.components.ScopeKind import org.jetbrains.kotlin.incremental.isKotlinFile +import org.jetbrains.kotlin.incremental.js.* import org.jetbrains.kotlin.incremental.makeModuleFile import org.jetbrains.kotlin.incremental.testingUtils.TouchPolicy import org.jetbrains.kotlin.incremental.testingUtils.copyTestSources import org.jetbrains.kotlin.incremental.testingUtils.getModificationsToPerform import org.jetbrains.kotlin.incremental.utils.TestMessageCollector -import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider -import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer -import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl import org.jetbrains.kotlin.jps.incremental.runJSCompiler import org.jetbrains.kotlin.jps.incremental.createTestingCompilerEnvironment import org.jetbrains.kotlin.test.KotlinTestUtils @@ -48,6 +45,34 @@ import java.io.* import java.util.* abstract class AbstractJvmLookupTrackerTest : AbstractLookupTrackerTest() { + + private val sourceToOutputMapping = hashMapOf>() + + override fun setUp() { + super.setUp() + sourceToOutputMapping.clear() + } + + override fun markDirty(removedAndModifiedSources: Iterable) { + for (sourceFile in removedAndModifiedSources) { + val outputs = sourceToOutputMapping.remove(sourceFile) ?: continue + for (output in outputs) { + output.delete() + } + } + } + + override fun processCompilationResults(outputItemsCollector: OutputItemsCollectorImpl, services: Services) { + for ((sourceFiles, outputFile) in outputItemsCollector.outputs) { + if (outputFile.extension == "kotlin_module") continue + + for (sourceFile in sourceFiles) { + val outputsForSource = sourceToOutputMapping.getOrPut(sourceFile) { hashSetOf() } + outputsForSource.add(outputFile) + } + } + } + override fun runCompiler(filesToCompile: Iterable, env: JpsCompilerEnvironment): Any? { val moduleFile = makeModuleFile( name = "test", @@ -81,66 +106,40 @@ abstract class AbstractJvmLookupTrackerTest : AbstractLookupTrackerTest() { } abstract class AbstractJsLookupTrackerTest : AbstractLookupTrackerTest() { - private lateinit var incrementalDataDir: File - private lateinit var binaryTreesDir: File - private lateinit var packagesMetadataDir: File - private lateinit var headerMetadataFile: File + private var header: ByteArray? = null + private val packageParts: MutableMap = hashMapOf() override fun setUp() { super.setUp() - incrementalDataDir = File(workingDir, "incremental-data") - binaryTreesDir = File(incrementalDataDir, "binary-trees") - packagesMetadataDir = File(incrementalDataDir, "packages-metadata") - headerMetadataFile = File(incrementalDataDir, "header.metadata") + header = null + packageParts.clear() } override fun Services.Builder.registerAdditionalServices() { - if (incrementalDataDir.exists()) { - register(IncrementalDataProvider::class.java, object : IncrementalDataProvider { - override val headerMetadata: ByteArray - get() = headerMetadataFile.readBytes() - override val packagePartsMetadata: List - get() = packagesMetadataDir.walk().filter { it.isFile }.map { it.readBytes() }.toList() - override val binaryTrees: List - get() = binaryTreesDir.walk().filter { it.isFile }.map { it.readBytes() }.toList() - }) + if (header != null) { + register(IncrementalDataProvider::class.java, IncrementalDataProviderImpl(header!!, packageParts!!)) } register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl()) } + override fun markDirty(removedAndModifiedSources: Iterable) { + removedAndModifiedSources.forEach { packageParts.remove(it) } + } + + override fun processCompilationResults(outputItemsCollector: OutputItemsCollectorImpl, services: Services) { + val incrementalResults = services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl + header = incrementalResults.headerMetadata + packageParts.putAll(incrementalResults.packageParts) + } + override fun runCompiler(filesToCompile: Iterable, env: JpsCompilerEnvironment): Any? { val args = K2JSCompilerArguments().apply { outputFile = File(outDir, "out.js").canonicalPath reportOutputFiles = true freeArgs.addAll(filesToCompile.map { it.canonicalPath }) } - val exitCode = runJSCompiler(args, env) - - if (exitCode != ExitCode.OK) return exitCode - - val incrementalResults = env.services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl - incrementalResults.apply { - packageParts.forEach { - val relativePath = it.sourceFile.toRelativeString(srcDir) - val treeFile = File(binaryTreesDir, relativePath + ".ast").apply { parentFile.mkdirs() } - treeFile.writeBytes(it.binaryAst) - - val partProtoFile = File(packagesMetadataDir, relativePath + ".proto").apply { parentFile.mkdirs() } - partProtoFile.writeBytes(it.proto) - - env.outputItemsCollector.outputs.apply { - val sources = listOf(it.sourceFile) - add(SimpleOutputItem(sources, treeFile)) - add(SimpleOutputItem(sources, partProtoFile)) - } - } - - headerMetadataFile.parentFile.mkdirs() - headerMetadataFile.writeBytes(headerMetadata) - } - - return exitCode + return runJSCompiler(args, env) } } @@ -167,6 +166,8 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() { super.tearDown() } + protected abstract fun markDirty(removedAndModifiedSources: Iterable) + protected abstract fun processCompilationResults(outputItemsCollector: OutputItemsCollectorImpl, services: Services) protected abstract fun runCompiler(filesToCompile: Iterable, env: JpsCompilerEnvironment): Any? fun doTest(path: String) { @@ -198,7 +199,6 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() { val testDir = File(path) val workToOriginalFileMap = HashMap(copyTestSources(testDir, srcDir, filePrefix = "")) var dirtyFiles = srcDir.walk().filterTo(HashSet()) { it.isKotlinFile() } - val incrementalData = IncrementalData() val steps = getModificationsToPerform(testDir, moduleNames = null, allowNoFilesWithSuffixInTestData = true, touchPolicy = TouchPolicy.CHECKSUM) .filter { it.isNotEmpty() } @@ -206,14 +206,14 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() { fun CompilerOutput.originalFilesToLookups() = compiledFiles.associateBy({ workToOriginalFileMap[it]!! }, { lookups[it] ?: emptyList() }) - make(dirtyFiles, incrementalData).apply { + make(dirtyFiles).apply { logOutput("INITIAL BUILD") filesToLookups.add(originalFilesToLookups()) } for ((i, modifications) in steps.withIndex()) { dirtyFiles = modifications.mapNotNullTo(HashSet()) { it.perform(workingDir, workToOriginalFileMap) } - make(dirtyFiles, incrementalData).apply { + make(dirtyFiles).apply { logOutput("STEP ${i + 1}") filesToLookups.add(originalFilesToLookups()) } @@ -237,22 +237,13 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() { val compiledFiles: Iterable, val lookups: Map> ) - private class IncrementalData(val sourceToOutput: MutableMap> = hashMapOf()) - private fun make( - filesToCompile: Iterable, - incrementalData: IncrementalData - ): CompilerOutput { + private fun make(filesToCompile: Iterable): CompilerOutput { filesToCompile.forEach { it.writeText(it.readText().replace(COMMENT_WITH_LOOKUP_INFO, "")) } - for (dirtyFile in filesToCompile) { - incrementalData.sourceToOutput.remove(dirtyFile)?.forEach { - it.delete() - } - } - + markDirty(filesToCompile) val lookupTracker = TestLookupTracker() val messageCollector = TestMessageCollector() val outputItemsCollector = OutputItemsCollectorImpl() @@ -263,16 +254,7 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() { } val environment = createTestingCompilerEnvironment(messageCollector, outputItemsCollector, services) val exitCode = runCompiler(filesToCompile, environment) - - for (output in outputItemsCollector.outputs) { - val outputFile = output.outputFile - if (outputFile.extension == "kotlin_module") continue - - for (sourceFile in output.sourceFiles) { - val outputsForSource = incrementalData.sourceToOutput.getOrPut(sourceFile) { hashSetOf() } - outputsForSource.add(outputFile) - } - } + processCompilationResults(outputItemsCollector, environment.services) val lookups = lookupTracker.lookups.groupBy { File(it.filePath) } val lookupsFromCompiledFiles = filesToCompile.associate { it to (lookups[it] ?: emptyList()) } diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt index 4359b01d3e0..0d501069908 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt @@ -68,8 +68,8 @@ abstract class AbstractJsProtoComparisonTest : AbstractProtoComparisonTest() - for ((sourceFile, protoBytes, _) in incrementalResults.packageParts) { - classes.putAll(getProtoData(sourceFile, protoBytes)) + for ((sourceFile, translationResult) in incrementalResults.packageParts) { + classes.putAll(getProtoData(sourceFile, translationResult.metadata)) } return classes diff --git a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt index be2098f12e1..e6096fd21e1 100644 --- a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt +++ b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt @@ -16,18 +16,17 @@ package org.jetbrains.kotlin.incremental.js +import java.io.File + // byte arrays are used to simplify passing to different classloaders interface IncrementalDataProvider { /** gets header metadata (serialized [JsProtoBuf.Header]) from previous compilation */ val headerMetadata: ByteArray - /** gets non-dirty package parts metadata (serialized [ProtoBuf.PackageFragment]) from previous compilation */ - val packagePartsMetadata: List - /** gets non-dirty package parts binary trees from previous compilation */ - val binaryTrees: List + /** gets non-dirty package parts data from previous compilation */ + val compiledPackageParts: Map } class IncrementalDataProviderImpl( override val headerMetadata: ByteArray, - override val packagePartsMetadata: List, - override val binaryTrees: List + override val compiledPackageParts: Map ) : IncrementalDataProvider \ No newline at end of file diff --git a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt index aa28e8c7c3a..04b5b2b0d68 100644 --- a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt +++ b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt @@ -29,8 +29,8 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer { lateinit var headerMetadata: ByteArray private set - private val _packageParts = arrayListOf() - val packageParts: List + private val _packageParts = hashMapOf() + val packageParts: Map get() = _packageParts override fun processHeader(headerMetadata: ByteArray) { @@ -38,9 +38,7 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer { } override fun processPackagePart(sourceFile: File, packagePartMetadata: ByteArray, binaryAst: ByteArray) { - _packageParts.add(PackagePartData(sourceFile, packagePartMetadata, binaryAst)) + _packageParts.put(sourceFile, TranslationResultValue(packagePartMetadata, binaryAst)) } - - data class PackagePartData(val sourceFile: File, val proto: ByteArray, val binaryAst: ByteArray) } diff --git a/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt new file mode 100644 index 00000000000..023488a860f --- /dev/null +++ b/js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.js + +data class TranslationResultValue(val metadata: ByteArray, val binaryAst: ByteArray) + diff --git a/js/js.frontend/src/org/jetbrains/kotlin/js/analyze/TopDownAnalyzerFacadeForJS.kt b/js/js.frontend/src/org/jetbrains/kotlin/js/analyze/TopDownAnalyzerFacadeForJS.kt index cb7ae3275d5..8df9b91e96f 100644 --- a/js/js.frontend/src/org/jetbrains/kotlin/js/analyze/TopDownAnalyzerFacadeForJS.kt +++ b/js/js.frontend/src/org/jetbrains/kotlin/js/analyze/TopDownAnalyzerFacadeForJS.kt @@ -68,7 +68,7 @@ object TopDownAnalyzerFacadeForJS { ): JsAnalysisResult { val lookupTracker = config.configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER) ?: LookupTracker.DO_NOTHING val packageFragment = config.configuration[JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER]?.let { - val metadata = PackagesWithHeaderMetadata(it.headerMetadata, it.packagePartsMetadata) + val metadata = PackagesWithHeaderMetadata(it.headerMetadata, it.compiledPackageParts.values.map { it.metadata }) KotlinJavascriptSerializationUtil.readDescriptors(metadata, moduleContext.storageManager, moduleContext.module, diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt index b573033f2e8..15d9a359756 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt @@ -47,6 +47,7 @@ import org.jetbrains.kotlin.js.parser.parse import org.jetbrains.kotlin.js.parser.sourcemaps.* import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder +import org.jetbrains.kotlin.incremental.js.TranslationResultValue import org.jetbrains.kotlin.js.test.utils.* import org.jetbrains.kotlin.js.util.TextOutputImpl import org.jetbrains.kotlin.psi.KtFile @@ -286,17 +287,19 @@ abstract class BasicBoxTest( JsTestUtils.getFilesInDirectoryByExtension(baseDir + "/", KotlinFileType.EXTENSION) } val additionalFiles = globalCommonFiles + localCommonFiles + additionalCommonFiles - val psiFiles = createPsiFiles(testFiles + additionalFiles) + val allSourceFiles = (testFiles + additionalFiles).map(::File) + val psiFiles = createPsiFiles(allSourceFiles.sortedBy { it.canonicalPath }.map { it.canonicalPath }) val sourceDirs = (testFiles + additionalFiles).map { File(it).parent }.distinct() - val config = createConfig(sourceDirs, module, dependencies, friends, multiModule, additionalMetadata = null) + val config = createConfig(sourceDirs, module, dependencies, friends, multiModule, incrementalData = null) val outputFile = File(outputFileName) - translateFiles(psiFiles.map(TranslationUnit::SourceFile), outputFile, config, outputPrefixFile, outputPostfixFile, mainCallParameters) + val incrementalData = IncrementalData() + translateFiles(psiFiles.map(TranslationUnit::SourceFile), outputFile, config, outputPrefixFile, outputPostfixFile, mainCallParameters, incrementalData) if (module.hasFilesToRecompile) { - checkIncrementalCompilation(sourceDirs, module, kotlinFiles, additionalFiles, dependencies, friends, multiModule, outputFile, - outputPrefixFile, outputPostfixFile, mainCallParameters) + checkIncrementalCompilation(sourceDirs, module, kotlinFiles, dependencies, friends, multiModule, outputFile, + outputPrefixFile, outputPostfixFile, mainCallParameters, incrementalData) } } @@ -304,38 +307,35 @@ abstract class BasicBoxTest( sourceDirs: List, module: TestModule, kotlinFiles: List, - additionalFiles: List, dependencies: List, friends: List, multiModule: Boolean, outputFile: File, outputPrefixFile: File?, outputPostfixFile: File?, - mainCallParameters: MainCallParameters + mainCallParameters: MainCallParameters, + incrementalData: IncrementalData ) { - val incrementalDir = File(outputFile.parentFile, "incremental/" + outputFile.nameWithoutExtension) - val serializedMetadata = mutableListOf() - val translationUnits = kotlinFiles.withIndex().map { (index, file) -> - if (file.recompile) { - TranslationUnit.SourceFile(createPsiFile(file.fileName)) - } - else { - serializedMetadata += File(incrementalDir, "$index.$METADATA_EXTENSION") - val astFile = File(incrementalDir, "$index.$AST_EXTENSION") - TranslationUnit.BinaryAst(FileUtil.loadFileBytes(astFile)) + val sourceToTranslationUnit = hashMapOf() + for (testFile in kotlinFiles) { + if (testFile.recompile) { + val sourceFile = File(testFile.fileName) + incrementalData.translatedFiles.remove(sourceFile) + sourceToTranslationUnit[sourceFile] = TranslationUnit.SourceFile(createPsiFile(testFile.fileName)) } } - val allTranslationUnits = translationUnits + additionalFiles.withIndex().map { (index, _) -> - val astFile = File(incrementalDir, "${index + translationUnits.size}.$AST_EXTENSION") - TranslationUnit.BinaryAst(FileUtil.loadFileBytes(astFile)) + for ((sourceFile, data) in incrementalData.translatedFiles) { + sourceToTranslationUnit[sourceFile] = TranslationUnit.BinaryAst(data.binaryAst) } + val translationUnits = sourceToTranslationUnit.keys + .sortedBy { it.canonicalPath } + .map { sourceToTranslationUnit[it]!! } - val headerFile = File(incrementalDir, HEADER_FILE) - val recompiledConfig = createConfig(sourceDirs, module, dependencies, friends, multiModule, Pair(headerFile,serializedMetadata)) + val recompiledConfig = createConfig(sourceDirs,module, dependencies, friends, multiModule, incrementalData) val recompiledOutputFile = File(outputFile.parentFile, outputFile.nameWithoutExtension + "-recompiled.js") - translateFiles(allTranslationUnits, recompiledOutputFile, recompiledConfig, outputPrefixFile, outputPostfixFile, - mainCallParameters) + translateFiles(translationUnits, recompiledOutputFile, recompiledConfig, outputPrefixFile, outputPostfixFile, + mainCallParameters, incrementalData) val originalOutput = FileUtil.loadFile(outputFile) val recompiledOutput = removeRecompiledSuffix(FileUtil.loadFile(recompiledOutputFile)) @@ -349,13 +349,16 @@ abstract class BasicBoxTest( private fun removeRecompiledSuffix(text: String): String = text.replace("-recompiled.js", ".js") + class IncrementalData(var header: ByteArray? = null, val translatedFiles: MutableMap = hashMapOf()) + protected fun translateFiles( units: List, outputFile: File, config: JsConfig, outputPrefixFile: File?, outputPostfixFile: File?, - mainCallParameters: MainCallParameters + mainCallParameters: MainCallParameters, + incrementalData: IncrementalData ) { val translator = K2JSTranslator(config) val translationResult = translator.translateUnits(ExceptionThrowingReporter, units, mainCallParameters) @@ -390,14 +393,11 @@ abstract class BasicBoxTest( config.configuration[JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER]?.let { val incrementalService = it as IncrementalResultsConsumerImpl - val incrementalDir = File(outputDir, "incremental/${outputFile.nameWithoutExtension}") - - for ((i, packagePart) in incrementalService.packageParts.withIndex()) { - FileUtil.writeToFile(File(incrementalDir, "$i.$AST_EXTENSION"), packagePart.binaryAst) - FileUtil.writeToFile(File(incrementalDir, "$i.$METADATA_EXTENSION"), packagePart.proto) + for ((srcFile, data) in incrementalService.packageParts) { + incrementalData.translatedFiles[srcFile] = data } - FileUtil.writeToFile(File(incrementalDir, HEADER_FILE), incrementalService.headerMetadata) + incrementalData.header = incrementalService.headerMetadata } processJsProgram(translationResult.program, units.filterIsInstance().map { it.file }) @@ -472,9 +472,7 @@ abstract class BasicBoxTest( private fun createPsiFiles(fileNames: List): List = fileNames.map(this::createPsiFile) private fun createConfig( - sourceDirs: List, - module: TestModule, dependencies: List, friends: List, multiModule: Boolean, - additionalMetadata: Pair>? + sourceDirs: List,module: TestModule, dependencies: List, friends: List, multiModule: Boolean, incrementalData: IncrementalData? ): JsConfig { val configuration = environment.configuration.copy() @@ -494,10 +492,10 @@ abstract class BasicBoxTest( val hasFilesToRecompile = module.hasFilesToRecompile configuration.put(JSConfigurationKeys.META_INFO, multiModule) if (hasFilesToRecompile) { - if (additionalMetadata != null) { - val (headerFile, packagePartFiles) = additionalMetadata + val header = incrementalData?.header + if (header != null) { configuration.put(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER, - IncrementalDataProviderImpl(headerFile.readBytes(), packagePartFiles.map { it.readBytes() }, emptyList())) + IncrementalDataProviderImpl(header, incrementalData.translatedFiles)) } configuration.put(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER, IncrementalResultsConsumerImpl()) @@ -660,9 +658,6 @@ abstract class BasicBoxTest( private val EXPECTED_REACHABLE_NODES = Pattern.compile("^// *$EXPECTED_REACHABLE_NODES_DIRECTIVE: *([0-9]+) *$", Pattern.MULTILINE) private val RECOMPILE_PATTERN = Pattern.compile("^// *RECOMPILE *$", Pattern.MULTILINE) private val SOURCE_MAP_SOURCE_EMBEDDING = Regex("^// *SOURCE_MAP_EMBED_SOURCES: ([A-Z]+)*\$", RegexOption.MULTILINE) - private val AST_EXTENSION = "jsast" - private val METADATA_EXTENSION = "jsmeta" - private val HEADER_FILE = "header.$METADATA_EXTENSION" val TEST_MODULE = "JS_TESTS" private val DEFAULT_MODULE = "main"