Refactor incremental services

compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java

@@ -16,9 +16,11 @@
 
 package org.jetbrains.kotlin.cli.js;
 
+import com.google.common.collect.Lists;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.ExceptionUtil;
@@ -47,6 +49,7 @@ import org.jetbrains.kotlin.config.CompilerConfiguration;
 import org.jetbrains.kotlin.config.ContentRootsKt;
 import org.jetbrains.kotlin.config.Services;
 import org.jetbrains.kotlin.incremental.components.LookupTracker;
+import org.jetbrains.kotlin.incremental.js.TranslationResultValue;
 import org.jetbrains.kotlin.js.analyze.TopDownAnalyzerFacadeForJS;
 import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult;
 import org.jetbrains.kotlin.js.config.EcmaVersion;
@@ -71,6 +74,7 @@ import org.jetbrains.kotlin.utils.StringsKt;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -115,12 +119,33 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
         K2JSTranslator translator = new K2JSTranslator(config);
         IncrementalDataProvider incrementalDataProvider = config.getConfiguration().get(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER);
         if (incrementalDataProvider != null) {
-            List<TranslationUnit> translationUnits = new ArrayList<>();
+            Map<File, KtFile> nonCompiledSources = new HashMap<File, KtFile>(allKotlinFiles.size());
             for (KtFile ktFile : allKotlinFiles) {
-                translationUnits.add(new TranslationUnit.SourceFile(ktFile));
+                nonCompiledSources.put(VfsUtilCore.virtualToIoFile(ktFile.getVirtualFile()), ktFile);
             }
-            for (byte[] binaryTree : incrementalDataProvider.getBinaryTrees()) {
+
-                translationUnits.add(new TranslationUnit.BinaryAst(binaryTree));
+            Map<File, TranslationResultValue> compiledParts = incrementalDataProvider.getCompiledPackageParts();
+
+            File[] allSources = new File[compiledParts.size() + allKotlinFiles.size()];
+            int i = 0;
+            for (File file : compiledParts.keySet()) {
+                allSources[i++] = file;
+            }
+            for (File file : nonCompiledSources.keySet()) {
+                allSources[i++] = file;
+            }
+            Arrays.sort(allSources);
+
+            List<TranslationUnit> translationUnits = new ArrayList<>();
+            for (i = 0; i < allSources.length; i++) {
+                KtFile nonCompiled = nonCompiledSources.get(allSources[i]);
+                if (nonCompiled != null) {
+                    translationUnits.add(new TranslationUnit.SourceFile(nonCompiled));
+                }
+                else {
+                    TranslationResultValue translatedValue = compiledParts.get(allSources[i]);
+                    translationUnits.add(new TranslationUnit.BinaryAst(translatedValue.getBinaryAst()));
+                }
             }
             return translator.translateUnits(reporter, translationUnits, mainCallParameters, jsAnalysisResult);
         }

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt

@@ -21,7 +21,6 @@ import com.intellij.util.containers.HashMap
 import com.intellij.util.containers.StringInterner
 import org.jetbrains.kotlin.TestWithWorkingDir
 import org.jetbrains.kotlin.build.JvmSourceRoot
-import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
@@ -33,14 +32,12 @@ import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.components.Position
 import org.jetbrains.kotlin.incremental.components.ScopeKind
 import org.jetbrains.kotlin.incremental.isKotlinFile
+import org.jetbrains.kotlin.incremental.js.*
 import org.jetbrains.kotlin.incremental.makeModuleFile
 import org.jetbrains.kotlin.incremental.testingUtils.TouchPolicy
 import org.jetbrains.kotlin.incremental.testingUtils.copyTestSources
 import org.jetbrains.kotlin.incremental.testingUtils.getModificationsToPerform
 import org.jetbrains.kotlin.incremental.utils.TestMessageCollector
-import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
-import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
-import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
 import org.jetbrains.kotlin.jps.incremental.runJSCompiler
 import org.jetbrains.kotlin.jps.incremental.createTestingCompilerEnvironment
 import org.jetbrains.kotlin.test.KotlinTestUtils
@@ -48,6 +45,34 @@ import java.io.*
 import java.util.*
 
 abstract class AbstractJvmLookupTrackerTest : AbstractLookupTrackerTest() {
+
+    private val sourceToOutputMapping = hashMapOf<File, MutableSet<File>>()
+
+    override fun setUp() {
+        super.setUp()
+        sourceToOutputMapping.clear()
+    }
+
+    override fun markDirty(removedAndModifiedSources: Iterable<File>) {
+        for (sourceFile in removedAndModifiedSources) {
+            val outputs = sourceToOutputMapping.remove(sourceFile) ?: continue
+            for (output in outputs) {
+                output.delete()
+            }
+        }
+    }
+
+    override fun processCompilationResults(outputItemsCollector: OutputItemsCollectorImpl, services: Services) {
+        for ((sourceFiles, outputFile) in outputItemsCollector.outputs) {
+            if (outputFile.extension == "kotlin_module") continue
+
+            for (sourceFile in sourceFiles) {
+                val outputsForSource = sourceToOutputMapping.getOrPut(sourceFile) { hashSetOf() }
+                outputsForSource.add(outputFile)
+            }
+        }
+    }
+
     override fun runCompiler(filesToCompile: Iterable<File>, env: JpsCompilerEnvironment): Any? {
         val moduleFile = makeModuleFile(
                 name = "test",
@@ -81,66 +106,40 @@ abstract class AbstractJvmLookupTrackerTest : AbstractLookupTrackerTest() {
 }
 
 abstract class AbstractJsLookupTrackerTest : AbstractLookupTrackerTest() {
-    private lateinit var incrementalDataDir: File
+    private var header: ByteArray? = null
-    private lateinit var binaryTreesDir: File
+    private val packageParts: MutableMap<File, TranslationResultValue> = hashMapOf()
-    private lateinit var packagesMetadataDir: File
-    private lateinit var headerMetadataFile: File
 
     override fun setUp() {
         super.setUp()
-        incrementalDataDir = File(workingDir, "incremental-data")
+        header = null
-        binaryTreesDir = File(incrementalDataDir, "binary-trees")
+        packageParts.clear()
-        packagesMetadataDir = File(incrementalDataDir, "packages-metadata")
-        headerMetadataFile = File(incrementalDataDir, "header.metadata")
     }
 
     override fun Services.Builder.registerAdditionalServices() {
-        if (incrementalDataDir.exists()) {
+        if (header != null) {
-            register(IncrementalDataProvider::class.java, object : IncrementalDataProvider {
+            register(IncrementalDataProvider::class.java, IncrementalDataProviderImpl(header!!, packageParts!!))
-                override val headerMetadata: ByteArray
-                    get() = headerMetadataFile.readBytes()
-                override val packagePartsMetadata: List<ByteArray>
-                    get() = packagesMetadataDir.walk().filter { it.isFile }.map { it.readBytes() }.toList()
-                override val binaryTrees: List<ByteArray>
-                    get() = binaryTreesDir.walk().filter { it.isFile }.map { it.readBytes() }.toList()
-            })
         }
 
         register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl())
     }
 
+    override fun markDirty(removedAndModifiedSources: Iterable<File>) {
+        removedAndModifiedSources.forEach { packageParts.remove(it) }
+    }
+
+    override fun processCompilationResults(outputItemsCollector: OutputItemsCollectorImpl, services: Services) {
+        val incrementalResults = services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl
+        header = incrementalResults.headerMetadata
+        packageParts.putAll(incrementalResults.packageParts)
+    }
+
     override fun runCompiler(filesToCompile: Iterable<File>, env: JpsCompilerEnvironment): Any? {
         val args = K2JSCompilerArguments().apply {
             outputFile = File(outDir, "out.js").canonicalPath
             reportOutputFiles = true
             freeArgs.addAll(filesToCompile.map { it.canonicalPath })
         }
-        val exitCode = runJSCompiler(args, env)
+        return runJSCompiler(args, env)
-
-        if (exitCode != ExitCode.OK) return exitCode
-
-        val incrementalResults = env.services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl
-        incrementalResults.apply {
-            packageParts.forEach {
-                val relativePath = it.sourceFile.toRelativeString(srcDir)
-                val treeFile = File(binaryTreesDir, relativePath + ".ast").apply { parentFile.mkdirs() }
-                treeFile.writeBytes(it.binaryAst)
-
-                val partProtoFile = File(packagesMetadataDir, relativePath + ".proto").apply { parentFile.mkdirs() }
-                partProtoFile.writeBytes(it.proto)
-
-                env.outputItemsCollector.outputs.apply {
-                    val sources = listOf(it.sourceFile)
-                    add(SimpleOutputItem(sources, treeFile))
-                    add(SimpleOutputItem(sources, partProtoFile))
-                }
-            }
-
-            headerMetadataFile.parentFile.mkdirs()
-            headerMetadataFile.writeBytes(headerMetadata)
-        }
-
-        return exitCode
     }
 }
 
@@ -167,6 +166,8 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() {
         super.tearDown()
     }
 
+    protected abstract fun markDirty(removedAndModifiedSources: Iterable<File>)
+    protected abstract fun processCompilationResults(outputItemsCollector: OutputItemsCollectorImpl, services: Services)
     protected abstract fun runCompiler(filesToCompile: Iterable<File>, env: JpsCompilerEnvironment): Any?
 
     fun doTest(path: String) {
@@ -198,7 +199,6 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() {
         val testDir = File(path)
         val workToOriginalFileMap = HashMap(copyTestSources(testDir, srcDir, filePrefix = ""))
         var dirtyFiles = srcDir.walk().filterTo(HashSet()) { it.isKotlinFile() }
-        val incrementalData = IncrementalData()
         val steps = getModificationsToPerform(testDir, moduleNames = null, allowNoFilesWithSuffixInTestData = true, touchPolicy = TouchPolicy.CHECKSUM)
                 .filter { it.isNotEmpty() }
 
@@ -206,14 +206,14 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() {
         fun CompilerOutput.originalFilesToLookups() =
                 compiledFiles.associateBy({ workToOriginalFileMap[it]!! }, { lookups[it] ?: emptyList() })
 
-        make(dirtyFiles, incrementalData).apply {
+        make(dirtyFiles).apply {
             logOutput("INITIAL BUILD")
             filesToLookups.add(originalFilesToLookups())
 
         }
         for ((i, modifications) in steps.withIndex()) {
             dirtyFiles = modifications.mapNotNullTo(HashSet()) { it.perform(workingDir, workToOriginalFileMap) }
-            make(dirtyFiles, incrementalData).apply {
+            make(dirtyFiles).apply {
                 logOutput("STEP ${i + 1}")
                 filesToLookups.add(originalFilesToLookups())
             }
@@ -237,22 +237,13 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() {
         val compiledFiles: Iterable<File>,
         val lookups: Map<File, List<LookupInfo>>
     )
-    private class IncrementalData(val sourceToOutput: MutableMap<File, MutableSet<File>> = hashMapOf())
 
-    private fun make(
+    private fun make(filesToCompile: Iterable<File>): CompilerOutput {
-            filesToCompile: Iterable<File>,
-            incrementalData: IncrementalData
-    ): CompilerOutput {
         filesToCompile.forEach {
             it.writeText(it.readText().replace(COMMENT_WITH_LOOKUP_INFO, ""))
         }
 
-        for (dirtyFile in filesToCompile) {
+        markDirty(filesToCompile)
-            incrementalData.sourceToOutput.remove(dirtyFile)?.forEach {
-                it.delete()
-            }
-        }
-
         val lookupTracker = TestLookupTracker()
         val messageCollector = TestMessageCollector()
         val outputItemsCollector = OutputItemsCollectorImpl()
@@ -263,16 +254,7 @@ abstract class AbstractLookupTrackerTest : TestWithWorkingDir() {
         }
         val environment = createTestingCompilerEnvironment(messageCollector, outputItemsCollector, services)
         val exitCode = runCompiler(filesToCompile, environment)
-
+        processCompilationResults(outputItemsCollector, environment.services)
-        for (output in outputItemsCollector.outputs) {
-            val outputFile = output.outputFile
-            if (outputFile.extension == "kotlin_module") continue
-
-            for (sourceFile in output.sourceFiles) {
-                val outputsForSource = incrementalData.sourceToOutput.getOrPut(sourceFile) { hashSetOf() }
-                outputsForSource.add(outputFile)
-            }
-        }
 
         val lookups = lookupTracker.lookups.groupBy { File(it.filePath) }
         val lookupsFromCompiledFiles = filesToCompile.associate { it to (lookups[it] ?: emptyList()) }

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt

@@ -68,8 +68,8 @@ abstract class AbstractJsProtoComparisonTest : AbstractProtoComparisonTest<Proto
 
         val classes = hashMapOf<ClassId, ProtoData>()
 
-        for ((sourceFile, protoBytes, _) in incrementalResults.packageParts) {
+        for ((sourceFile, translationResult) in incrementalResults.packageParts) {
-            classes.putAll(getProtoData(sourceFile, protoBytes))
+            classes.putAll(getProtoData(sourceFile, translationResult.metadata))
         }
 
         return classes

js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt

@@ -16,18 +16,17 @@
 
 package org.jetbrains.kotlin.incremental.js
 
+import java.io.File
+
 // byte arrays are used to simplify passing to different classloaders
 interface IncrementalDataProvider {
     /** gets header metadata (serialized [JsProtoBuf.Header]) from previous compilation */
     val headerMetadata: ByteArray
-    /** gets non-dirty package parts metadata (serialized [ProtoBuf.PackageFragment]) from previous compilation */
+    /** gets non-dirty package parts data from previous compilation */
-    val packagePartsMetadata: List<ByteArray>
+    val compiledPackageParts: Map<File, TranslationResultValue>
-    /** gets non-dirty package parts binary trees from previous compilation */
-    val binaryTrees: List<ByteArray>
 }
 
 class IncrementalDataProviderImpl(
     override val headerMetadata: ByteArray,
-    override val packagePartsMetadata: List<ByteArray>,
+    override val compiledPackageParts: Map<File, TranslationResultValue>
-    override val binaryTrees: List<ByteArray>
 ) : IncrementalDataProvider

js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt

@@ -29,8 +29,8 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer {
     lateinit var headerMetadata: ByteArray
         private set
 
-    private val _packageParts = arrayListOf<PackagePartData>()
+    private val _packageParts = hashMapOf<File, TranslationResultValue>()
-    val packageParts: List<PackagePartData>
+    val packageParts: Map<File, TranslationResultValue>
         get() = _packageParts
 
     override fun processHeader(headerMetadata: ByteArray) {
@@ -38,9 +38,7 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer {
     }
 
     override fun processPackagePart(sourceFile: File, packagePartMetadata: ByteArray, binaryAst: ByteArray) {
-        _packageParts.add(PackagePartData(sourceFile, packagePartMetadata, binaryAst))
+        _packageParts.put(sourceFile, TranslationResultValue(packagePartMetadata, binaryAst))
     }
-
-    data class PackagePartData(val sourceFile: File, val proto: ByteArray, val binaryAst: ByteArray)
 }
 

js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.js
+
+data class TranslationResultValue(val metadata: ByteArray, val binaryAst: ByteArray)
+

js/js.frontend/src/org/jetbrains/kotlin/js/analyze/TopDownAnalyzerFacadeForJS.kt

@@ -68,7 +68,7 @@ object TopDownAnalyzerFacadeForJS {
     ): JsAnalysisResult {
         val lookupTracker = config.configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER) ?: LookupTracker.DO_NOTHING
         val packageFragment = config.configuration[JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER]?.let {
-            val metadata = PackagesWithHeaderMetadata(it.headerMetadata, it.packagePartsMetadata)
+            val metadata = PackagesWithHeaderMetadata(it.headerMetadata, it.compiledPackageParts.values.map { it.metadata })
             KotlinJavascriptSerializationUtil.readDescriptors(metadata,
                                                               moduleContext.storageManager,
                                                               moduleContext.module,

js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt

@@ -47,6 +47,7 @@ import org.jetbrains.kotlin.js.parser.parse
 import org.jetbrains.kotlin.js.parser.sourcemaps.*
 import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
 import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
+import org.jetbrains.kotlin.incremental.js.TranslationResultValue
 import org.jetbrains.kotlin.js.test.utils.*
 import org.jetbrains.kotlin.js.util.TextOutputImpl
 import org.jetbrains.kotlin.psi.KtFile
@@ -286,17 +287,19 @@ abstract class BasicBoxTest(
             JsTestUtils.getFilesInDirectoryByExtension(baseDir + "/", KotlinFileType.EXTENSION)
         }
         val additionalFiles = globalCommonFiles + localCommonFiles + additionalCommonFiles
-        val psiFiles = createPsiFiles(testFiles + additionalFiles)
+        val allSourceFiles = (testFiles + additionalFiles).map(::File)
+        val psiFiles = createPsiFiles(allSourceFiles.sortedBy { it.canonicalPath }.map { it.canonicalPath })
 
         val sourceDirs = (testFiles + additionalFiles).map { File(it).parent }.distinct()
-        val config = createConfig(sourceDirs, module, dependencies, friends, multiModule, additionalMetadata = null)
+        val config = createConfig(sourceDirs, module, dependencies, friends, multiModule, incrementalData = null)
         val outputFile = File(outputFileName)
 
-        translateFiles(psiFiles.map(TranslationUnit::SourceFile), outputFile, config, outputPrefixFile, outputPostfixFile, mainCallParameters)
+        val incrementalData = IncrementalData()
+        translateFiles(psiFiles.map(TranslationUnit::SourceFile), outputFile, config, outputPrefixFile, outputPostfixFile, mainCallParameters, incrementalData)
 
         if (module.hasFilesToRecompile) {
-            checkIncrementalCompilation(sourceDirs, module, kotlinFiles, additionalFiles, dependencies, friends, multiModule, outputFile,
+            checkIncrementalCompilation(sourceDirs, module, kotlinFiles, dependencies, friends, multiModule, outputFile,
-                                        outputPrefixFile, outputPostfixFile, mainCallParameters)
+                                        outputPrefixFile, outputPostfixFile, mainCallParameters, incrementalData)
         }
     }
 
@@ -304,38 +307,35 @@ abstract class BasicBoxTest(
             sourceDirs: List<String>,
             module: TestModule,
             kotlinFiles: List<TestFile>,
-            additionalFiles: List<String>,
             dependencies: List<String>,
             friends: List<String>,
             multiModule: Boolean,
             outputFile: File,
             outputPrefixFile: File?,
             outputPostfixFile: File?,
-            mainCallParameters: MainCallParameters
+            mainCallParameters: MainCallParameters,
+            incrementalData: IncrementalData
     ) {
-        val incrementalDir = File(outputFile.parentFile, "incremental/" + outputFile.nameWithoutExtension)
+        val sourceToTranslationUnit = hashMapOf<File, TranslationUnit>()
-        val serializedMetadata = mutableListOf<File>()
+        for (testFile in kotlinFiles) {
-        val translationUnits = kotlinFiles.withIndex().map { (index, file) ->
+            if (testFile.recompile) {
-            if (file.recompile) {
+                val sourceFile = File(testFile.fileName)
-                TranslationUnit.SourceFile(createPsiFile(file.fileName))
+                incrementalData.translatedFiles.remove(sourceFile)
-            }
+                sourceToTranslationUnit[sourceFile] = TranslationUnit.SourceFile(createPsiFile(testFile.fileName))
-            else {
-                serializedMetadata += File(incrementalDir, "$index.$METADATA_EXTENSION")
-                val astFile = File(incrementalDir, "$index.$AST_EXTENSION")
-                TranslationUnit.BinaryAst(FileUtil.loadFileBytes(astFile))
             }
         }
-        val allTranslationUnits = translationUnits + additionalFiles.withIndex().map { (index, _) ->
+        for ((sourceFile, data) in incrementalData.translatedFiles) {
-            val astFile = File(incrementalDir, "${index + translationUnits.size}.$AST_EXTENSION")
+            sourceToTranslationUnit[sourceFile] = TranslationUnit.BinaryAst(data.binaryAst)
-            TranslationUnit.BinaryAst(FileUtil.loadFileBytes(astFile))
         }
+        val translationUnits = sourceToTranslationUnit.keys
+                .sortedBy { it.canonicalPath }
+                .map { sourceToTranslationUnit[it]!! }
 
-        val headerFile = File(incrementalDir, HEADER_FILE)
+        val recompiledConfig = createConfig(sourceDirs,module, dependencies, friends, multiModule, incrementalData)
-        val recompiledConfig = createConfig(sourceDirs, module, dependencies, friends, multiModule, Pair(headerFile,serializedMetadata))
         val recompiledOutputFile = File(outputFile.parentFile, outputFile.nameWithoutExtension + "-recompiled.js")
 
-        translateFiles(allTranslationUnits, recompiledOutputFile, recompiledConfig, outputPrefixFile, outputPostfixFile,
+        translateFiles(translationUnits, recompiledOutputFile, recompiledConfig, outputPrefixFile, outputPostfixFile,
-                       mainCallParameters)
+                       mainCallParameters, incrementalData)
 
         val originalOutput = FileUtil.loadFile(outputFile)
         val recompiledOutput = removeRecompiledSuffix(FileUtil.loadFile(recompiledOutputFile))
@@ -349,13 +349,16 @@ abstract class BasicBoxTest(
 
     private fun removeRecompiledSuffix(text: String): String = text.replace("-recompiled.js", ".js")
 
+    class IncrementalData(var header: ByteArray? = null, val translatedFiles: MutableMap<File, TranslationResultValue> = hashMapOf())
+
     protected fun translateFiles(
             units: List<TranslationUnit>,
             outputFile: File,
             config: JsConfig,
             outputPrefixFile: File?,
             outputPostfixFile: File?,
-            mainCallParameters: MainCallParameters
+            mainCallParameters: MainCallParameters,
+            incrementalData: IncrementalData
     ) {
         val translator = K2JSTranslator(config)
         val translationResult = translator.translateUnits(ExceptionThrowingReporter, units, mainCallParameters)
@@ -390,14 +393,11 @@ abstract class BasicBoxTest(
         config.configuration[JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER]?.let {
             val incrementalService = it as IncrementalResultsConsumerImpl
 
-            val incrementalDir = File(outputDir, "incremental/${outputFile.nameWithoutExtension}")
+            for ((srcFile, data) in incrementalService.packageParts) {
-
+                incrementalData.translatedFiles[srcFile] = data
-            for ((i, packagePart) in incrementalService.packageParts.withIndex()) {
-                FileUtil.writeToFile(File(incrementalDir, "$i.$AST_EXTENSION"), packagePart.binaryAst)
-                FileUtil.writeToFile(File(incrementalDir, "$i.$METADATA_EXTENSION"), packagePart.proto)
             }
 
-            FileUtil.writeToFile(File(incrementalDir, HEADER_FILE), incrementalService.headerMetadata)
+            incrementalData.header = incrementalService.headerMetadata
         }
 
         processJsProgram(translationResult.program, units.filterIsInstance<TranslationUnit.SourceFile>().map { it.file })
@@ -472,9 +472,7 @@ abstract class BasicBoxTest(
     private fun createPsiFiles(fileNames: List<String>): List<KtFile> = fileNames.map(this::createPsiFile)
 
     private fun createConfig(
-            sourceDirs: List<String>,
+            sourceDirs: List<String>,module: TestModule, dependencies: List<String>, friends: List<String>, multiModule: Boolean, incrementalData: IncrementalData?
-            module: TestModule, dependencies: List<String>, friends: List<String>, multiModule: Boolean,
-            additionalMetadata: Pair<File, List<File>>?
     ): JsConfig {
         val configuration = environment.configuration.copy()
 
@@ -494,10 +492,10 @@ abstract class BasicBoxTest(
         val hasFilesToRecompile = module.hasFilesToRecompile
         configuration.put(JSConfigurationKeys.META_INFO, multiModule)
         if (hasFilesToRecompile) {
-            if (additionalMetadata != null) {
+            val header = incrementalData?.header
-                val (headerFile, packagePartFiles) = additionalMetadata
+            if (header != null) {
                 configuration.put(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER,
-                                  IncrementalDataProviderImpl(headerFile.readBytes(), packagePartFiles.map { it.readBytes() }, emptyList()))
+                                  IncrementalDataProviderImpl(header, incrementalData.translatedFiles))
             }
 
             configuration.put(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER, IncrementalResultsConsumerImpl())
@@ -660,9 +658,6 @@ abstract class BasicBoxTest(
         private val EXPECTED_REACHABLE_NODES = Pattern.compile("^// *$EXPECTED_REACHABLE_NODES_DIRECTIVE: *([0-9]+) *$", Pattern.MULTILINE)
         private val RECOMPILE_PATTERN = Pattern.compile("^// *RECOMPILE *$", Pattern.MULTILINE)
         private val SOURCE_MAP_SOURCE_EMBEDDING = Regex("^// *SOURCE_MAP_EMBED_SOURCES: ([A-Z]+)*\$", RegexOption.MULTILINE)
-        private val AST_EXTENSION = "jsast"
-        private val METADATA_EXTENSION = "jsmeta"
-        private val HEADER_FILE = "header.$METADATA_EXTENSION"
 
         val TEST_MODULE = "JS_TESTS"
         private val DEFAULT_MODULE = "main"