Improve performance of JS tests

* Do not read protos for descriptors of stdlib and kotlin-tests
  repeatedly
* Parse libraries lazily in inline, so that when no inline function
  exist in a test, we won't parse huge kotlin.js file
* Speed-up source map parser

js/js.frontend/src/org/jetbrains/kotlin/js/config/JsConfig.java

@@ -21,12 +21,11 @@ import com.intellij.openapi.vfs.StandardFileSystems;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileManager;
 import com.intellij.openapi.vfs.VirtualFileSystem;
-import com.intellij.util.PathUtil;
 import com.intellij.util.SmartList;
 import com.intellij.util.io.URLUtil;
 import kotlin.Unit;
 import kotlin.collections.CollectionsKt;
-import kotlin.jvm.functions.Function2;
+import kotlin.jvm.functions.Function1;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.kotlin.config.*;
@@ -36,6 +35,7 @@ import org.jetbrains.kotlin.js.resolve.JsPlatform;
 import org.jetbrains.kotlin.name.Name;
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration;
 import org.jetbrains.kotlin.serialization.js.JsModuleDescriptor;
+import org.jetbrains.kotlin.serialization.js.KotlinJavaScriptLibraryParts;
 import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil;
 import org.jetbrains.kotlin.serialization.js.ModuleKind;
 import org.jetbrains.kotlin.storage.LockBasedStorageManager;
@@ -73,9 +73,23 @@ public class JsConfig {
 
     private boolean initialized = false;
 
+    @Nullable
+    private final List<JsModuleDescriptor<KotlinJavaScriptLibraryParts>> metadataCache;
+
+    @Nullable
+    private final Set<String> librariesToSkip;
+
     public JsConfig(@NotNull Project project, @NotNull CompilerConfiguration configuration) {
+        this(project, configuration, null, null);
+    }
+
+    public JsConfig(@NotNull Project project, @NotNull CompilerConfiguration configuration,
+            @Nullable List<JsModuleDescriptor<KotlinJavaScriptLibraryParts>> metadataCache,
+            @Nullable Set<String> librariesToSkip) {
         this.project = project;
         this.configuration = configuration;
+        this.metadataCache = metadataCache;
+        this.librariesToSkip = librariesToSkip;
     }
 
     @NotNull
@@ -132,17 +146,13 @@ public class JsConfig {
     }
 
     public boolean checkLibFilesAndReportErrors(@NotNull JsConfig.Reporter report) {
-        return checkLibFilesAndReportErrors(report, null);
+        return checkLibFilesAndReportErrors(getLibraries(), report, null);
-    }
-
-    private boolean checkLibFilesAndReportErrors(@NotNull JsConfig.Reporter report, @Nullable Function2<VirtualFile, String, Unit> action) {
-        return checkLibFilesAndReportErrors(getLibraries(), report, action);
     }
 
     private boolean checkLibFilesAndReportErrors(
             @NotNull Collection<String> libraries,
             @NotNull JsConfig.Reporter report,
-            @Nullable Function2<VirtualFile, String, Unit> action
+            @Nullable Function1<List<KotlinJavascriptMetadata>, Unit> action
     ) {
         if (libraries.isEmpty()) {
             return false;
@@ -157,6 +167,8 @@ public class JsConfig {
                 getLanguageVersionSettings(configuration).isFlagEnabled(AnalysisFlags.getSkipMetadataVersionCheck());
 
         for (String path : libraries) {
+            if (librariesToSkip != null && librariesToSkip.contains(path)) continue;
+
             VirtualFile file;
 
             File filePath = new File(path);
@@ -177,7 +189,7 @@ public class JsConfig {
                 return true;
             }
 
-            List<KotlinJavascriptMetadata> metadataList = KotlinJavascriptMetadataUtils.loadMetadata(filePath);
+            List<KotlinJavascriptMetadata> metadataList = KotlinJavascriptMetadataUtils.loadMetadata(path);
             if (metadataList.isEmpty()) {
                 report.warning("'" + path + "' is not a valid Kotlin Javascript library");
                 continue;
@@ -196,7 +208,7 @@ public class JsConfig {
             }
 
             if (action != null) {
-                action.invoke(file, path);
+                action.invoke(metadataList);
             }
         }
 
@@ -216,6 +228,27 @@ public class JsConfig {
             kotlinModuleDescriptors.add(descriptor.getData());
         }
 
+        if (metadataCache != null) {
+            LanguageVersionSettings languageVersionSettings = CommonConfigurationKeysKt.getLanguageVersionSettings(configuration);
+            for (JsModuleDescriptor<KotlinJavaScriptLibraryParts> cached : metadataCache) {
+                ModuleDescriptorImpl moduleDescriptor = new ModuleDescriptorImpl(
+                        Name.special("<" + cached.getName() + ">"), storageManager, JsPlatform.INSTANCE.getBuiltIns()
+                );
+
+                JsModuleDescriptor<PackageFragmentProvider> rawDescriptor = KotlinJavascriptSerializationUtil.readModuleFromProto(
+                        cached, storageManager, moduleDescriptor,
+                        new CompilerDeserializationConfiguration(languageVersionSettings)
+                );
+
+                PackageFragmentProvider provider = rawDescriptor.getData();
+                moduleDescriptor.initialize(provider != null ? provider : PackageFragmentProvider.Empty.INSTANCE);
+
+                JsModuleDescriptor<ModuleDescriptorImpl> jsModuleDescriptor = cached.copy(moduleDescriptor);
+                moduleDescriptors.add(jsModuleDescriptor);
+                kotlinModuleDescriptors.add(jsModuleDescriptor.getData());
+            }
+        }
+
         for (JsModuleDescriptor<ModuleDescriptorImpl> module : moduleDescriptors) {
             // TODO: remove downcast
             setDependencies(module.getData(), kotlinModuleDescriptors);
@@ -253,8 +286,7 @@ public class JsConfig {
                 }
             };
 
-            boolean hasErrors = checkLibFilesAndReportErrors(getFriends(), reporter, (file, path) -> {
+            boolean hasErrors = checkLibFilesAndReportErrors(getFriends(), reporter, metaList -> {
-                List<KotlinJavascriptMetadata> metaList = loadMetadata(file, "friendPath");
                 metadata.addAll(metaList);
                 friends.addAll(metaList);
 
@@ -262,9 +294,8 @@ public class JsConfig {
             });
 
 
-            hasErrors |= checkLibFilesAndReportErrors(CollectionsKt.subtract(getLibraries(), getFriends()), reporter, (file, path) -> {
+            hasErrors |= checkLibFilesAndReportErrors(CollectionsKt.subtract(getLibraries(), getFriends()), reporter, metaList -> {
-                metadata.addAll(loadMetadata(file, "libraryPath"));
+                metadata.addAll(metaList);
-
 
                 return Unit.INSTANCE;
             });
@@ -275,13 +306,6 @@ public class JsConfig {
         initialized = true;
     }
 
-    @NotNull
-    private static List<KotlinJavascriptMetadata> loadMetadata(@NotNull VirtualFile file, @NotNull String name) {
-        String libraryPath = PathUtil.getLocalPath(file);
-        assert libraryPath != null : name + " for " + file + " should not be null";
-        return  KotlinJavascriptMetadataUtils.loadMetadata(libraryPath);
-    }
-
     private final IdentityHashMap<KotlinJavascriptMetadata, JsModuleDescriptor<ModuleDescriptorImpl>> factoryMap = new IdentityHashMap<>();
 
     private JsModuleDescriptor<ModuleDescriptorImpl> createModuleDescriptor(KotlinJavascriptMetadata metadata) {

js/js.inliner/src/org/jetbrains/kotlin/js/inline/FunctionReader.kt

@@ -70,20 +70,16 @@ class FunctionReader(
             val fileContent: String,
             val moduleVariable: String,
             val kotlinVariable: String,
-            val offsetToSourceMapping: OffsetToSourceMapping,
+            offsetToSourceMappingProvider: () -> OffsetToSourceMapping,
             val sourceMap: SourceMap?
-    )
+    ) {
+        val offsetToSourceMapping by lazy(offsetToSourceMappingProvider)
+    }
 
-    private val moduleNameToInfo = HashMultimap.create<String, ModuleInfo>()
+    private val moduleNameToInfo by lazy {
+        val result = HashMultimap.create<String, ModuleInfo>()
 
-    private val moduleNameMap: Map<String, JsExpression>
+        JsLibraryUtils.traverseJsLibraries(config.libraries.map(::File)) { (content, path, sourceMapContent) ->
-
-    init {
-        val libs = config.libraries.map(::File)
-
-        moduleNameMap = buildModuleNameMap(fragments)
-
-        JsLibraryUtils.traverseJsLibraries(libs) { (content, path, sourceMapContent) ->
             var current = 0
 
             while (true) {
@@ -100,11 +96,11 @@ class FunctionReader(
                 val kotlinVariable = preciseMatcher.group(1)
 
                 val sourceMap = sourceMapContent?.let {
-                    val result = SourceMapParser.parse(StringReader(it))
+                    val sourceMapResult = SourceMapParser.parse(StringReader(it))
-                    when (result) {
+                    when (sourceMapResult) {
-                        is SourceMapSuccess -> result.value
+                        is SourceMapSuccess -> sourceMapResult.value
                         is SourceMapError -> {
-                            reporter.warning("Error parsing source map file for $path: ${result.message}")
+                            reporter.warning("Error parsing source map file for $path: ${sourceMapResult.message}")
                             null
                         }
                     }
@@ -115,13 +111,21 @@ class FunctionReader(
                         fileContent = content,
                         moduleVariable = moduleVariable,
                         kotlinVariable = kotlinVariable,
-                        offsetToSourceMapping = OffsetToSourceMapping(content),
+                        offsetToSourceMappingProvider = { OffsetToSourceMapping(content) },
                         sourceMap = sourceMap
                 )
 
-                moduleNameToInfo.put(moduleName, moduleInfo)
+                result.put(moduleName, moduleInfo)
             }
         }
+
+        result
+    }
+
+    private val moduleNameMap: Map<String, JsExpression>
+
+    init {
+        moduleNameMap = buildModuleNameMap(fragments)
     }
 
     // Since we compile each source file in its own context (and we may loose these context when performing incremental compilation)

js/js.parser/src/org/jetbrains/kotlin/js/parser/OffsetToSourceMapping.kt

@@ -22,16 +22,31 @@ class OffsetToSourceMapping(text: String) {
     private val data: IntArray
 
     init {
-        val lineSeparators = LINE_SEPARATOR.findAll(text).map { it.range.endInclusive + 1 }
+        var i = 0
-        data = (sequenceOf(0) + lineSeparators).toList().toIntArray()
+        val lineSeparators = mutableListOf<Int>()
+        lineSeparators += 0
+        while (i < text.length) {
+            val c = text[i++]
+            val isNewLine = when (c) {
+                '\r' -> {
+                    if (i < text.length && text[i] == '\n') {
+                        ++i
+                    }
+                    true
+                }
+                '\n' -> true
+                else -> false
+            }
+            if (isNewLine) {
+                lineSeparators += i
+            }
+        }
+
+        data = lineSeparators.toIntArray()
     }
 
     operator fun get(offset: Int): CodePosition {
         val lineNumber = data.binarySearch(offset).let { if (it >= 0) it else -it - 2 }
         return CodePosition(lineNumber, offset - data[lineNumber])
     }
-
-    private companion object {
-        private val LINE_SEPARATOR = Regex("\\r\\n|\\r|\\n")
-    }
 }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt

@@ -44,8 +44,19 @@ object KotlinJavascriptSerializationUtil {
     @JvmStatic
     fun readModule(
             metadata: ByteArray, storageManager: StorageManager, module: ModuleDescriptor, configuration: DeserializationConfiguration
+    ): JsModuleDescriptor<PackageFragmentProvider?> =
+        readModuleFromProto(readModuleAsProto(metadata, module.name.asString()), storageManager, module, configuration)
+
+    @JvmStatic
+    fun readModuleAsProto(metadata: ByteArray, name: String): JsModuleDescriptor<KotlinJavaScriptLibraryParts> =
+        metadata.deserializeToLibraryParts(name)
+
+    @JvmStatic
+    fun readModuleFromProto(
+            jsModule: JsModuleDescriptor<KotlinJavaScriptLibraryParts>,
+            storageManager: StorageManager, module: ModuleDescriptor,
+            configuration: DeserializationConfiguration
     ): JsModuleDescriptor<PackageFragmentProvider?> {
-        val jsModule = metadata.deserializeToLibraryParts(module.name.asString())
         val (header, packageFragmentProtos) = jsModule.data
         return jsModule.copy(createKotlinJavascriptPackageFragmentProvider(
                 storageManager, module, header, packageFragmentProtos, configuration
@@ -244,7 +255,7 @@ object KotlinJavascriptSerializationUtil {
         }.toByteArray()
     }
 
-    private fun ByteArray.deserializeToLibraryParts(name: String): JsModuleDescriptor<Pair<JsProtoBuf.Header, List<ProtoBuf.PackageFragment>>> {
+    private fun ByteArray.deserializeToLibraryParts(name: String): JsModuleDescriptor<KotlinJavaScriptLibraryParts> {
         val (header, content) = GZIPInputStream(ByteArrayInputStream(this)).use { stream ->
             JsProtoBuf.Header.parseDelimitedFrom(stream, JsSerializerProtocol.extensionRegistry) to
             JsProtoBuf.Library.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
@@ -252,7 +263,7 @@ object KotlinJavascriptSerializationUtil {
 
         return JsModuleDescriptor(
                 name = name,
-                data = header to content.packageFragmentList,
+                data = KotlinJavaScriptLibraryParts(header, content.packageFragmentList),
                 kind = when (content.kind) {
                     null, JsProtoBuf.Library.Kind.PLAIN -> ModuleKind.PLAIN
                     JsProtoBuf.Library.Kind.AMD -> ModuleKind.AMD
@@ -263,3 +274,5 @@ object KotlinJavascriptSerializationUtil {
         )
     }
 }
+
+data class KotlinJavaScriptLibraryParts(val header: JsProtoBuf.Header, val body: List<ProtoBuf.PackageFragment>)

js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt

@@ -50,6 +50,7 @@ import org.jetbrains.kotlin.js.util.TextOutputImpl
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi.KtNamedFunction
 import org.jetbrains.kotlin.psi.KtPsiFactory
+import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil
 import org.jetbrains.kotlin.serialization.js.ModuleKind
 import org.jetbrains.kotlin.serialization.js.PackagesWithHeaderMetadata
 import org.jetbrains.kotlin.test.InTextDirectivesUtils
@@ -58,6 +59,7 @@ import org.jetbrains.kotlin.test.KotlinTestUtils.TestFileFactory
 import org.jetbrains.kotlin.test.KotlinTestWithEnvironment
 import org.jetbrains.kotlin.test.TargetBackend
 import org.jetbrains.kotlin.utils.DFS
+import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
 import java.io.*
 import java.nio.charset.Charset
 import java.util.regex.Pattern
@@ -97,8 +99,8 @@ abstract class BasicBoxTest(
             val orderedModules = DFS.topologicalOrder(modules.values) { module -> module.dependencies.mapNotNull { modules[it] } }
 
             val generatedJsFiles = orderedModules.asReversed().mapNotNull { module ->
-                val dependencies = module.dependencies.mapNotNull { modules[it]?.outputFileName(outputDir) + ".meta.js" }
+                val dependencies = module.dependencies.map { modules[it]?.outputFileName(outputDir) + ".meta.js" }
-                val friends = module.friends.mapNotNull { modules[it]?.outputFileName(outputDir) + ".meta.js" }
+                val friends = module.friends.map { modules[it]?.outputFileName(outputDir) + ".meta.js" }
 
                 val outputFileName = module.outputFileName(outputDir) + ".js"
                 generateJavaScriptFile(file.parent, module, outputFileName, dependencies, friends, modules.size > 1,
@@ -510,7 +512,7 @@ abstract class BasicBoxTest(
             configuration.put(JSConfigurationKeys.TYPED_ARRAYS_ENABLED, true)
         }
 
-        return JsConfig(project, configuration)
+        return JsConfig(project, configuration, METADATA_CACHE, (JsConfig.JS_STDLIB + JsConfig.JS_KOTLIN_TEST).toSet())
     }
 
     private fun minifyAndRun(
@@ -636,6 +638,14 @@ abstract class BasicBoxTest(
     }
 
     companion object {
+        val METADATA_CACHE = (JsConfig.JS_STDLIB.asSequence() + JsConfig.JS_KOTLIN_TEST)
+                .flatMap {
+                    KotlinJavascriptMetadataUtils
+                            .loadMetadata(it).asSequence()
+                            .map { KotlinJavascriptSerializationUtil.readModuleAsProto(it.body, it.moduleName) }
+                }
+                .toList()
+
         const val TEST_DATA_DIR_PATH = "js/js.translator/testData/"
         const val DIST_DIR_JS_PATH = "dist/js/"