azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve performance of JS tests

Browse Source 
* Do not read protos for descriptors of stdlib and kotlin-tests
  repeatedly
* Parse libraries lazily in inline, so that when no inline function
  exist in a test, we won't parse huge kotlin.js file
* Speed-up source map parser
This commit is contained in:
Alexey Andreev
2017-07-09 14:09:20 +03:00
committed by Alexey Andreev
parent 83ec8aa918
commit 5a9adcca2d
5 changed files with 117 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.frontend/src/org/jetbrains/kotlin/js/config/JsConfig.java
+46 -22
View File
@@ -21,12 +21,11 @@ import com.intellij.openapi.vfs.StandardFileSystems;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileManager;
import com.intellij.openapi.vfs.VirtualFileSystem;
import com.intellij.util.PathUtil;
import com.intellij.util.SmartList;
import com.intellij.util.io.URLUtil;
import kotlin.Unit;
import kotlin.collections.CollectionsKt;
import kotlin.jvm.functions.Function2;
import kotlin.jvm.functions.Function1;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.config.*;
@@ -36,6 +35,7 @@ import org.jetbrains.kotlin.js.resolve.JsPlatform;
import org.jetbrains.kotlin.name.Name;
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration;
import org.jetbrains.kotlin.serialization.js.JsModuleDescriptor;
import org.jetbrains.kotlin.serialization.js.KotlinJavaScriptLibraryParts;
import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil;
import org.jetbrains.kotlin.serialization.js.ModuleKind;
import org.jetbrains.kotlin.storage.LockBasedStorageManager;
@@ -73,9 +73,23 @@ public class JsConfig {
private boolean initialized = false;
@Nullable
private final List<JsModuleDescriptor<KotlinJavaScriptLibraryParts>> metadataCache;
@Nullable
private final Set<String> librariesToSkip;
public JsConfig(@NotNull Project project, @NotNull CompilerConfiguration configuration) {
this(project, configuration, null, null);
}
public JsConfig(@NotNull Project project, @NotNull CompilerConfiguration configuration,
@Nullable List<JsModuleDescriptor<KotlinJavaScriptLibraryParts>> metadataCache,
@Nullable Set<String> librariesToSkip) {
this.project = project;
this.configuration = configuration;
this.metadataCache = metadataCache;
this.librariesToSkip = librariesToSkip;
}
@NotNull
@@ -132,17 +146,13 @@ public class JsConfig {
}
public boolean checkLibFilesAndReportErrors(@NotNull JsConfig.Reporter report) {
return checkLibFilesAndReportErrors(report, null);
}
private boolean checkLibFilesAndReportErrors(@NotNull JsConfig.Reporter report, @Nullable Function2<VirtualFile, String, Unit> action) {
return checkLibFilesAndReportErrors(getLibraries(), report, action);
return checkLibFilesAndReportErrors(getLibraries(), report, null);
}
private boolean checkLibFilesAndReportErrors(
@NotNull Collection<String> libraries,
@NotNull JsConfig.Reporter report,
@Nullable Function2<VirtualFile, String, Unit> action
@Nullable Function1<List<KotlinJavascriptMetadata>, Unit> action
) {
if (libraries.isEmpty()) {
return false;
@@ -157,6 +167,8 @@ public class JsConfig {
getLanguageVersionSettings(configuration).isFlagEnabled(AnalysisFlags.getSkipMetadataVersionCheck());
for (String path : libraries) {
if (librariesToSkip != null && librariesToSkip.contains(path)) continue;
VirtualFile file;
File filePath = new File(path);
@@ -177,7 +189,7 @@ public class JsConfig {
return true;
}
List<KotlinJavascriptMetadata> metadataList = KotlinJavascriptMetadataUtils.loadMetadata(filePath);
List<KotlinJavascriptMetadata> metadataList = KotlinJavascriptMetadataUtils.loadMetadata(path);
if (metadataList.isEmpty()) {
report.warning("'" + path + "' is not a valid Kotlin Javascript library");
continue;
@@ -196,7 +208,7 @@ public class JsConfig {
}
if (action != null) {
action.invoke(file, path);
action.invoke(metadataList);
}
}
@@ -216,6 +228,27 @@ public class JsConfig {
kotlinModuleDescriptors.add(descriptor.getData());
}
if (metadataCache != null) {
LanguageVersionSettings languageVersionSettings = CommonConfigurationKeysKt.getLanguageVersionSettings(configuration);
for (JsModuleDescriptor<KotlinJavaScriptLibraryParts> cached : metadataCache) {
ModuleDescriptorImpl moduleDescriptor = new ModuleDescriptorImpl(
Name.special("<" + cached.getName() + ">"), storageManager, JsPlatform.INSTANCE.getBuiltIns()
);
JsModuleDescriptor<PackageFragmentProvider> rawDescriptor = KotlinJavascriptSerializationUtil.readModuleFromProto(
cached, storageManager, moduleDescriptor,
new CompilerDeserializationConfiguration(languageVersionSettings)
);
PackageFragmentProvider provider = rawDescriptor.getData();
moduleDescriptor.initialize(provider != null ? provider : PackageFragmentProvider.Empty.INSTANCE);
JsModuleDescriptor<ModuleDescriptorImpl> jsModuleDescriptor = cached.copy(moduleDescriptor);
moduleDescriptors.add(jsModuleDescriptor);
kotlinModuleDescriptors.add(jsModuleDescriptor.getData());
}
}
for (JsModuleDescriptor<ModuleDescriptorImpl> module : moduleDescriptors) {
// TODO: remove downcast
setDependencies(module.getData(), kotlinModuleDescriptors);
@@ -253,8 +286,7 @@ public class JsConfig {
}
};
boolean hasErrors = checkLibFilesAndReportErrors(getFriends(), reporter, (file, path) -> {
List<KotlinJavascriptMetadata> metaList = loadMetadata(file, "friendPath");
boolean hasErrors = checkLibFilesAndReportErrors(getFriends(), reporter, metaList -> {
metadata.addAll(metaList);
friends.addAll(metaList);
@@ -262,9 +294,8 @@ public class JsConfig {
});
hasErrors |= checkLibFilesAndReportErrors(CollectionsKt.subtract(getLibraries(), getFriends()), reporter, (file, path) -> {
metadata.addAll(loadMetadata(file, "libraryPath"));
hasErrors |= checkLibFilesAndReportErrors(CollectionsKt.subtract(getLibraries(), getFriends()), reporter, metaList -> {
metadata.addAll(metaList);
return Unit.INSTANCE;
});
@@ -275,13 +306,6 @@ public class JsConfig {
initialized = true;
}
@NotNull
private static List<KotlinJavascriptMetadata> loadMetadata(@NotNull VirtualFile file, @NotNull String name) {
String libraryPath = PathUtil.getLocalPath(file);
assert libraryPath != null : name + " for " + file + " should not be null";
return KotlinJavascriptMetadataUtils.loadMetadata(libraryPath);
}
private final IdentityHashMap<KotlinJavascriptMetadata, JsModuleDescriptor<ModuleDescriptorImpl>> factoryMap = new IdentityHashMap<>();
private JsModuleDescriptor<ModuleDescriptorImpl> createModuleDescriptor(KotlinJavascriptMetadata metadata) {
js/js.inliner/src/org/jetbrains/kotlin/js/inline/FunctionReader.kt
+21 -17
View File
@@ -70,20 +70,16 @@ class FunctionReader(
val fileContent: String,
val moduleVariable: String,
val kotlinVariable: String,
val offsetToSourceMapping: OffsetToSourceMapping,
offsetToSourceMappingProvider: () -> OffsetToSourceMapping,
val sourceMap: SourceMap?
)
) {
val offsetToSourceMapping by lazy(offsetToSourceMappingProvider)
}
private val moduleNameToInfo = HashMultimap.create<String, ModuleInfo>()
private val moduleNameToInfo by lazy {
val result = HashMultimap.create<String, ModuleInfo>()
private val moduleNameMap: Map<String, JsExpression>
init {
val libs = config.libraries.map(::File)
moduleNameMap = buildModuleNameMap(fragments)
JsLibraryUtils.traverseJsLibraries(libs) { (content, path, sourceMapContent) ->
JsLibraryUtils.traverseJsLibraries(config.libraries.map(::File)) { (content, path, sourceMapContent) ->
var current = 0
while (true) {
@@ -100,11 +96,11 @@ class FunctionReader(
val kotlinVariable = preciseMatcher.group(1)
val sourceMap = sourceMapContent?.let {
val result = SourceMapParser.parse(StringReader(it))
when (result) {
is SourceMapSuccess -> result.value
val sourceMapResult = SourceMapParser.parse(StringReader(it))
when (sourceMapResult) {
is SourceMapSuccess -> sourceMapResult.value
is SourceMapError -> {
reporter.warning("Error parsing source map file for $path: ${result.message}")
reporter.warning("Error parsing source map file for $path: ${sourceMapResult.message}")
null
}
}
@@ -115,13 +111,21 @@ class FunctionReader(
fileContent = content,
moduleVariable = moduleVariable,
kotlinVariable = kotlinVariable,
offsetToSourceMapping = OffsetToSourceMapping(content),
offsetToSourceMappingProvider = { OffsetToSourceMapping(content) },
sourceMap = sourceMap
)
moduleNameToInfo.put(moduleName, moduleInfo)
result.put(moduleName, moduleInfo)
}
}
result
}
private val moduleNameMap: Map<String, JsExpression>
init {
moduleNameMap = buildModuleNameMap(fragments)
}
// Since we compile each source file in its own context (and we may loose these context when performing incremental compilation)
js/js.parser/src/org/jetbrains/kotlin/js/parser/OffsetToSourceMapping.kt
+21 -6
View File
@@ -22,16 +22,31 @@ class OffsetToSourceMapping(text: String) {
private val data: IntArray
init {
val lineSeparators = LINE_SEPARATOR.findAll(text).map { it.range.endInclusive + 1 }
data = (sequenceOf(0) + lineSeparators).toList().toIntArray()
var i = 0
val lineSeparators = mutableListOf<Int>()
lineSeparators += 0
while (i < text.length) {
val c = text[i++]
val isNewLine = when (c) {
'\r' -> {
if (i < text.length && text[i] == '\n') {
++i
}
true
}
'\n' -> true
else -> false
}
if (isNewLine) {
lineSeparators += i
}
}
data = lineSeparators.toIntArray()
}
operator fun get(offset: Int): CodePosition {
val lineNumber = data.binarySearch(offset).let { if (it >= 0) it else -it - 2 }
return CodePosition(lineNumber, offset - data[lineNumber])
}
private companion object {
private val LINE_SEPARATOR = Regex("\\r\\n|\\r|\\n")
}
}
js/js.serializer/src/org/jetbrains/kotlin/serialization/js/KotlinJavascriptSerializationUtil.kt
+16 -3
View File
@@ -44,8 +44,19 @@ object KotlinJavascriptSerializationUtil {
@JvmStatic
fun readModule(
metadata: ByteArray, storageManager: StorageManager, module: ModuleDescriptor, configuration: DeserializationConfiguration
): JsModuleDescriptor<PackageFragmentProvider?> =
readModuleFromProto(readModuleAsProto(metadata, module.name.asString()), storageManager, module, configuration)
@JvmStatic
fun readModuleAsProto(metadata: ByteArray, name: String): JsModuleDescriptor<KotlinJavaScriptLibraryParts> =
metadata.deserializeToLibraryParts(name)
@JvmStatic
fun readModuleFromProto(
jsModule: JsModuleDescriptor<KotlinJavaScriptLibraryParts>,
storageManager: StorageManager, module: ModuleDescriptor,
configuration: DeserializationConfiguration
): JsModuleDescriptor<PackageFragmentProvider?> {
val jsModule = metadata.deserializeToLibraryParts(module.name.asString())
val (header, packageFragmentProtos) = jsModule.data
return jsModule.copy(createKotlinJavascriptPackageFragmentProvider(
storageManager, module, header, packageFragmentProtos, configuration
@@ -244,7 +255,7 @@ object KotlinJavascriptSerializationUtil {
}.toByteArray()
}
private fun ByteArray.deserializeToLibraryParts(name: String): JsModuleDescriptor<Pair<JsProtoBuf.Header, List<ProtoBuf.PackageFragment>>> {
private fun ByteArray.deserializeToLibraryParts(name: String): JsModuleDescriptor<KotlinJavaScriptLibraryParts> {
val (header, content) = GZIPInputStream(ByteArrayInputStream(this)).use { stream ->
JsProtoBuf.Header.parseDelimitedFrom(stream, JsSerializerProtocol.extensionRegistry) to
JsProtoBuf.Library.parseFrom(stream, JsSerializerProtocol.extensionRegistry)
@@ -252,7 +263,7 @@ object KotlinJavascriptSerializationUtil {
return JsModuleDescriptor(
name = name,
data = header to content.packageFragmentList,
data = KotlinJavaScriptLibraryParts(header, content.packageFragmentList),
kind = when (content.kind) {
null, JsProtoBuf.Library.Kind.PLAIN -> ModuleKind.PLAIN
JsProtoBuf.Library.Kind.AMD -> ModuleKind.AMD
@@ -263,3 +274,5 @@ object KotlinJavascriptSerializationUtil {
)
}
}
data class KotlinJavaScriptLibraryParts(val header: JsProtoBuf.Header, val body: List<ProtoBuf.PackageFragment>)
js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt
+13 -3
View File
@@ -50,6 +50,7 @@ import org.jetbrains.kotlin.js.util.TextOutputImpl
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi.KtNamedFunction
import org.jetbrains.kotlin.psi.KtPsiFactory
import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil
import org.jetbrains.kotlin.serialization.js.ModuleKind
import org.jetbrains.kotlin.serialization.js.PackagesWithHeaderMetadata
import org.jetbrains.kotlin.test.InTextDirectivesUtils
@@ -58,6 +59,7 @@ import org.jetbrains.kotlin.test.KotlinTestUtils.TestFileFactory
import org.jetbrains.kotlin.test.KotlinTestWithEnvironment
import org.jetbrains.kotlin.test.TargetBackend
import org.jetbrains.kotlin.utils.DFS
import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
import java.io.*
import java.nio.charset.Charset
import java.util.regex.Pattern
@@ -97,8 +99,8 @@ abstract class BasicBoxTest(
val orderedModules = DFS.topologicalOrder(modules.values) { module -> module.dependencies.mapNotNull { modules[it] } }
val generatedJsFiles = orderedModules.asReversed().mapNotNull { module ->
val dependencies = module.dependencies.mapNotNull { modules[it]?.outputFileName(outputDir) + ".meta.js" }
val friends = module.friends.mapNotNull { modules[it]?.outputFileName(outputDir) + ".meta.js" }
val dependencies = module.dependencies.map { modules[it]?.outputFileName(outputDir) + ".meta.js" }
val friends = module.friends.map { modules[it]?.outputFileName(outputDir) + ".meta.js" }
val outputFileName = module.outputFileName(outputDir) + ".js"
generateJavaScriptFile(file.parent, module, outputFileName, dependencies, friends, modules.size > 1,
@@ -510,7 +512,7 @@ abstract class BasicBoxTest(
configuration.put(JSConfigurationKeys.TYPED_ARRAYS_ENABLED, true)
}
return JsConfig(project, configuration)
return JsConfig(project, configuration, METADATA_CACHE, (JsConfig.JS_STDLIB + JsConfig.JS_KOTLIN_TEST).toSet())
}
private fun minifyAndRun(
@@ -636,6 +638,14 @@ abstract class BasicBoxTest(
}
companion object {
val METADATA_CACHE = (JsConfig.JS_STDLIB.asSequence() + JsConfig.JS_KOTLIN_TEST)
.flatMap {
KotlinJavascriptMetadataUtils
.loadMetadata(it).asSequence()
.map { KotlinJavascriptSerializationUtil.readModuleAsProto(it.body, it.moduleName) }
}
.toList()
const val TEST_DATA_DIR_PATH = "js/js.translator/testData/"
const val DIST_DIR_JS_PATH = "dist/js/"