azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce services to pass data between IC and JS compiler

Browse Source
This commit is contained in:
Alexey Tsvetkov
2017-07-28 16:56:24 +03:00
parent 8ffd141d17
commit f5e77c740d
9 changed files with 182 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java
+39 -2
View File
@@ -56,6 +56,10 @@ import org.jetbrains.kotlin.js.facade.K2JSTranslator;
import org.jetbrains.kotlin.js.facade.MainCallParameters;
import org.jetbrains.kotlin.js.facade.TranslationResult;
import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver;
import org.jetbrains.kotlin.js.facade.TranslationUnit;
import org.jetbrains.kotlin.js.facade.exceptions.TranslationException;
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider;
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer;
import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus;
import org.jetbrains.kotlin.psi.KtFile;
import org.jetbrains.kotlin.serialization.js.ModuleKind;
@@ -99,6 +103,30 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
return new K2JSCompilerArguments();
}
@NotNull
protected TranslationResult translate(
@NotNull JsConfig.Reporter reporter,
@NotNull List<KtFile> allKotlinFiles,
@NotNull JsAnalysisResult jsAnalysisResult,
@NotNull MainCallParameters mainCallParameters,
@NotNull JsConfig config
) throws TranslationException {
K2JSTranslator translator = new K2JSTranslator(config);
IncrementalDataProvider incrementalDataProvider = config.getConfiguration().get(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER);
if (incrementalDataProvider != null) {
List<TranslationUnit> translationUnits = new ArrayList<>();
for (KtFile ktFile : allKotlinFiles) {
translationUnits.add(new TranslationUnit.SourceFile(ktFile));
}
for (byte[] binaryTree : incrementalDataProvider.getBinaryTrees()) {
translationUnits.add(new TranslationUnit.BinaryAst(binaryTree));
}
return translator.translateUnits(reporter, translationUnits, mainCallParameters, jsAnalysisResult);
}
return translator.translate(reporter, allKotlinFiles, mainCallParameters, jsAnalysisResult);
}
@NotNull
@Override
protected ExitCode doExecute(
@@ -205,10 +233,9 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
MainCallParameters mainCallParameters = createMainCallParameters(arguments.getMain());
TranslationResult translationResult;
K2JSTranslator translator = new K2JSTranslator(config);
try {
//noinspection unchecked
translationResult = translator.translate(reporter, sourcesFiles, mainCallParameters, jsAnalysisResult);
translationResult = translate(reporter, sourcesFiles, jsAnalysisResult, mainCallParameters, config);
}
catch (Exception e) {
throw ExceptionUtilsKt.rethrow(e);
@@ -344,6 +371,16 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
}
configuration.put(JSConfigurationKeys.MODULE_KIND, moduleKind);
IncrementalDataProvider incrementalDataProvider = services.get(IncrementalDataProvider.class);
if (incrementalDataProvider != null) {
configuration.put(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER, incrementalDataProvider);
}
IncrementalResultsConsumer incrementalResultsConsumer = services.get(IncrementalResultsConsumer.class);
if (incrementalResultsConsumer != null) {
configuration.put(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER, incrementalResultsConsumer);
}
String sourceMapEmbedContentString = arguments.getSourceMapEmbedSources();
SourceMapSourceEmbedding sourceMapContentEmbedding = sourceMapEmbedContentString != null ?
sourceMapContentEmbeddingMap.get(sourceMapEmbedContentString) :
js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.incremental.js
// byte arrays are used to simplify passing to different classloaders
interface IncrementalDataProvider {
/** gets header metadata (serialized [JsProtoBuf.Header]) from previous compilation */
val headerMetadata: ByteArray
/** gets non-dirty package parts metadata (serialized [ProtoBuf.PackageFragment]) from previous compilation */
val packagePartsMetadata: List<ByteArray>
/** gets non-dirty package parts binary trees from previous compilation */
val binaryTrees: List<ByteArray>
}
class IncrementalDataProviderImpl(
override val headerMetadata: ByteArray,
override val packagePartsMetadata: List<ByteArray>,
override val binaryTrees: List<ByteArray>
) : IncrementalDataProvider
js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt
+46
View File
@@ -0,0 +1,46 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.incremental.js
import java.io.File
interface IncrementalResultsConsumer {
/** processes new header metadata (serialized [JsProtoBuf.Header]) */
fun processHeader(headerMetadata: ByteArray)
/** processes new package part metadata and binary tree for compiled source file */
fun processPackagePart(sourceFile: File, packagePartMetadata: ByteArray, binaryAst: ByteArray)
}
class IncrementalResultsConsumerImpl : IncrementalResultsConsumer {
lateinit var headerMetadata: ByteArray
private set
private val _packageParts = arrayListOf<PackagePartData>()
val packageParts: List<PackagePartData>
get() = _packageParts
override fun processHeader(headerMetadata: ByteArray) {
this.headerMetadata = headerMetadata
}
override fun processPackagePart(sourceFile: File, packagePartMetadata: ByteArray, binaryAst: ByteArray) {
_packageParts.add(PackagePartData(sourceFile, packagePartMetadata, binaryAst))
}
data class PackagePartData(val sourceFile: File, val proto: ByteArray, val binaryAst: ByteArray)
}
js/js.frontend/src/org/jetbrains/kotlin/js/analyze/TopDownAnalyzerFacadeForJS.kt
+6 -2
View File
@@ -32,6 +32,7 @@ import org.jetbrains.kotlin.resolve.*
import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProviderFactory
import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil
import org.jetbrains.kotlin.serialization.js.PackagesWithHeaderMetadata
object TopDownAnalyzerFacadeForJS {
@JvmStatic
@@ -57,8 +58,11 @@ object TopDownAnalyzerFacadeForJS {
moduleContext: ModuleContext,
config: JsConfig
): JsAnalysisResult {
val packageFragment = config.configuration[JSConfigurationKeys.FALLBACK_METADATA]?.let {
KotlinJavascriptSerializationUtil.readDescriptors(it, moduleContext.storageManager, moduleContext.module,
val packageFragment = config.configuration[JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER]?.let {
val metadata = PackagesWithHeaderMetadata(it.headerMetadata, it.packagePartsMetadata)
KotlinJavascriptSerializationUtil.readDescriptors(metadata,
moduleContext.storageManager,
moduleContext.module,
DeserializationConfiguration.Default)
}
val analyzerForJs = createTopDownAnalyzerForJs(
js/js.frontend/src/org/jetbrains/kotlin/js/config/JSConfigurationKeys.java
+6 -4
View File
@@ -17,8 +17,9 @@
package org.jetbrains.kotlin.js.config;
import org.jetbrains.kotlin.config.CompilerConfigurationKey;
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider;
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer;
import org.jetbrains.kotlin.serialization.js.ModuleKind;
import org.jetbrains.kotlin.serialization.js.PackagesWithHeaderMetadata;
import java.util.List;
@@ -50,10 +51,11 @@ public class JSConfigurationKeys {
public static final CompilerConfigurationKey<Boolean> TYPED_ARRAYS_ENABLED =
CompilerConfigurationKey.create("TypedArrays enabled");
public static final CompilerConfigurationKey<PackagesWithHeaderMetadata> FALLBACK_METADATA =
CompilerConfigurationKey.create("fallback metadata");
public static final CompilerConfigurationKey<IncrementalDataProvider> INCREMENTAL_DATA_PROVIDER =
CompilerConfigurationKey.create("incremental data provider");
public static final CompilerConfigurationKey<Boolean> SERIALIZE_FRAGMENTS = CompilerConfigurationKey.create("serialize fragments");
public static final CompilerConfigurationKey<IncrementalResultsConsumer> INCREMENTAL_RESULTS_CONSUMER =
CompilerConfigurationKey.create("incremental results consumer");
public static final CompilerConfigurationKey<Boolean> FRIEND_PATHS_DISABLED =
CompilerConfigurationKey.create("disable support for friend paths");
js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt
+26 -26
View File
@@ -31,6 +31,8 @@ import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.incremental.js.IncrementalDataProviderImpl
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.js.JavaScript
import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor
import org.jetbrains.kotlin.js.backend.ast.*
@@ -52,7 +54,6 @@ import org.jetbrains.kotlin.psi.KtNamedFunction
import org.jetbrains.kotlin.psi.KtPsiFactory
import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil
import org.jetbrains.kotlin.serialization.js.ModuleKind
import org.jetbrains.kotlin.serialization.js.PackagesWithHeaderMetadata
import org.jetbrains.kotlin.test.InTextDirectivesUtils
import org.jetbrains.kotlin.test.KotlinTestUtils
import org.jetbrains.kotlin.test.KotlinTestUtils.TestFileFactory
@@ -348,10 +349,14 @@ abstract class BasicBoxTest(
private fun removeRecompiledSuffix(text: String): String = text.replace("-recompiled.js", ".js")
protected fun translateFiles(units: List<TranslationUnit>, outputFile: File, config: JsConfig,
protected fun translateFiles(
units: List<TranslationUnit>,
outputFile: File,
config: JsConfig,
outputPrefixFile: File?,
outputPostfixFile: File?,
mainCallParameters: MainCallParameters) {
mainCallParameters: MainCallParameters
) {
val translator = K2JSTranslator(config)
val translationResult = translator.translateUnits(ExceptionThrowingReporter, units, mainCallParameters)
@@ -382,25 +387,19 @@ abstract class BasicBoxTest(
FileUtil.writeToFile(outputFile, wrappedContent)
}
val incrementalDir = File(outputDir, "incremental/${outputFile.nameWithoutExtension}")
config.configuration[JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER]?.let {
val incrementalService = it as IncrementalResultsConsumerImpl
for ((index, unit) in units.withIndex()) {
if (unit !is TranslationUnit.SourceFile) continue
val fileTranslationResult = translationResult.fileTranslationResults[unit.file]!!
val basePath = "$index."
val binaryAst = fileTranslationResult.binaryAst
val binaryMetadata = fileTranslationResult.metadata
val incrementalDir = File(outputDir, "incremental/${outputFile.nameWithoutExtension}")
if (binaryAst != null) {
FileUtil.writeToFile(File(incrementalDir, basePath + AST_EXTENSION), binaryAst)
for ((i, packagePart) in incrementalService.packageParts.withIndex()) {
FileUtil.writeToFile(File(incrementalDir, "$i.$AST_EXTENSION"), packagePart.binaryAst)
FileUtil.writeToFile(File(incrementalDir, "$i.$METADATA_EXTENSION"), packagePart.proto)
}
if (binaryMetadata != null) {
FileUtil.writeToFile(File(incrementalDir, basePath + METADATA_EXTENSION), binaryMetadata)
}
}
translationResult.metadataHeader?.let {
FileUtil.writeToFile(File(incrementalDir, HEADER_FILE), it)
incrementalService.headerMetadata?.let {
FileUtil.writeToFile(File(incrementalDir, HEADER_FILE), it)
}
}
processJsProgram(translationResult.program, units.filterIsInstance<TranslationUnit.SourceFile>().map { it.file })
@@ -496,18 +495,19 @@ abstract class BasicBoxTest(
val hasFilesToRecompile = module.hasFilesToRecompile
configuration.put(JSConfigurationKeys.META_INFO, multiModule)
configuration.put(JSConfigurationKeys.SERIALIZE_FRAGMENTS, hasFilesToRecompile)
if (hasFilesToRecompile) {
if (additionalMetadata != null) {
val (headerFile, packagePartFiles) = additionalMetadata
configuration.put(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER,
IncrementalDataProviderImpl(headerFile.readBytes(), packagePartFiles.map { it.readBytes() }, emptyList()))
}
configuration.put(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER, IncrementalResultsConsumerImpl())
}
configuration.put(JSConfigurationKeys.SOURCE_MAP, hasFilesToRecompile || generateSourceMap)
configuration.put(JSConfigurationKeys.SOURCE_MAP_SOURCE_ROOTS, sourceDirs)
configuration.put(JSConfigurationKeys.SOURCE_MAP_EMBED_SOURCES, module.sourceMapSourceEmbedding)
if (additionalMetadata != null) {
val metadata = PackagesWithHeaderMetadata(
FileUtil.loadFileBytes(additionalMetadata.first),
additionalMetadata.second.map { FileUtil.loadFileBytes(it) })
configuration.put(JSConfigurationKeys.FALLBACK_METADATA, metadata)
}
if (typedArraysEnabled) {
configuration.put(JSConfigurationKeys.TYPED_ARRAYS_ENABLED, true)
}
js/js.translator/src/org/jetbrains/kotlin/js/facade/K2JSTranslator.java
+25 -28
View File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.js.facade;
import com.intellij.openapi.vfs.VfsUtilCore;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor;
@@ -28,13 +29,13 @@ import org.jetbrains.kotlin.js.config.JSConfigurationKeys;
import org.jetbrains.kotlin.js.config.JsConfig;
import org.jetbrains.kotlin.js.coroutine.CoroutineTransformer;
import org.jetbrains.kotlin.js.facade.exceptions.TranslationException;
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer;
import org.jetbrains.kotlin.js.inline.JsInliner;
import org.jetbrains.kotlin.js.inline.clean.LabeledBlockToDoWhileTransformation;
import org.jetbrains.kotlin.js.inline.clean.RemoveUnusedImportsKt;
import org.jetbrains.kotlin.js.inline.clean.ResolveTemporaryNamesKt;
import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver;
import org.jetbrains.kotlin.js.translate.general.AstGenerationResult;
import org.jetbrains.kotlin.js.translate.general.FileTranslationResult;
import org.jetbrains.kotlin.js.translate.general.Translation;
import org.jetbrains.kotlin.js.translate.utils.ExpandIsCallsKt;
import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus;
@@ -49,9 +50,7 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.jetbrains.kotlin.diagnostics.DiagnosticUtils.hasError;
@@ -64,8 +63,12 @@ public final class K2JSTranslator {
@NotNull
private final JsConfig config;
@Nullable
private final IncrementalResultsConsumer incrementalResults;
public K2JSTranslator(@NotNull JsConfig config) {
this.config = config;
this.incrementalResults = config.getConfiguration().get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER);
}
@NotNull
@@ -151,7 +154,6 @@ public final class K2JSTranslator {
List<File> sourceRoots = config.getSourceMapRoots().stream().map(File::new).collect(Collectors.toList());
SourceFilePathResolver pathResolver = new SourceFilePathResolver(sourceRoots);
Map<KtFile, FileTranslationResult> fileMap = new HashMap<>();
JsAstSerializer serializer = new JsAstSerializer(file -> {
try {
return pathResolver.getPathRelativeToSourceRoots(file);
@@ -160,32 +162,27 @@ public final class K2JSTranslator {
throw new RuntimeException("IO error occurred resolving path to source file", e);
}
});
byte[] metadataHeader = null;
boolean serializeFragments = config.getConfiguration().get(JSConfigurationKeys.SERIALIZE_FRAGMENTS, false);
for (KtFile file : files) {
List<DeclarationDescriptor> scope = translationResult.getFileMemberScopes().get(file);
byte[] binaryAst = null;
byte[] binaryMetadata = null;
if (serializeFragments) {
JsProgramFragment fragment = translationResult.getFragmentMap().get(file);
if (fragment != null) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
serializer.serialize(fragment, output);
binaryAst = output.toByteArray();
}
if (scope != null) {
ProtoBuf.PackageFragment part = KotlinJavascriptSerializationUtil.INSTANCE.serializeDescriptors(
bindingTrace.getBindingContext(), moduleDescriptor, scope, file.getPackageFqName());
binaryMetadata = part.toByteArray();
}
if (incrementalResults != null) {
KotlinJavascriptSerializationUtil serializationUtil = KotlinJavascriptSerializationUtil.INSTANCE;
for (KtFile file : files) {
JsProgramFragment fragment = translationResult.getFragmentMap().get(file);
assert fragment != null : "Could not find AST for file: " + file;
ByteArrayOutputStream output = new ByteArrayOutputStream();
serializer.serialize(fragment, output);
byte[] binaryAst = output.toByteArray();
List<DeclarationDescriptor> scope = translationResult.getFileMemberScopes().get(file);
assert scope != null : "Could not find descriptors for file: " + file;
ProtoBuf.PackageFragment packagePart = serializationUtil.serializeDescriptors(
bindingTrace.getBindingContext(), moduleDescriptor, scope, file.getPackageFqName());
File ioFile = VfsUtilCore.virtualToIoFile(file.getVirtualFile());
incrementalResults.processPackagePart(ioFile, packagePart.toByteArray(), binaryAst);
}
fileMap.put(file, new FileTranslationResult(file, binaryMetadata, binaryAst));
}
if (serializeFragments) {
metadataHeader = KotlinJavascriptSerializationUtil.INSTANCE.serializeHeader(null).toByteArray();
incrementalResults.processHeader(serializationUtil.serializeHeader(null).toByteArray());
}
ResolveTemporaryNamesKt.resolveTemporaryNames(translationResult.getProgram());
@@ -198,6 +195,6 @@ public final class K2JSTranslator {
}
return new TranslationResult.Success(config, files, translationResult.getProgram(), diagnostics, importedModules,
moduleDescriptor, bindingTrace.getBindingContext(), metadataHeader, fileMap);
moduleDescriptor, bindingTrace.getBindingContext());
}
}
js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt
+1 -4
View File
@@ -29,7 +29,6 @@ import org.jetbrains.kotlin.js.config.SourceMapSourceEmbedding
import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
import org.jetbrains.kotlin.js.backend.SourceLocationConsumer
import org.jetbrains.kotlin.js.translate.general.FileTranslationResult
import org.jetbrains.kotlin.js.util.TextOutput
import org.jetbrains.kotlin.js.util.TextOutputImpl
import org.jetbrains.kotlin.psi.KtFile
@@ -51,9 +50,7 @@ abstract class TranslationResult protected constructor(val diagnostics: Diagnost
diagnostics: Diagnostics,
private val importedModules: List<String>,
val moduleDescriptor: ModuleDescriptor,
val bindingContext: BindingContext,
val metadataHeader: ByteArray?,
val fileTranslationResults: Map<KtFile, FileTranslationResult>
val bindingContext: BindingContext
) : TranslationResult(diagnostics) {
@Suppress("unused") // Used in kotlin-web-demo in WebDemoTranslatorFacade
fun getCode(): String {
js/js.translator/src/org/jetbrains/kotlin/js/translate/general/FileTranslationResult.kt
-21
View File
@@ -1,21 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.js.translate.general
import org.jetbrains.kotlin.psi.KtFile
class FileTranslationResult(val file: KtFile, val metadata: ByteArray?, val binaryAst: ByteArray?)