From ec03c6decb70db8cb44093e6a15cfb5e94cd5cb1 Mon Sep 17 00:00:00 2001 From: Alexander Udalov Date: Mon, 21 Nov 2016 12:18:26 +0300 Subject: [PATCH] Introduce K2MetadataCompiler, extract MetadataSerializer out of BuiltInsSerializer K2MetadataCompiler is a compiler facade similar to K2JVMCompiler and K2JSCompiler and it produces .kotlin_metadata files. Each .kotlin_metadata file contains the binary data which is a serialized BuiltIns protobuf message. There's no 'kotlinc-***' script yet though, so to run this compiler currently invoke this: KOTLIN_COMPILER=org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler kotlinc --- .../serialization/MetadataSerializer.kt | 147 ++++++++++++++++++ .../builtins/BuiltInsSerializer.kt | 138 +++------------- .../K2MetadataCompilerArguments.java | 29 ++++ compiler/cli/cli.iml | 1 + .../cli/common/CLIConfigurationKeys.java | 7 + .../kotlin/cli/metadata/K2MetadataCompiler.kt | 107 +++++++++++++ 6 files changed, 313 insertions(+), 116 deletions(-) create mode 100644 compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt create mode 100644 compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2MetadataCompilerArguments.java create mode 100644 compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt diff --git a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt new file mode 100644 index 00000000000..c443f72f433 --- /dev/null +++ b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt @@ -0,0 +1,147 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.serialization + +import org.jetbrains.kotlin.analyzer.AnalysisResult +import org.jetbrains.kotlin.analyzer.common.DefaultAnalyzerFacade +import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol +import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion +import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.descriptors.ClassDescriptor +import org.jetbrains.kotlin.descriptors.ClassKind +import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor +import org.jetbrains.kotlin.descriptors.PackageViewDescriptor +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter +import org.jetbrains.kotlin.resolve.scopes.MemberScope +import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf +import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.io.File + +open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { + protected var totalSize = 0 + protected var totalFiles = 0 + + protected open fun getPackageFilePath(fqName: FqName): String = + fqName.asString().replace('.', '/') + "/" + (if (fqName.isRoot) "default-package" else fqName.shortName().asString()) + "." + METADATA_FILE_EXTENSION + + fun serialize(environment: KotlinCoreEnvironment) { + val configuration = environment.configuration + val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) + val files = environment.getSourceFiles() + val moduleName = Name.special("<${configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)}>") + + val destDir = configuration.get(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY) ?: run { + messageCollector.report(CompilerMessageSeverity.ERROR, "Specify destination via -d", CompilerMessageLocation.NO_LOCATION) + return + } + + val analyzer = AnalyzerWithCompilerReport(messageCollector) + analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer { + override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns) + }) + + val moduleDescriptor = analyzer.analysisResult.moduleDescriptor + + destDir.deleteRecursively() + + if (!destDir.mkdirs()) { + throw AssertionError("Could not make directories: " + destDir) + } + + files.map { it.packageFqName }.toSet().forEach { + fqName -> + PackageSerializer(moduleDescriptor.getPackage(fqName), destDir, { bytesWritten -> + totalSize += bytesWritten + totalFiles++ + }).run() + } + } + + private inner class PackageSerializer( + private val packageView: PackageViewDescriptor, + private val destDir: File, + private val onFileWrite: (bytesWritten: Int) -> Unit + ) { + private val fqName = packageView.fqName + private val fragments = packageView.fragments + private val proto = BuiltInsProtoBuf.BuiltIns.newBuilder() + private val extension = BuiltInsSerializerExtension(fragments) + + fun run() { + serializeClasses(packageView.memberScope) + serializePackageFragments(fragments) + serializeStringTable() + serializeBuiltInsFile() + } + + private fun serializeClass(classDescriptor: ClassDescriptor) { + val classProto = DescriptorSerializer.createTopLevel(extension).classProto(classDescriptor).build() + proto.addClass_(classProto) + + serializeClasses(classDescriptor.unsubstitutedInnerClassesScope) + } + + private fun serializeClasses(scope: MemberScope) { + for (descriptor in DescriptorSerializer.sort(scope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))) { + if (descriptor is ClassDescriptor && descriptor.kind != ClassKind.ENUM_ENTRY) { + serializeClass(descriptor) + } + } + } + + private fun serializePackageFragments(fragments: List) { + proto.`package` = DescriptorSerializer.createTopLevel(extension).packageProto(fragments).build() + } + + private fun serializeStringTable() { + val (strings, qualifiedNames) = extension.stringTable.buildProto() + proto.strings = strings + proto.qualifiedNames = qualifiedNames + } + + private fun serializeBuiltInsFile() { + val stream = ByteArrayOutputStream() + with(DataOutputStream(stream)) { + val version = BuiltInsBinaryVersion.INSTANCE.toArray() + writeInt(version.size) + version.forEach { writeInt(it) } + } + proto.build().writeTo(stream) + write(getPackageFilePath(fqName), stream) + } + + private fun write(fileName: String, stream: ByteArrayOutputStream) { + onFileWrite(stream.size()) + val file = File(destDir, fileName) + file.parentFile.mkdirs() + file.writeBytes(stream.toByteArray()) + } + } + + companion object { + private val METADATA_FILE_EXTENSION = ".kotlin_metadata" + } +} diff --git a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt index 9313d6cd011..aa3dce53162 100644 --- a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt +++ b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 JetBrains s.r.o. + * Copyright 2010-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,35 +16,21 @@ package org.jetbrains.kotlin.serialization.builtins -import com.intellij.openapi.Disposable import com.intellij.openapi.util.Disposer -import org.jetbrains.kotlin.analyzer.AnalysisResult -import org.jetbrains.kotlin.analyzer.common.DefaultAnalyzerFacade import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol -import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.messages.* import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots +import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.addKotlinSourceRoots -import org.jetbrains.kotlin.descriptors.ClassDescriptor -import org.jetbrains.kotlin.descriptors.ClassKind -import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor -import org.jetbrains.kotlin.descriptors.PackageViewDescriptor -import org.jetbrains.kotlin.name.Name -import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter -import org.jetbrains.kotlin.resolve.scopes.MemberScope -import org.jetbrains.kotlin.serialization.DescriptorSerializer -import java.io.ByteArrayOutputStream -import java.io.DataOutputStream +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.serialization.MetadataSerializer import java.io.File -class BuiltInsSerializer(private val dependOnOldBuiltIns: Boolean) { - private var totalSize = 0 - private var totalFiles = 0 - +class BuiltInsSerializer(dependOnOldBuiltIns: Boolean) : MetadataSerializer(dependOnOldBuiltIns) { fun serialize( destDir: File, srcDirs: List, @@ -52,51 +38,30 @@ class BuiltInsSerializer(private val dependOnOldBuiltIns: Boolean) { onComplete: (totalSize: Int, totalFiles: Int) -> Unit ) { val rootDisposable = Disposer.newDisposable() + val messageCollector = createMessageCollector() try { - serialize(rootDisposable, destDir, srcDirs, extraClassPath) + val configuration = CompilerConfiguration().apply { + put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, messageCollector) + + addKotlinSourceRoots(srcDirs.map { it.path }) + addJvmClasspathRoots(extraClassPath) + + put(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY, destDir) + put(CommonConfigurationKeys.MODULE_NAME, "module for built-ins serialization") + } + + val environment = KotlinCoreEnvironment.createForTests(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) + + serialize(environment) + onComplete(totalSize, totalFiles) } finally { + messageCollector.flush() Disposer.dispose(rootDisposable) } } - private fun serialize(disposable: Disposable, destDir: File, srcDirs: List, extraClassPath: List) { - val configuration = CompilerConfiguration().apply { - put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, createMessageCollector()) - - addKotlinSourceRoots(srcDirs.map { it.path }) - addJvmClasspathRoots(extraClassPath) - } - - val environment = KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) - - val files = environment.getSourceFiles() - - val analyzer = AnalyzerWithCompilerReport(MessageCollector.NONE) - analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer { - override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles( - files, Name.special(""), dependOnOldBuiltIns - ) - }) - - val moduleDescriptor = analyzer.analysisResult.moduleDescriptor - - destDir.deleteRecursively() - - if (!destDir.mkdirs()) { - throw AssertionError("Could not make directories: " + destDir) - } - - files.map { it.packageFqName }.toSet().forEach { - fqName -> - PackageSerializer(moduleDescriptor.getPackage(fqName), destDir, { bytesWritten -> - totalSize += bytesWritten - totalFiles++ - }).run() - } - } - private fun createMessageCollector() = object : GroupingMessageCollector( PrintingMessageCollector(System.err, MessageRenderer.PLAIN_RELATIVE_PATHS, /* verbose = */ false) ) { @@ -109,64 +74,5 @@ class BuiltInsSerializer(private val dependOnOldBuiltIns: Boolean) { } } - private class PackageSerializer( - private val packageView: PackageViewDescriptor, - private val destDir: File, - private val onFileWrite: (bytesWritten: Int) -> Unit - ) { - private val fqName = packageView.fqName - private val fragments = packageView.fragments - private val proto = BuiltInsProtoBuf.BuiltIns.newBuilder() - private val extension = BuiltInsSerializerExtension(fragments) - - fun run() { - serializeClasses(packageView.memberScope) - serializePackageFragments(fragments) - serializeStringTable() - serializeBuiltInsFile() - } - - private fun serializeClass(classDescriptor: ClassDescriptor) { - val classProto = DescriptorSerializer.createTopLevel(extension).classProto(classDescriptor).build() - proto.addClass_(classProto) - - serializeClasses(classDescriptor.unsubstitutedInnerClassesScope) - } - - private fun serializeClasses(scope: MemberScope) { - for (descriptor in DescriptorSerializer.sort(scope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))) { - if (descriptor is ClassDescriptor && descriptor.kind != ClassKind.ENUM_ENTRY) { - serializeClass(descriptor) - } - } - } - - private fun serializePackageFragments(fragments: List) { - proto.`package` = DescriptorSerializer.createTopLevel(extension).packageProto(fragments).build() - } - - private fun serializeStringTable() { - val (strings, qualifiedNames) = extension.stringTable.buildProto() - proto.strings = strings - proto.qualifiedNames = qualifiedNames - } - - private fun serializeBuiltInsFile() { - val stream = ByteArrayOutputStream() - with(DataOutputStream(stream)) { - val version = BuiltInsBinaryVersion.INSTANCE.toArray() - writeInt(version.size) - version.forEach { writeInt(it) } - } - proto.build().writeTo(stream) - write(BuiltInSerializerProtocol.getBuiltInsFilePath(fqName), stream) - } - - private fun write(fileName: String, stream: ByteArrayOutputStream) { - onFileWrite(stream.size()) - val file = File(destDir, fileName) - file.parentFile.mkdirs() - file.writeBytes(stream.toByteArray()) - } - } + override fun getPackageFilePath(fqName: FqName): String = BuiltInSerializerProtocol.getBuiltInsFilePath(fqName) } diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2MetadataCompilerArguments.java b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2MetadataCompilerArguments.java new file mode 100644 index 00000000000..4113cb6b528 --- /dev/null +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2MetadataCompilerArguments.java @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.cli.common.arguments; + +import com.sampullara.cli.Argument; + +public class K2MetadataCompilerArguments extends CommonCompilerArguments { + @Argument(value = "d", description = "Destination for generated .kotlin_metadata files") + @ValueDescription("") + public String destination; + + @Argument(value = "classpath", alias = "cp", description = "Paths where to find library .kotlin_metadata files") + @ValueDescription("") + public String classpath; +} diff --git a/compiler/cli/cli.iml b/compiler/cli/cli.iml index f1f8ee6d816..8be341ef098 100644 --- a/compiler/cli/cli.iml +++ b/compiler/cli/cli.iml @@ -21,5 +21,6 @@ + \ No newline at end of file diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLIConfigurationKeys.java b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLIConfigurationKeys.java index 33e0a585fbc..69aef735300 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLIConfigurationKeys.java +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLIConfigurationKeys.java @@ -20,6 +20,8 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.jvm.compiler.CompilerJarLocator; import org.jetbrains.kotlin.config.CompilerConfigurationKey; +import java.io.File; + public class CLIConfigurationKeys { public static final CompilerConfigurationKey MESSAGE_COLLECTOR_KEY = CompilerConfigurationKey.create("message collector"); @@ -32,6 +34,11 @@ public class CLIConfigurationKeys { public static final CompilerConfigurationKey COMPILER_JAR_LOCATOR = CompilerConfigurationKey.create("compiler jar locator"); + // See K2MetadataCompilerArguments + + public static final CompilerConfigurationKey METADATA_DESTINATION_DIRECTORY = + CompilerConfigurationKey.create("metadata destination directory"); + private CLIConfigurationKeys() { } } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt new file mode 100644 index 00000000000..9d2048dd6c7 --- /dev/null +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.cli.metadata + +import com.intellij.openapi.Disposable +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.MessageUtil +import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil +import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots +import org.jetbrains.kotlin.codegen.CompilationException +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.config.addKotlinSourceRoot +import org.jetbrains.kotlin.load.java.JvmAbi +import org.jetbrains.kotlin.serialization.MetadataSerializer +import java.io.File + +class K2MetadataCompiler : CLICompiler() { + override fun createArguments() = K2MetadataCompilerArguments() + + override fun setupPlatformSpecificArgumentsAndServices( + configuration: CompilerConfiguration, arguments: K2MetadataCompilerArguments, services: Services + ) { + // No specific arguments yet + } + + override fun doExecute( + arguments: K2MetadataCompilerArguments, configuration: CompilerConfiguration, rootDisposable: Disposable + ): ExitCode { + val collector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) + + for (arg in arguments.freeArgs) { + configuration.addKotlinSourceRoot(arg) + } + if (arguments.classpath != null) { + configuration.addJvmClasspathRoots(arguments.classpath.split(File.pathSeparatorChar).map(::File)) + } + + configuration.put(CommonConfigurationKeys.MODULE_NAME, JvmAbi.DEFAULT_MODULE_NAME) + + val destination = arguments.destination + if (destination != null) { + if (destination.endsWith(".jar")) { + // TODO: support .jar destination + collector.report( + CompilerMessageSeverity.WARNING, + ".jar destination is not yet supported, results will be written to the directory with the given name", + CompilerMessageLocation.NO_LOCATION + ) + } + configuration.put(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY, File(destination)) + } + + val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) + + if (environment.getSourceFiles().isEmpty()) { + if (arguments.version) { + return ExitCode.OK + } + collector.report(CompilerMessageSeverity.ERROR, "No source files", CompilerMessageLocation.NO_LOCATION) + return ExitCode.COMPILATION_ERROR + } + + try { + MetadataSerializer(true).serialize(environment) + } + catch (e: CompilationException) { + collector.report( + CompilerMessageSeverity.EXCEPTION, + OutputMessageUtil.renderException(e), + MessageUtil.psiElementToMessageLocation(e.element) + ) + return ExitCode.INTERNAL_ERROR + } + + return ExitCode.OK + } + + companion object { + @JvmStatic + fun main(args: Array) { + doMain(K2MetadataCompiler(), args) + } + } +}