Introduce K2MetadataCompiler, extract MetadataSerializer out of BuiltInsSerializer

K2MetadataCompiler is a compiler facade similar to K2JVMCompiler and
K2JSCompiler and it produces .kotlin_metadata files. Each .kotlin_metadata file
contains the binary data which is a serialized BuiltIns protobuf message.

There's no 'kotlinc-***' script yet though, so to run this compiler currently
invoke this:

    KOTLIN_COMPILER=org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler kotlinc

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.serialization
+
+import org.jetbrains.kotlin.analyzer.AnalysisResult
+import org.jetbrains.kotlin.analyzer.common.DefaultAnalyzerFacade
+import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
+import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.descriptors.ClassDescriptor
+import org.jetbrains.kotlin.descriptors.ClassKind
+import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.PackageViewDescriptor
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
+import org.jetbrains.kotlin.resolve.scopes.MemberScope
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension
+import java.io.ByteArrayOutputStream
+import java.io.DataOutputStream
+import java.io.File
+
+open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
+    protected var totalSize = 0
+    protected var totalFiles = 0
+
+    protected open fun getPackageFilePath(fqName: FqName): String =
+            fqName.asString().replace('.', '/') + "/" + (if (fqName.isRoot) "default-package" else fqName.shortName().asString()) + "." + METADATA_FILE_EXTENSION
+
+    fun serialize(environment: KotlinCoreEnvironment) {
+        val configuration = environment.configuration
+        val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
+        val files = environment.getSourceFiles()
+        val moduleName = Name.special("<${configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)}>")
+
+        val destDir = configuration.get(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY) ?: run {
+            messageCollector.report(CompilerMessageSeverity.ERROR, "Specify destination via -d", CompilerMessageLocation.NO_LOCATION)
+            return
+        }
+
+        val analyzer = AnalyzerWithCompilerReport(messageCollector)
+        analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer {
+            override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns)
+        })
+
+        val moduleDescriptor = analyzer.analysisResult.moduleDescriptor
+
+        destDir.deleteRecursively()
+
+        if (!destDir.mkdirs()) {
+            throw AssertionError("Could not make directories: " + destDir)
+        }
+
+        files.map { it.packageFqName }.toSet().forEach {
+            fqName ->
+            PackageSerializer(moduleDescriptor.getPackage(fqName), destDir, { bytesWritten ->
+                totalSize += bytesWritten
+                totalFiles++
+            }).run()
+        }
+    }
+
+    private inner class PackageSerializer(
+            private val packageView: PackageViewDescriptor,
+            private val destDir: File,
+            private val onFileWrite: (bytesWritten: Int) -> Unit
+    ) {
+        private val fqName = packageView.fqName
+        private val fragments = packageView.fragments
+        private val proto = BuiltInsProtoBuf.BuiltIns.newBuilder()
+        private val extension = BuiltInsSerializerExtension(fragments)
+
+        fun run() {
+            serializeClasses(packageView.memberScope)
+            serializePackageFragments(fragments)
+            serializeStringTable()
+            serializeBuiltInsFile()
+        }
+
+        private fun serializeClass(classDescriptor: ClassDescriptor) {
+            val classProto = DescriptorSerializer.createTopLevel(extension).classProto(classDescriptor).build()
+            proto.addClass_(classProto)
+
+            serializeClasses(classDescriptor.unsubstitutedInnerClassesScope)
+        }
+
+        private fun serializeClasses(scope: MemberScope) {
+            for (descriptor in DescriptorSerializer.sort(scope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))) {
+                if (descriptor is ClassDescriptor && descriptor.kind != ClassKind.ENUM_ENTRY) {
+                    serializeClass(descriptor)
+                }
+            }
+        }
+
+        private fun serializePackageFragments(fragments: List<PackageFragmentDescriptor>) {
+            proto.`package` = DescriptorSerializer.createTopLevel(extension).packageProto(fragments).build()
+        }
+
+        private fun serializeStringTable() {
+            val (strings, qualifiedNames) = extension.stringTable.buildProto()
+            proto.strings = strings
+            proto.qualifiedNames = qualifiedNames
+        }
+
+        private fun serializeBuiltInsFile() {
+            val stream = ByteArrayOutputStream()
+            with(DataOutputStream(stream)) {
+                val version = BuiltInsBinaryVersion.INSTANCE.toArray()
+                writeInt(version.size)
+                version.forEach { writeInt(it) }
+            }
+            proto.build().writeTo(stream)
+            write(getPackageFilePath(fqName), stream)
+        }
+
+        private fun write(fileName: String, stream: ByteArrayOutputStream) {
+            onFileWrite(stream.size())
+            val file = File(destDir, fileName)
+            file.parentFile.mkdirs()
+            file.writeBytes(stream.toByteArray())
+        }
+    }
+
+    companion object {
+        private val METADATA_FILE_EXTENSION = ".kotlin_metadata"
+    }
+}

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/builtins/BuiltInsSerializer.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2015 JetBrains s.r.o.
+ * Copyright 2010-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,35 +16,21 @@
 
 package org.jetbrains.kotlin.serialization.builtins
 
-import com.intellij.openapi.Disposable
 import com.intellij.openapi.util.Disposer
-import org.jetbrains.kotlin.analyzer.AnalysisResult
-import org.jetbrains.kotlin.analyzer.common.DefaultAnalyzerFacade
 import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
-import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.addKotlinSourceRoots
-import org.jetbrains.kotlin.descriptors.ClassDescriptor
-import org.jetbrains.kotlin.descriptors.ClassKind
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
-import org.jetbrains.kotlin.descriptors.PackageViewDescriptor
-import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
-import org.jetbrains.kotlin.resolve.scopes.MemberScope
-import org.jetbrains.kotlin.serialization.DescriptorSerializer
-import java.io.ByteArrayOutputStream
-import java.io.DataOutputStream
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.serialization.MetadataSerializer
 import java.io.File
 
-class BuiltInsSerializer(private val dependOnOldBuiltIns: Boolean) {
-    private var totalSize = 0
-    private var totalFiles = 0
-
+class BuiltInsSerializer(dependOnOldBuiltIns: Boolean) : MetadataSerializer(dependOnOldBuiltIns) {
     fun serialize(
             destDir: File,
             srcDirs: List<File>,
@@ -52,51 +38,30 @@ class BuiltInsSerializer(private val dependOnOldBuiltIns: Boolean) {
             onComplete: (totalSize: Int, totalFiles: Int) -> Unit
     ) {
         val rootDisposable = Disposer.newDisposable()
+        val messageCollector = createMessageCollector()
         try {
-            serialize(rootDisposable, destDir, srcDirs, extraClassPath)
+            val configuration = CompilerConfiguration().apply {
+                put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, messageCollector)
+
+                addKotlinSourceRoots(srcDirs.map { it.path })
+                addJvmClasspathRoots(extraClassPath)
+
+                put(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY, destDir)
+                put(CommonConfigurationKeys.MODULE_NAME, "module for built-ins serialization")
+            }
+
+            val environment = KotlinCoreEnvironment.createForTests(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
+
+            serialize(environment)
+
             onComplete(totalSize, totalFiles)
         }
         finally {
+            messageCollector.flush()
             Disposer.dispose(rootDisposable)
         }
     }
 
-    private fun serialize(disposable: Disposable, destDir: File, srcDirs: List<File>, extraClassPath: List<File>) {
-        val configuration = CompilerConfiguration().apply {
-            put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, createMessageCollector())
-
-            addKotlinSourceRoots(srcDirs.map { it.path })
-            addJvmClasspathRoots(extraClassPath)
-        }
-
-        val environment = KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
-
-        val files = environment.getSourceFiles()
-
-        val analyzer = AnalyzerWithCompilerReport(MessageCollector.NONE)
-        analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer {
-            override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(
-                    files, Name.special("<module for resolving builtin source files>"), dependOnOldBuiltIns
-            )
-        })
-
-        val moduleDescriptor = analyzer.analysisResult.moduleDescriptor
-
-        destDir.deleteRecursively()
-
-        if (!destDir.mkdirs()) {
-            throw AssertionError("Could not make directories: " + destDir)
-        }
-
-        files.map { it.packageFqName }.toSet().forEach {
-            fqName ->
-            PackageSerializer(moduleDescriptor.getPackage(fqName), destDir, { bytesWritten ->
-                totalSize += bytesWritten
-                totalFiles++
-            }).run()
-        }
-    }
-
     private fun createMessageCollector() = object : GroupingMessageCollector(
             PrintingMessageCollector(System.err, MessageRenderer.PLAIN_RELATIVE_PATHS, /* verbose = */ false)
     ) {
@@ -109,64 +74,5 @@ class BuiltInsSerializer(private val dependOnOldBuiltIns: Boolean) {
         }
     }
 
-    private class PackageSerializer(
-            private val packageView: PackageViewDescriptor,
-            private val destDir: File,
-            private val onFileWrite: (bytesWritten: Int) -> Unit
-    ) {
-        private val fqName = packageView.fqName
-        private val fragments = packageView.fragments
-        private val proto = BuiltInsProtoBuf.BuiltIns.newBuilder()
-        private val extension = BuiltInsSerializerExtension(fragments)
-
-        fun run() {
-            serializeClasses(packageView.memberScope)
-            serializePackageFragments(fragments)
-            serializeStringTable()
-            serializeBuiltInsFile()
-        }
-
-        private fun serializeClass(classDescriptor: ClassDescriptor) {
-            val classProto = DescriptorSerializer.createTopLevel(extension).classProto(classDescriptor).build()
-            proto.addClass_(classProto)
-
-            serializeClasses(classDescriptor.unsubstitutedInnerClassesScope)
-        }
-
-        private fun serializeClasses(scope: MemberScope) {
-            for (descriptor in DescriptorSerializer.sort(scope.getContributedDescriptors(DescriptorKindFilter.CLASSIFIERS))) {
-                if (descriptor is ClassDescriptor && descriptor.kind != ClassKind.ENUM_ENTRY) {
-                    serializeClass(descriptor)
-                }
-            }
-        }
-
-        private fun serializePackageFragments(fragments: List<PackageFragmentDescriptor>) {
-            proto.`package` = DescriptorSerializer.createTopLevel(extension).packageProto(fragments).build()
-        }
-
-        private fun serializeStringTable() {
-            val (strings, qualifiedNames) = extension.stringTable.buildProto()
-            proto.strings = strings
-            proto.qualifiedNames = qualifiedNames
-        }
-
-        private fun serializeBuiltInsFile() {
-            val stream = ByteArrayOutputStream()
-            with(DataOutputStream(stream)) {
-                val version = BuiltInsBinaryVersion.INSTANCE.toArray()
-                writeInt(version.size)
-                version.forEach { writeInt(it) }
-            }
-            proto.build().writeTo(stream)
-            write(BuiltInSerializerProtocol.getBuiltInsFilePath(fqName), stream)
-        }
-
-        private fun write(fileName: String, stream: ByteArrayOutputStream) {
-            onFileWrite(stream.size())
-            val file = File(destDir, fileName)
-            file.parentFile.mkdirs()
-            file.writeBytes(stream.toByteArray())
-        }
-    }
+    override fun getPackageFilePath(fqName: FqName): String = BuiltInSerializerProtocol.getBuiltInsFilePath(fqName)
 }

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2MetadataCompilerArguments.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.cli.common.arguments;
+
+import com.sampullara.cli.Argument;
+
+public class K2MetadataCompilerArguments extends CommonCompilerArguments {
+    @Argument(value = "d", description = "Destination for generated .kotlin_metadata files")
+    @ValueDescription("<directory|jar>")
+    public String destination;
+
+    @Argument(value = "classpath", alias = "cp", description = "Paths where to find library .kotlin_metadata files")
+    @ValueDescription("<path>")
+    public String classpath;
+}

compiler/cli/cli.iml

@@ -21,5 +21,6 @@
     <orderEntry type="module" module-name="js.serializer" />
     <orderEntry type="module" module-name="util" />
     <orderEntry type="module" module-name="annotation-collector" />
+    <orderEntry type="module" module-name="builtins-serializer" />
   </component>
 </module>

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLIConfigurationKeys.java

@@ -20,6 +20,8 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector;
 import org.jetbrains.kotlin.cli.jvm.compiler.CompilerJarLocator;
 import org.jetbrains.kotlin.config.CompilerConfigurationKey;
 
+import java.io.File;
+
 public class CLIConfigurationKeys {
     public static final CompilerConfigurationKey<MessageCollector> MESSAGE_COLLECTOR_KEY =
             CompilerConfigurationKey.create("message collector");
@@ -32,6 +34,11 @@ public class CLIConfigurationKeys {
     public static final CompilerConfigurationKey<CompilerJarLocator> COMPILER_JAR_LOCATOR =
             CompilerConfigurationKey.create("compiler jar locator");
 
+    // See K2MetadataCompilerArguments
+
+    public static final CompilerConfigurationKey<File> METADATA_DESTINATION_DIRECTORY =
+            CompilerConfigurationKey.create("metadata destination directory");
+
     private CLIConfigurationKeys() {
     }
 }

compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.cli.metadata
+
+import com.intellij.openapi.Disposable
+import org.jetbrains.kotlin.cli.common.CLICompiler
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.MessageUtil
+import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil
+import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
+import org.jetbrains.kotlin.codegen.CompilationException
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.config.addKotlinSourceRoot
+import org.jetbrains.kotlin.load.java.JvmAbi
+import org.jetbrains.kotlin.serialization.MetadataSerializer
+import java.io.File
+
+class K2MetadataCompiler : CLICompiler<K2MetadataCompilerArguments>() {
+    override fun createArguments() = K2MetadataCompilerArguments()
+
+    override fun setupPlatformSpecificArgumentsAndServices(
+            configuration: CompilerConfiguration, arguments: K2MetadataCompilerArguments, services: Services
+    ) {
+        // No specific arguments yet
+    }
+
+    override fun doExecute(
+            arguments: K2MetadataCompilerArguments, configuration: CompilerConfiguration, rootDisposable: Disposable
+    ): ExitCode {
+        val collector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
+
+        for (arg in arguments.freeArgs) {
+            configuration.addKotlinSourceRoot(arg)
+        }
+        if (arguments.classpath != null) {
+            configuration.addJvmClasspathRoots(arguments.classpath.split(File.pathSeparatorChar).map(::File))
+        }
+
+        configuration.put(CommonConfigurationKeys.MODULE_NAME, JvmAbi.DEFAULT_MODULE_NAME)
+
+        val destination = arguments.destination
+        if (destination != null) {
+            if (destination.endsWith(".jar")) {
+                // TODO: support .jar destination
+                collector.report(
+                        CompilerMessageSeverity.WARNING,
+                        ".jar destination is not yet supported, results will be written to the directory with the given name",
+                        CompilerMessageLocation.NO_LOCATION
+                )
+            }
+            configuration.put(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY, File(destination))
+        }
+
+        val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
+
+        if (environment.getSourceFiles().isEmpty()) {
+            if (arguments.version) {
+                return ExitCode.OK
+            }
+            collector.report(CompilerMessageSeverity.ERROR, "No source files", CompilerMessageLocation.NO_LOCATION)
+            return ExitCode.COMPILATION_ERROR
+        }
+
+        try {
+            MetadataSerializer(true).serialize(environment)
+        }
+        catch (e: CompilationException) {
+            collector.report(
+                    CompilerMessageSeverity.EXCEPTION,
+                    OutputMessageUtil.renderException(e),
+                    MessageUtil.psiElementToMessageLocation(e.element)
+            )
+            return ExitCode.INTERNAL_ERROR
+        }
+
+        return ExitCode.OK
+    }
+
+    companion object {
+        @JvmStatic
+        fun main(args: Array<String>) {
+            doMain(K2MetadataCompiler(), args)
+        }
+    }
+}