azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework BuiltInsSerializer

Browse Source 
BuiltInsSerializer will be distributed with Kotlin compiler from now on. This
will allow to serialize binary data of built-ins on 'ant dist', as opposed to
storing all *.kotlin_class files in the repository: ant dist will just invoke
this serializer from bootstrap-compiler.jar
This commit is contained in:
Alexander Udalov
2014-01-20 22:36:34 +04:00
parent d4c98ec18c
commit db9d0e381b
12 changed files with 288 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/builtins-serializer/builtins-serializer.iml
+18
View File
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module" module-name="cli" />
<orderEntry type="module" module-name="frontend" />
<orderEntry type="module" module-name="frontend.java" />
<orderEntry type="module" module-name="serialization" />
<orderEntry type="module" module-name="util" />
</component>
</module>
compiler/builtins-serializer/src/BuiltInsSerializer.kt
+148
View File
@@ -0,0 +1,148 @@
/*
* Copyright 2010-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.utils.builtinsSerializer
import java.io.File
import java.io.PrintStream
import com.intellij.openapi.util.Disposer
import com.intellij.openapi.util.io.FileUtil
import org.jetbrains.jet.config.CompilerConfiguration
import org.jetbrains.jet.cli.jvm.compiler.JetCoreEnvironment
import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns
import org.jetbrains.jet.descriptors.serialization.DescriptorSerializer
import org.jetbrains.jet.descriptors.serialization.SerializerExtension
import org.jetbrains.jet.lang.descriptors.ClassDescriptor
import java.util.ArrayList
import org.jetbrains.jet.lang.resolve.name.Name
import org.jetbrains.jet.descriptors.serialization.ProtoBuf
import java.io.ByteArrayOutputStream
import java.io.DataOutputStream
import org.jetbrains.jet.lang.types.lang.BuiltInsSerializationUtil
import org.jetbrains.jet.descriptors.serialization.NameSerializationUtil
import org.jetbrains.jet.lang.resolve.DescriptorUtils
import com.intellij.openapi.Disposable
import org.jetbrains.jet.cli.common.CLIConfigurationKeys
import org.jetbrains.jet.config.CommonConfigurationKeys
import org.jetbrains.jet.cli.common.messages.MessageCollector
import org.jetbrains.jet.lang.resolve.java.AnalyzerFacadeForJVM
import org.jetbrains.jet.lang.resolve.BindingTraceContext
import org.jetbrains.jet.di.InjectorForJavaDescriptorResolverUtil
public class BuiltInsSerializer(val out: PrintStream?) {
private var totalSize = 0
private var totalFiles = 0
public fun serialize(destDir: File, srcDirs: Collection<File>) {
val rootDisposable = Disposer.newDisposable()
try {
serialize(rootDisposable, destDir, srcDirs)
}
finally {
Disposer.dispose(rootDisposable)
}
}
fun serialize(disposable: Disposable, destDir: File, srcDirs: Collection<File>) {
val configuration = CompilerConfiguration()
configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, MessageCollector.NONE)
val sourceRoots = srcDirs map { it.path }
configuration.put(CommonConfigurationKeys.SOURCE_ROOTS_KEY, sourceRoots)
val environment = JetCoreEnvironment.createForTests(disposable, configuration)
val files = environment.getSourceFiles() ?: error("No source files in $sourceRoots")
val project = environment.getProject()
val trace = BindingTraceContext()
val session = AnalyzerFacadeForJVM.createLazyResolveSession(project, files, trace,
InjectorForJavaDescriptorResolverUtil.create(project, trace), false)
val module = session.getModuleDescriptor() ?: error("No module resolved for $sourceRoots")
val fqName = KotlinBuiltIns.BUILT_INS_PACKAGE_FQ_NAME
val packageView = module.getPackage(fqName) ?: error("No package resolved in $module")
// TODO: perform some kind of validation? At the moment not possible because DescriptorValidator is in compiler-tests
// DescriptorValidator.validate(packageView)
if (!FileUtil.delete(destDir)) {
System.err.println("Could not delete: " + destDir)
}
if (!destDir.mkdirs()) {
System.err.println("Could not make directories: " + destDir)
}
val serializer = DescriptorSerializer(object : SerializerExtension() {
private val set = setOf("Any", "Nothing")
override fun hasSupertypes(descriptor: ClassDescriptor): Boolean {
return descriptor.getName().asString() !in set
}
})
val classNames = ArrayList<Name>()
val allDescriptors = DescriptorSerializer.sort(packageView.getMemberScope().getAllDescriptors())
ClassSerializationUtil.serializeClasses(allDescriptors, serializer, object : ClassSerializationUtil.Sink {
override fun writeClass(classDescriptor: ClassDescriptor, classProto: ProtoBuf.Class) {
val stream = ByteArrayOutputStream()
classProto.writeTo(stream)
write(destDir, getFileName(classDescriptor), stream)
if (DescriptorUtils.isTopLevelDeclaration(classDescriptor)) {
classNames.add(classDescriptor.getName())
}
}
})
val classNamesStream = ByteArrayOutputStream()
writeClassNames(serializer, classNames, classNamesStream)
write(destDir, BuiltInsSerializationUtil.getClassNamesFilePath(fqName), classNamesStream)
val packageStream = ByteArrayOutputStream()
val fragments = module.getPackageFragmentProvider().getPackageFragments(fqName)
val packageProto = serializer.packageProto(fragments).build() ?: error("Package fragments not serialized: $fragments")
packageProto.writeTo(packageStream)
write(destDir, BuiltInsSerializationUtil.getPackageFilePath(fqName), packageStream)
val nameStream = ByteArrayOutputStream()
NameSerializationUtil.serializeNameTable(nameStream, serializer.getNameTable())
write(destDir, BuiltInsSerializationUtil.getNameTableFilePath(fqName), nameStream)
out?.println("Total bytes written: $totalSize to $totalFiles files")
}
fun writeClassNames(serializer: DescriptorSerializer, classNames: List<Name>, stream: ByteArrayOutputStream) {
val nameTable = serializer.getNameTable()
DataOutputStream(stream) use { output ->
output.writeInt(classNames.size())
for (className in classNames) {
output.writeInt(nameTable.getSimpleNameIndex(className))
}
}
}
fun write(destDir: File, fileName: String, stream: ByteArrayOutputStream) {
totalSize += stream.size()
totalFiles++
FileUtil.writeToFile(File(destDir, fileName), stream.toByteArray())
}
fun getFileName(classDescriptor: ClassDescriptor): String {
return BuiltInsSerializationUtil.getClassMetadataPath(ClassSerializationUtil.getClassId(classDescriptor))
}
}
compiler/builtins-serializer/src/ClassSerializationUtil.kt
+59
View File
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.utils.builtinsSerializer
import org.jetbrains.jet.descriptors.serialization.ClassId
import org.jetbrains.jet.descriptors.serialization.DescriptorSerializer
import org.jetbrains.jet.descriptors.serialization.ProtoBuf
import org.jetbrains.jet.lang.descriptors.ClassDescriptor
import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor
import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
import org.jetbrains.jet.lang.resolve.name.FqNameUnsafe
public object ClassSerializationUtil {
public trait Sink {
fun writeClass(classDescriptor: ClassDescriptor, classProto: ProtoBuf.Class)
}
private fun serializeClass(classDescriptor: ClassDescriptor, serializer: DescriptorSerializer, sink: Sink) {
val classProto = serializer.classProto(classDescriptor).build() ?: error("Class not serialized: $classDescriptor")
sink.writeClass(classDescriptor, classProto)
serializeClasses(classDescriptor.getUnsubstitutedInnerClassesScope().getAllDescriptors(), serializer, sink)
val classObjectDescriptor = classDescriptor.getClassObjectDescriptor()
if (classObjectDescriptor != null) {
serializeClass(classObjectDescriptor, serializer, sink)
}
}
public fun serializeClasses(descriptors: Collection<DeclarationDescriptor>, serializer: DescriptorSerializer, sink: Sink) {
for (descriptor in descriptors) {
if (descriptor is ClassDescriptor) {
serializeClass(descriptor, serializer, sink)
}
}
}
public fun getClassId(classDescriptor: ClassDescriptor): ClassId {
val owner = classDescriptor.getContainingDeclaration()
if (owner is PackageFragmentDescriptor) {
return ClassId(owner.getFqName(), FqNameUnsafe.topLevel(classDescriptor.getName()))
}
return getClassId(owner as ClassDescriptor).createNestedClassId(classDescriptor.getName())
}
}
compiler/builtins-serializer/src/run.kt
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright 2010-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.utils.builtinsSerializer
import java.io.File
import java.util.ArrayList
// To regenerate built-ins in Kotlin project, launch main() with these arguments:
public val BUILT_INS_DEST_DIR: String = "compiler/frontend/builtins"
public val BUILT_INS_SRC_DIR: String = "idea/builtinsSrc"
fun main(args: Array<String>) {
System.setProperty("java.awt.headless", "true")
if (args.size < 2) {
println(
"""Kotlin built-ins serializer
Usage: ... <destination dir> (<built-ins src dir>)+
Analyzes Kotlin sources found in the given source directories and serializes
found top-level declarations to <destination dir> (files such as
.kotlin_class_names, .kotlin_name_table, .kotlin_package, *.kotlin_class)"""
)
return
}
val destDir = File(args[0])
val srcDirs = args.iterator().skip(1).map({ File(it) }).toList()
assert(srcDirs all { it.exists() }) { "Some of the built-ins source directories don't exist: $srcDirs" }
BuiltInsSerializer(System.out).serialize(destDir, srcDirs)
}
compiler/frontend/serialization/serialization.iml
+1 -1
View File
@@ -8,7 +8,7 @@
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="protobuf-java" level="project" />
<orderEntry type="library" exported="" name="protobuf-java" level="project" />
<orderEntry type="library" name="trove4j" level="project" />
<orderEntry type="library" name="intellij-core" level="project" />
<orderEntry type="module" module-name="util.runtime" exported="" />
compiler/tests/compiler-tests.iml
+1
View File
@@ -19,6 +19,7 @@
<orderEntry type="module" module-name="serialization.java" />
<orderEntry type="module" module-name="descriptor.loader.java" />
<orderEntry type="module" module-name="util" />
<orderEntry type="module" module-name="builtins-serializer" />
</component>
</module>
compiler/tests/org/jetbrains/jet/descriptors/serialization/AbstractDescriptorSerializationTest.java
+3 -3
View File
@@ -37,12 +37,12 @@ import org.jetbrains.jet.lang.resolve.scopes.JetScope;
import org.jetbrains.jet.lang.resolve.scopes.WritableScope;
import org.jetbrains.jet.storage.LockBasedStorageManager;
import org.jetbrains.jet.test.util.RecursiveDescriptorComparator;
import org.jetbrains.jet.utils.builtinsSerializer.ClassSerializationUtil;
import java.io.File;
import java.io.IOException;
import java.util.*;
import static org.jetbrains.jet.descriptors.serialization.ClassSerializationUtil.getClassId;
import static org.jetbrains.jet.descriptors.serialization.NameSerializationUtil.createNameResolver;
import static org.jetbrains.jet.descriptors.serialization.descriptors.AnnotationDeserializer.UNSUPPORTED;
import static org.jetbrains.jet.lang.resolve.java.DescriptorSearchRule.IGNORE_KOTLIN_SOURCES;
@@ -100,7 +100,7 @@ public abstract class AbstractDescriptorSerializationTest extends KotlinTestWith
javaDescriptorResolver, classDataMap, packageFragment.getContainingDeclaration().getPackageFragmentProvider());
for (ClassDescriptor classDescriptor : classesAndObjects) {
ClassId classId = getClassId(classDescriptor);
ClassId classId = ClassSerializationUtil.instance$.getClassId(classDescriptor);
ClassDescriptor descriptor = descriptorFinder.findClass(classId);
assert descriptor != null : "Class not loaded: " + classId;
packageFragment.getMemberScope().addClassifierDescriptor(descriptor);
@@ -153,7 +153,7 @@ public abstract class AbstractDescriptorSerializationTest extends KotlinTestWith
final Map<ClassDescriptor, byte[]> serializedClasses = new HashMap<ClassDescriptor, byte[]>();
final DescriptorSerializer serializer = new DescriptorSerializer();
ClassSerializationUtil.serializeClasses(classes, serializer, new ClassSerializationUtil.Sink() {
ClassSerializationUtil.instance$.serializeClasses(classes, serializer, new ClassSerializationUtil.Sink() {
@Override
public void writeClass(@NotNull ClassDescriptor classDescriptor, @NotNull ProtoBuf.Class classProto) {
ClassData data = new ClassData(createNameResolver(serializer.getNameTable()), classProto);
compiler/tests/org/jetbrains/jet/descriptors/serialization/ClassSerializationUtil.java
-71
View File
@@ -1,71 +0,0 @@
/*
* Copyright 2010-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.descriptors.serialization;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.lang.descriptors.ClassDescriptor;
import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor;
import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor;
import org.jetbrains.jet.lang.resolve.name.FqNameUnsafe;
import java.util.Collection;
public class ClassSerializationUtil {
private ClassSerializationUtil() {
}
public interface Sink {
void writeClass(@NotNull ClassDescriptor classDescriptor, @NotNull ProtoBuf.Class classProto);
}
private static void serializeClass(
@NotNull ClassDescriptor classDescriptor,
@NotNull DescriptorSerializer serializer,
@NotNull Sink sink
) {
ProtoBuf.Class classProto = serializer.classProto(classDescriptor).build();
sink.writeClass(classDescriptor, classProto);
serializeClasses(classDescriptor.getUnsubstitutedInnerClassesScope().getAllDescriptors(), serializer, sink);
ClassDescriptor classObjectDescriptor = classDescriptor.getClassObjectDescriptor();
if (classObjectDescriptor != null) {
serializeClass(classObjectDescriptor, serializer, sink);
}
}
public static void serializeClasses(
@NotNull Collection<? extends DeclarationDescriptor> descriptors,
@NotNull DescriptorSerializer serializer,
@NotNull Sink sink
) {
for (DeclarationDescriptor descriptor : descriptors) {
if (descriptor instanceof ClassDescriptor) {
serializeClass((ClassDescriptor) descriptor, serializer, sink);
}
}
}
@NotNull
public static ClassId getClassId(@NotNull ClassDescriptor classDescriptor) {
DeclarationDescriptor owner = classDescriptor.getContainingDeclaration();
if (owner instanceof PackageFragmentDescriptor) {
return new ClassId(((PackageFragmentDescriptor) owner).getFqName(), FqNameUnsafe.topLevel(classDescriptor.getName()));
}
return getClassId((ClassDescriptor) owner).createNestedClassId(classDescriptor.getName());
}
}
compiler/tests/org/jetbrains/jet/utils/builtinsSerializer/BuiltInsSerializerTest.kt
+59
View File
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.utils.builtinsSerializer
import com.intellij.openapi.util.io.FileUtil
import com.intellij.testFramework.UsefulTestCase
import org.jetbrains.jet.JetTestUtils
import java.io.File
import java.util.Arrays
import java.util.HashSet
import java.util.regex.Pattern
import junit.framework.Assert
public open class BuiltInsSerializerTest : UsefulTestCase() {
public fun testBuiltIns() {
val actual = JetTestUtils.tmpDir("builtins")
BuiltInsSerializer(null).serialize(actual, listOf(File(BUILT_INS_SRC_DIR)))
val expected = File(BUILT_INS_DEST_DIR)
val actualFiles = getAllFiles(actual)
val expectedFiles = getAllFiles(expected)
val actualNames = getFileNames(actualFiles)
val expectedNames = getFileNames(expectedFiles)
Assert.assertEquals("File name sets differ. Re-run BuiltInsSerializer", expectedNames, actualNames)
for (actualFile in actualFiles) {
if (actualFile.isDirectory()) continue
val relativePath = FileUtil.getRelativePath(actual, actualFile)!!
val expectedFile = File(expected, relativePath)
val expectedBytes = FileUtil.loadFileBytes(expectedFile)
val actualBytes = FileUtil.loadFileBytes(actualFile)
Assert.assertTrue("File contents differ for $expectedFile and $actualFile. Re-run BuiltInsSerializer",
Arrays.equals(expectedBytes, actualBytes))
}
println("${actualFiles.size()} files checked")
}
private fun getAllFiles(actual: File) = FileUtil.findFilesByMask(Pattern.compile(".*"), actual)
private fun getFileNames(actualFiles: List<File>) = HashSet(actualFiles map { f -> f.getName() })
}