From 37bf7d7ef9bf7f47c9050e2cbbac59d635f0fdf7 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov <1580082+sbogolepov@users.noreply.github.com> Date: Mon, 18 May 2020 15:03:31 +0700 Subject: [PATCH] [Interop][Tests] Add infrastructure to test klib metadata (#4134) We need a structural testing tool for interop metadata itself because it is not always possible to detect metadata errors via usual compiler tests. --- backend.native/tests/build.gradle | 5 + .../tests/interop/basics/typedefs.def | 2 + build-tools/build.gradle.kts | 8 + .../kotlin/MetadataComparisonTest.kt | 87 ++++++++ .../kotlin/klib/metadata/KmComparator.kt | 186 ++++++++++++++++ .../kotlin/klib/metadata/KmComparatorUtils.kt | 60 ++++++ .../klib/metadata/SortedMergeStrategy.kt | 54 +++++ .../klib/metadata/TrivialLibraryProvider.kt | 25 +++ .../kotlin/klib/metadata/comparison.kt | 202 ++++++++++++++++++ build.gradle | 2 + 10 files changed, 631 insertions(+) create mode 100644 backend.native/tests/interop/basics/typedefs.def create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/MetadataComparisonTest.kt create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparator.kt create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparatorUtils.kt create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/SortedMergeStrategy.kt create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/TrivialLibraryProvider.kt create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/comparison.kt diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index d003ee6d82b..35c38e682f1 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -4359,6 +4359,11 @@ if (UtilsKt.getTestTargetSupportsCodeCoverage(project)) { } } +task metadata_compare_typedefs(type: MetadataComparisonTest) { + enabled = (project.testTarget != 'wasm32') + defFile = "interop/basics/typedefs.def" +} + /** * Creates tasks to build and execute stdlib tests. */ diff --git a/backend.native/tests/interop/basics/typedefs.def b/backend.native/tests/interop/basics/typedefs.def new file mode 100644 index 00000000000..97880cddca5 --- /dev/null +++ b/backend.native/tests/interop/basics/typedefs.def @@ -0,0 +1,2 @@ +--- +typedef int my_int; \ No newline at end of file diff --git a/build-tools/build.gradle.kts b/build-tools/build.gradle.kts index 9aad8bb0dd0..33aa932732a 100644 --- a/build-tools/build.gradle.kts +++ b/build-tools/build.gradle.kts @@ -40,6 +40,7 @@ repositories { maven(buildKotlinCompilerRepo) maven("https://cache-redirector.jetbrains.com/maven-central") maven("https://kotlin.bintray.com/kotlinx") + maven("https://dl.bintray.com/kotlin/kotlin-dev") } val kotlinCompilerJar by configurations.creating @@ -67,6 +68,8 @@ dependencies { // Located in /shared and always provided by the composite build. api("org.jetbrains.kotlin:kotlin-native-shared:$konanVersion") implementation("com.github.jengelman.gradle.plugins:shadow:5.1.0") + + implementation("org.jetbrains.kotlinx:kotlinx-metadata-klib:0.0.1-dev-5") } sourceSets["main"].withConvention(KotlinSourceSet::class) { @@ -93,6 +96,11 @@ gradlePlugin { val compileKotlin: KotlinCompile by tasks val compileGroovy: GroovyCompile by tasks +// https://youtrack.jetbrains.com/issue/KT-37435 +compileKotlin.apply { + kotlinOptions.freeCompilerArgs += "-Xno-optimized-callable-references" +} + // Add Kotlin classes to a classpath for the Groovy compiler compileGroovy.apply { classpath += project.files(compileKotlin.destinationDir) diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/MetadataComparisonTest.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MetadataComparisonTest.kt new file mode 100644 index 00000000000..baca92269d9 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/MetadataComparisonTest.kt @@ -0,0 +1,87 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction +import org.jetbrains.kotlin.klib.metadata.CInteropComparisonConfig +import org.jetbrains.kotlin.klib.metadata.MetadataCompareResult +import org.jetbrains.kotlin.klib.metadata.compareKlibMetadata +import org.jetbrains.kotlin.klib.metadata.expandFail +import org.jetbrains.kotlin.konan.target.HostManager +import java.io.File + +/** + * Produces 2 interop libraries from the given [defFile]. + * One in legacy `sourcecode` mode and another one in `metadata` mode. + * Then structurally compares metadata of the produced klibs. + * + * + * Motivation for this kind of tests: + * + * `sourcecode` mode generates Kotlin source file that is passes to a + * regular Kotlin/Native compiler. + * Thus, the produced metadata is correct (from compiler's point of view) + * because it is produced by the compiler itself. + * + * `metadata` mode generates metadata directly from parsed Clang AST. + * Since another algorithm is used, produced metadata may be incorrect. + * + * So we have to check that these two are more or less the same. + */ +open class MetadataComparisonTest : DefaultTask() { + + private enum class Mode { + METADATA, SOURCECODE + } + + /** + * Path to a cinterop *.def file. + */ + @Input + lateinit var defFile: String + + @TaskAction + fun run() { + val metadataLibrary = cinterop(project.file(defFile), Mode.METADATA) + val sourcecodeLibrary = cinterop(project.file(defFile), Mode.SOURCECODE) + compareKlibMetadata(CInteropComparisonConfig(), sourcecodeLibrary.absolutePath, metadataLibrary.absolutePath).let { result -> + if (result is MetadataCompareResult.Fail) { + val message = StringBuilder().also { + expandFail(result, it::appendln) + }.toString() + throw TestFailedException(message) + } + } + } + + private fun cinterop(defFile: File, mode: Mode): File { + val dist = project.kotlinNativeDist + val output = "${project.buildDir.absolutePath}/${defFile.nameWithoutExtension}_$mode" + val tool = if (HostManager.hostIsMingw) "cinterop.bat" else "cinterop" + val cinterop = File("${dist.canonicalPath}/bin/$tool").absolutePath + val args = listOf( + "-def", defFile.absolutePath, + "-mode", mode.name.toLowerCase(), + "-target", project.testTarget.visibleName, + "-no-default-libs", "-no-endorsed-libs", + "-o", output + ) + runProcess(localExecutor(project), cinterop, args).let { result -> + if (result.exitCode != 0) { + println(""" + cinterop failed. + exitCode: ${result.exitCode} + stdout: + ${result.stdOut} + stderr: + ${result.stdErr} + """.trimIndent()) + } + } + return File(output) + } +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparator.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparator.kt new file mode 100644 index 00000000000..a8b043bbd7b --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparator.kt @@ -0,0 +1,186 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin.klib.metadata + +import kotlinx.metadata.* +import kotlinx.metadata.klib.* + +/** + * Structural comparison of Km* metadata. + * [configuration] allows to tune comparison process. + * + */ +internal class KmComparator(private val configuration: ComparisonConfig) { + + fun compare(kmClass1: KmClass, kmClass2: KmClass): MetadataCompareResult = serialComparator( + compare(KmClass::name, ::compare) to "Different names: ${kmClass1.name}, ${kmClass2.name}", + ::compareClassFlags to "Different flags for ${kmClass1.name}", + compare(KmClass::constructors, compareLists(::compare)) to "Constructors mismatch for ${kmClass1.name}", + compare(KmClass::properties, compareLists(::compare, KmProperty::mangle)) to "Properties mismatch for ${kmClass1.name}", + compare(KmClass::functions, compareLists(::compare, KmFunction::mangle)) to "Functions mismatch for ${kmClass1.name}", + )(kmClass1, kmClass2) + + fun compare(typealias1: KmTypeAlias, typealias2: KmTypeAlias): MetadataCompareResult = serialComparator( + compare(KmTypeAlias::name, ::compare) to "Different names", + compare(KmTypeAlias::underlyingType, ::compareTypes) to "Underlying types mismatch", + compare(KmTypeAlias::expandedType, ::compareTypes) to "Expanded types mismatch", + compare(KmTypeAlias::typeParameters, compareLists(::compare)) to "Type parameters mismatch" + )(typealias1, typealias2) + + fun compare(function1: KmFunction, function2: KmFunction): MetadataCompareResult = serialComparator( + compare(KmFunction::name, ::compare) to "Different names", + compare(KmFunction::returnType, ::compareTypes) to "Return type mismatch", + compare(KmFunction::valueParameters, compareLists(::compare)) to "Value parameters mismatch" + )(function1, function2) + + fun compare(property1: KmProperty, property2: KmProperty): MetadataCompareResult = serialComparator( + compare(KmProperty::name, ::compare) to "Different names", + compare(KmProperty::returnType, ::compareTypes) to "Return type mismatch", + ::comparePropertyFlags to "Flags mismatch", + (compare(KmProperty::getterFlags, ::comparePropertyAccessorFlags) to "Getter flags mismatch") + .takeIf { Flag.Property.HAS_GETTER(property1.flags) && Flag.Property.HAS_GETTER(property2.flags) }, + (compare(KmProperty::setterFlags, ::comparePropertyAccessorFlags) to "Setter flags mismatch") + .takeIf { Flag.Property.HAS_SETTER(property1.flags) && Flag.Property.HAS_SETTER(property2.flags) } + )(property1, property2) + + private fun compare(entry1: KlibEnumEntry, entry2: KlibEnumEntry): MetadataCompareResult = serialComparator( + compare(KlibEnumEntry::annotations, compareLists(::compare)) to "Different annotations", + compare(KlibEnumEntry::name, ::compare) to "Different names", + compare(KlibEnumEntry::ordinal, compareNullable(::compare)) to "Different ordinals" + )(entry1, entry2) + + private fun checkFlag(flag: Flag, flagName: String? = null): (Flags, Flags) -> MetadataCompareResult = { f1, f2 -> + when { + flag(f1) != flag(f2) -> Fail("Flags mismatch: ${flag(f1)}, ${flag(f2)} for $flagName") + else -> Ok + } + } + + private fun compare(value1: Int, value2: Int): MetadataCompareResult = when { + value1 == value2 -> Ok + else -> Fail("$value1 != $value2") + } + + private fun compare(string1: String, string2: String): MetadataCompareResult = when { + string1 == string2 -> Ok + else -> Fail("$string1 != $string2") + } + + private fun comparePropertyFlags(property1: KmProperty, property2: KmProperty): MetadataCompareResult = + serialComparator( + checkFlag(Flag.Property.IS_CONST, "IS_CONST"), + checkFlag(Flag.Property.HAS_SETTER, "HAS_SETTER"), + checkFlag(Flag.Property.HAS_GETTER, "HAS_GETTER"), + checkFlag(Flag.Property.IS_VAR, "IS_VAR"), + checkFlag(Flag.Property.HAS_CONSTANT, "HAS_CONSTANT"), + checkFlag(Flag.Property.IS_DECLARATION, "IS_DECLARATION"), + checkFlag(Flag.Property.IS_EXTERNAL, "IS_EXTERNAL") + )(property1.flags, property2.flags) + + private fun comparePropertyAccessorFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator( + checkFlag(Flag.PropertyAccessor.IS_NOT_DEFAULT, "IS_NOT_DEFAULT"), + checkFlag(Flag.PropertyAccessor.IS_INLINE, "IS_INLINE"), + checkFlag(Flag.PropertyAccessor.IS_EXTERNAL, "IS_EXTERNAL"), + ::compareVisibilityFlags, + ::compareModalityFlags + )(flags1, flags2) + + + private fun compareClassFlags(class1: KmClass, class2: KmClass): MetadataCompareResult = serialComparator( + checkFlag(Flag.Class.IS_CLASS, "IS_CLASS"), + checkFlag(Flag.Class.IS_COMPANION_OBJECT, "IS_COMPANION_OBJECT"), + checkFlag(Flag.Class.IS_ENUM_CLASS, "IS_ENUM_CLASS"), + checkFlag(Flag.Class.IS_ENUM_ENTRY, "IS_ENUM_ENTRY"), + checkFlag(Flag.Class.IS_OBJECT, "IS_OBJECT"), + checkFlag(Flag.IS_FINAL, "IS_FINAL"), + checkFlag(Flag.IS_OPEN, "IS_OPEN"), + checkFlag(Flag.HAS_ANNOTATIONS, "HAS_ANNOTATIONS"), + ::compareVisibilityFlags, + ::compareModalityFlags + )(class1.flags, class2.flags) + + private fun compareVisibilityFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator( + checkFlag(Flag.IS_PUBLIC, "IS_PUBLIC"), + checkFlag(Flag.IS_PRIVATE_TO_THIS, "IS_PRIVATE_TO_THIS"), + checkFlag(Flag.IS_PRIVATE, "IS_PRIVATE"), + checkFlag(Flag.IS_PROTECTED, "IS_PROTECTED"), + checkFlag(Flag.IS_INTERNAL, "IS_INTERNAL") + )(flags1, flags2) + + private fun compareModalityFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator( + checkFlag(Flag.IS_FINAL, "IS_FINAL"), + checkFlag(Flag.IS_ABSTRACT, "IS_ABSTRACT"), + checkFlag(Flag.IS_OPEN, "IS_OPEN"), + checkFlag(Flag.IS_SEALED, "IS_SEALED") + )(flags1, flags2) + + private fun compare(annotation1: KmAnnotation, annotation2: KmAnnotation): MetadataCompareResult = when { + annotation1.className != annotation2.className -> Fail("${annotation1.className} != ${annotation2.className}") + // TODO: compare values + else -> Ok + } + + private fun compare(p1: KmValueParameter, p2: KmValueParameter): MetadataCompareResult = serialComparator( + compare(KmValueParameter::name, ::compare) to "Different names", + compare(KmValueParameter::type, compareNullable(::compareTypes)) to "Type mismatch", + compare(KmValueParameter::annotations, compareLists(::compare)) to "Annotations mismatch" + )(p1, p2) + + private fun compareTypeFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator( + checkFlag(Flag.Type.IS_NULLABLE) to "Nullable flag mismatch", + checkFlag(Flag.Type.IS_SUSPEND) to "Suspend flag mismatch" + )(flags1, flags2) + + private fun compareConstructorFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator( + checkFlag(Flag.Constructor.IS_PRIMARY) to "IS_PRIMARY mismatch" + )(flags1, flags2) + + private fun compare(constructor1: KmConstructor, constructor2: KmConstructor): MetadataCompareResult = serialComparator( + ::compareVisibilityFlags, + ::compareConstructorFlags + )(constructor1.flags, constructor2.flags) + + private fun compare(typeArgument1: KmTypeProjection, typeArgument2: KmTypeProjection): MetadataCompareResult = serialComparator( + compare(KmTypeProjection::type, compareNullable(::compareTypes)) + )(typeArgument1, typeArgument2) + + private fun compare(typeParameter1: KmTypeParameter, typeParameter2: KmTypeParameter): MetadataCompareResult = + when { + typeParameter1.variance != typeParameter2.variance -> Fail("Different variance") + typeParameter1.name != typeParameter2.name -> Fail("${typeParameter1.name}, ${typeParameter2.name}") + else -> Ok + } + + private fun compareTypes(type1: KmType, type2: KmType): MetadataCompareResult = serialComparator( + compare(KmType::classifier, ::compare) to "Classifiers mismatch", + compare(KmType::arguments, compareLists(::compare)) to "Type arguments mismatch", + compare(KmType::flags, ::compareTypeFlags) to "Type flags mismatch for", + compare(KmType::abbreviatedType, compareNullable(::compareTypes)) to "Abbreviated types mismatch" + )(type1, type2) + + private fun compare(class1: KmClassifier, class2: KmClassifier): MetadataCompareResult = when { + class1 is KmClassifier.TypeAlias && class2 is KmClassifier.TypeAlias -> { + if (class1.name == class2.name) Ok else Fail("Different type aliases: ${class1.name}, ${class2.name}") + } + class1 is KmClassifier.Class && class2 is KmClassifier.Class -> { + when (class1.name) { + class2.name -> Ok + else -> Fail("Different classes: ${class1.name}, ${class2.name}") + } + } + class1 is KmClassifier.TypeParameter && class2 is KmClassifier.TypeParameter -> { + // TODO: How to correctly compare type ids? + Ok + } + else -> Fail("class1 is $class1 and class2 is $class2") + } + + private fun compare( + property: T.() -> R, + comparator: (R, R) -> MetadataCompareResult + ): (T, T) -> MetadataCompareResult = { o1, o2 -> + if (configuration.shouldCheck(property)) comparator(o1.property(), o2.property()) else Ok + } +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparatorUtils.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparatorUtils.kt new file mode 100644 index 00000000000..1e744812d54 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/KmComparatorUtils.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin.klib.metadata + +import kotlinx.metadata.* + +private fun render(element: Any?): String = when (element) { + is KmTypeAlias -> "typealias ${element.name}" + is KmFunction -> "function ${element.name}" + is KmProperty -> "property ${element.name}" + is KmClass -> "class ${element.name}" + is KmType -> "`type ${element.classifier}`" + else -> element.toString() +} + +internal fun serialComparator( + vararg comparators: Pair<(T, T) -> MetadataCompareResult, String>? +): (T, T) -> MetadataCompareResult = { o1, o2 -> + comparators.filterNotNull().map { (comparator, message) -> + comparator(o1, o2).let { result -> + if (result is Fail) Fail(message, result) else result + } + }.wrap() +} + +internal fun serialComparator( + vararg comparators: (T, T) -> MetadataCompareResult +): (T, T) -> MetadataCompareResult = { o1, o2 -> + comparators + .map { comparator -> comparator(o1, o2) } + .wrap() +} + +internal fun Collection.wrap(): MetadataCompareResult = + filterIsInstance().let { fails -> + when (fails.size) { + 0 -> Ok + else -> Fail(fails) + } + } + +internal fun compareNullable( + comparator: (T, T) -> MetadataCompareResult +): (T?, T?) -> MetadataCompareResult = { a, b -> + when { + a != null && b != null -> comparator(a, b) + a == null && b == null -> Ok + else -> Fail("${render(a)} ${render(b)}") + } +} + +internal fun compareLists(elementComparator: (T, T) -> MetadataCompareResult, sortBy: T.() -> String? = { null }) = + { list1: List, list2: List -> compareLists(list1.sortedBy(sortBy), list2.sortedBy(sortBy), elementComparator) } + +private fun compareLists(l1: List, l2: List, comparator: (T, T) -> MetadataCompareResult) = when { + l1.size != l2.size -> Fail("${l1.size} != ${l2.size}") + else -> l1.zip(l2).map { comparator(it.first, it.second) }.wrap() +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/SortedMergeStrategy.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/SortedMergeStrategy.kt new file mode 100644 index 00000000000..93d040e77b8 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/SortedMergeStrategy.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin.klib.metadata + +import kotlinx.metadata.* +import kotlinx.metadata.klib.KlibModuleFragmentReadStrategy +import kotlinx.metadata.klib.fqName + +/** + * Output klib declarations in predictable sorted order. + */ +internal class SortedMergeStrategy : KlibModuleFragmentReadStrategy { + + override fun processModuleParts(parts: List): List = + parts.fold(KmModuleFragment(), ::joinFragments).let(::listOf) +} + +/** + * We need a stable order for overloaded functions. + */ +internal fun KmFunction.mangle(): String { + val typeParameters = typeParameters.joinToString(prefix = "<", postfix = ">", transform = KmTypeParameter::name) + val valueParameters = valueParameters.joinToString(prefix = "(", postfix = ")", transform = KmValueParameter::name) + val receiver = receiverParameterType?.classifier + return "$receiver.${name}.$typeParameters.$valueParameters" +} + +internal fun KmProperty.mangle(): String { + val receiver = receiverParameterType?.classifier + return "$receiver.$name" +} + +private fun joinAndSortPackages(pkg1: KmPackage, pkg2: KmPackage) = KmPackage().apply { + functions += (pkg1.functions + pkg2.functions).sortedBy(KmFunction::mangle) + properties += (pkg1.properties + pkg2.properties).sortedBy(KmProperty::name) + typeAliases += (pkg1.typeAliases + pkg2.typeAliases).sortedBy(KmTypeAlias::name) +} + +/** + * Merges two fragments of a single module into one. + */ +internal fun joinFragments(fragment1: KmModuleFragment, fragment2: KmModuleFragment) = KmModuleFragment().apply { + assert(fragment1.fqName == fragment2.fqName) + pkg = when { + fragment1.pkg != null && fragment2.pkg != null -> joinAndSortPackages(fragment1.pkg!!, fragment2.pkg!!) + fragment1.pkg != null -> joinAndSortPackages(fragment1.pkg!!, KmPackage()) + fragment2.pkg != null -> joinAndSortPackages(KmPackage(), fragment2.pkg!!) + else -> null + } + fqName = fragment1.fqName + classes += fragment1.classes.sortedBy(KmClass::name) + fragment2.classes.sortedBy(KmClass::name) +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/TrivialLibraryProvider.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/TrivialLibraryProvider.kt new file mode 100644 index 00000000000..2f81c86f1b9 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/TrivialLibraryProvider.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin.klib.metadata + +import kotlinx.metadata.klib.KlibModuleMetadata +import org.jetbrains.kotlin.library.MetadataLibrary + +/** + * Provides access to metadata using default compiler's routine. + */ +internal class TrivialLibraryProvider( + private val library: MetadataLibrary +) : KlibModuleMetadata.MetadataLibraryProvider { + + override val moduleHeaderData: ByteArray + get() = library.moduleHeaderData + + override fun packageMetadata(fqName: String, partName: String): ByteArray = + library.packageMetadata(fqName, partName) + + override fun packageMetadataParts(fqName: String): Set = + library.packageMetadataParts(fqName) +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/comparison.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/comparison.kt new file mode 100644 index 00000000000..f1fe3abbbdf --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/klib/metadata/comparison.kt @@ -0,0 +1,202 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +package org.jetbrains.kotlin.klib.metadata + +import kotlinx.metadata.* +import kotlinx.metadata.klib.KlibModuleMetadata +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.library.CompilerSingleFileKlibResolveAllowingIrProvidersStrategy +import org.jetbrains.kotlin.library.resolveSingleFileKlib + +private inline fun compareElements( + comparisonConfig: ComparisonConfig, + elements: Map>, + crossinline comparator: (T, T) -> MetadataCompareResult +): MetadataCompareResult = elements + .entries + .asSequence() + .filter { comparisonConfig.shouldCheckDeclaration(it.key) } + .map { comparator(it.value.first, it.value.second).messageIfFail("${it.key} mismatch") } + .toList() + .wrap() + +private class JoinedFragments( + val classes: JoinResult, + val functions: JoinResult, + val properties: JoinResult, + val typeAliases: JoinResult, +) + +private fun processMissing(comparisonConfig: ComparisonConfig, joinResult: JoinResult<*>): MetadataCompareResult { + val missingInFirst = joinResult.missingInFirst + .filter(comparisonConfig::shouldCheckDeclaration) + .map { Fail("$it missing in first fragment") } + val missingInSecond = joinResult.missingInSecond + .filter(comparisonConfig::shouldCheckDeclaration) + .map { Fail("$it missing in second fragment") } + return (missingInFirst + missingInSecond).let { + if (it.isEmpty()) Ok else Fail(it) + } +} + +private fun MetadataCompareResult.messageIfFail(message: String): MetadataCompareResult = + if (this is Fail) Fail(message, this) else this + +private fun processMissing( + comparisonConfig: ComparisonConfig, + joinedFragments: JoinedFragments +): MetadataCompareResult = listOf( + processMissing(comparisonConfig, joinedFragments.classes) + .messageIfFail("Missing classes"), + processMissing(comparisonConfig, joinedFragments.functions) + .messageIfFail("Missing functions"), + processMissing(comparisonConfig, joinedFragments.typeAliases) + .messageIfFail("Missing type aliases"), + processMissing(comparisonConfig, joinedFragments.properties) + .messageIfFail("Missing properties"), +).wrap() + +private data class JoinResult( + val joined: Map>, + val missingInFirst: List, + val missingInSecond: List +) + +private fun buildJoined(e1: List, e2: List, key: T.() -> String): JoinResult { + val m1 = e1.associateBy { it.key() } + val m2 = e2.associateBy { it.key() } + val joinedKeys = e1.map(key).filter { it in m2 }.toSet() + val joined = m1 + .filterKeys(joinedKeys::contains) + .mapValues { (key, value) -> value to m2.getValue(key) } + return JoinResult( + joined, + (m1 - joinedKeys).keys.toList(), + (m2 - joinedKeys).keys.toList() + ) +} + +/** + * Wrapper around direct access to [fragment] that allows + * to uniformly process all its components. + */ +private fun processFragment( + fragment: KmModuleFragment, + action: (List, List, List, List) -> T +): T { + val classes = fragment.classes + val pkg = fragment.pkg + return when { + pkg != null -> action(classes, pkg.functions, pkg.properties, pkg.typeAliases) + else -> action(classes, emptyList(), emptyList(), emptyList()) + } +} + +private fun compareMetadata( + comparisonConfig: ComparisonConfig, + metadataModuleA: KlibModuleMetadata, + metadataModuleB: KlibModuleMetadata +): MetadataCompareResult { + val fragmentA = metadataModuleA.fragments.fold(KmModuleFragment(), ::joinFragments) + val fragmentB = metadataModuleB.fragments.fold(KmModuleFragment(), ::joinFragments) + + val joinedFragments = processFragment(fragmentA) { classesA, functionsA, propertiesA, typeAliasesA -> + processFragment(fragmentB) { classesB, functionsB, propertiesB, typeAliasesB -> + JoinedFragments( + buildJoined(classesA, classesB, KmClass::name), + buildJoined(functionsA, functionsB, KmFunction::name), + buildJoined(propertiesA, propertiesB, KmProperty::name), + buildJoined(typeAliasesA, typeAliasesB, KmTypeAlias::name) + ) + } + } + + val comparator = KmComparator(comparisonConfig) + return joinedFragments.run { + listOf( + compareElements(comparisonConfig, classes.joined, comparator::compare), + compareElements(comparisonConfig, functions.joined, comparator::compare), + compareElements(comparisonConfig, properties.joined, comparator::compare), + compareElements(comparisonConfig, typeAliases.joined, comparator::compare), + processMissing(comparisonConfig, this) + ) + }.wrap() + +} + +sealed class MetadataCompareResult { + class Fail( + val children: Collection, val message: String? = null + ) : MetadataCompareResult() { + constructor(message: String, child: Fail? = null) + : this(listOfNotNull(child), message) + } + + object Ok : MetadataCompareResult() +} + +// A neat way to have short names internally without polluting client's namespace. +internal typealias Fail = MetadataCompareResult.Fail +internal typealias Ok = MetadataCompareResult.Ok + +fun expandFail(fail: Fail, output: (String) -> Unit, padding: String = "") { + fail.message?.let { output("$padding$it") } + fail.children.forEach { + expandFail(it, output, "$padding ") + } +} + +/** + * Configure what should be tested and what shouldn't. + * + * TODO: Add a way to conditionally disable property comparison. + * E.g. "If class name is Companion do not compare flags". + */ +interface ComparisonConfig { + /** + * Should the declaration be compared at all. + */ + fun shouldCheckDeclaration(element: String): Boolean + + /** + * Should we check property of declaration. + */ + fun shouldCheck(property: T.() -> R): Boolean +} + +/** + * Configuration for comparing `metadata` and `sourcecode` cinterop modes. + */ +class CInteropComparisonConfig : ComparisonConfig { + override fun shouldCheck(property: T.() -> R): Boolean = when (property) { + // Kotlin compiler may incorrectly omit abbreviatedType in some cases. + KmType::abbreviatedType -> false + else -> true + } + + override fun shouldCheckDeclaration(element: String): Boolean = when { + // kniBridge is generated only in sourcecode mode. + element.startsWith("kniBridge") -> false + else -> true + } +} + +/** + * Structurally compares metadata of given libraries. + */ +fun compareKlibMetadata( + comparisonConfig: ComparisonConfig, + pathToFirstLibrary: String, + pathToSecondLibrary: String +): MetadataCompareResult { + val resolveStrategy = CompilerSingleFileKlibResolveAllowingIrProvidersStrategy( + knownIrProviders = listOf("kotlin.native.cinterop") + ) + val klib1 = resolveSingleFileKlib(File(pathToFirstLibrary), strategy = resolveStrategy) + val klib2 = resolveSingleFileKlib(File(pathToSecondLibrary), strategy = resolveStrategy) + val metadata1 = KlibModuleMetadata.read(TrivialLibraryProvider(klib1), SortedMergeStrategy()) + val metadata2 = KlibModuleMetadata.read(TrivialLibraryProvider(klib2), SortedMergeStrategy()) + return compareMetadata(comparisonConfig, metadata1, metadata2) +} \ No newline at end of file diff --git a/build.gradle b/build.gradle index 86eb3d86b0d..fc352b106df 100644 --- a/build.gradle +++ b/build.gradle @@ -35,6 +35,7 @@ buildscript { maven { url kotlinCompilerRepo } maven { url "https://kotlin.bintray.com/kotlinx" } maven { url "https://cache-redirector.jetbrains.com/maven-central" } + maven { url "https://dl.bintray.com/kotlin/kotlin-dev" } jcenter() } @@ -103,6 +104,7 @@ allprojects { maven { url 'https://cache-redirector.jetbrains.com/jcenter' } + maven { url "https://dl.bintray.com/kotlin/kotlin-dev" } } } if (path != ":dependencies") {