Introduce jdk-api-validator to ensure kotlin-reflect uses jdk6 API

Merge-request: KT-MR-6930 Merged-by: Abduqodiri Qurbonzoda <abduqodiri.qurbonzoda@jetbrains.com>
2023-07-12 05:13:08 +00:00
parent 5424c54fae
commit 7346cf4777
13 changed files with 285 additions and 2 deletions
@@ -282,6 +282,7 @@
 /libraries/kotlin-dom-api-compat/ "Kotlin JS"
 /libraries/tools/kotlin-build-tools-enum-compat/ "Kotlin Build Tools"
 /libraries/tools/gradle/ "Kotlin Build Tools"
+/libraries/tools/jdk-api-validator/ "Kotlin Libraries"
 /libraries/tools/kotlin-allopen/ "Kotlin Build Tools"
 /libraries/tools/kotlin-annotations-jvm/ "Kotlin Libraries"
 /libraries/tools/kotlin-assignment/ "Kotlin Build Tools"
@@ -609,6 +609,7 @@ tasks {
            ":kotlin-test:kotlin-test-js-ir:kotlin-test-js-ir-it".takeIf { !kotlinBuildProperties.isInJpsBuildIdeaSync },
            ":kotlinx-metadata-jvm",
            ":tools:binary-compatibility-validator",
+            ":tools:jdk-api-validator",
            //":kotlin-stdlib-wasm",
        )).forEach {
            dependsOn("$it:check")
@@ -43,6 +43,7 @@ internal fun <V : Any> createCache(compute: (Class<*>) -> V): CacheByClass<V> {
 * ClassValue -> KPackageImpl.getClass() -> UrlClassloader -> all loaded classes by this CL ->
 *  -> kotlin.reflect.jvm.internal.ClassValueCache -> ClassValue
 */
+@kotlin.reflect.jvm.internal.SuppressJdk6SignatureCheck
 private class ComputableClassValue<V>(@JvmField val compute: (Class<*>) -> V) : ClassValue<SoftReference<V>>() {
    override fun computeValue(type: Class<*>): SoftReference<V> {
        return SoftReference(compute(type))
@@ -51,6 +52,7 @@ private class ComputableClassValue<V>(@JvmField val compute: (Class<*>) -> V) :
    fun createNewCopy() = ComputableClassValue(compute)
 }

+@kotlin.reflect.jvm.internal.SuppressJdk6SignatureCheck
 private class ClassValueCache<V>(compute: (Class<*>) -> V) : CacheByClass<V>() {

    @Volatile
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.reflect.jvm.internal
+
+/**
+ * Suppresses verification errors of the jdk-api-validator tool for certain scope.
+ * Such scopes include references to Java 8 API that are not available in Android API,
+ * but can be desugared by R8 or their execution is prevented on Android platform.
+ */
+@Retention(AnnotationRetention.BINARY)
+@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION)
+internal annotation class SuppressJdk6SignatureCheck
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin
+
+/**
+ * Suppresses verification errors of the jdk-api-validator tool for certain scope.
+ * Such scopes include references to Java 8 API that are not available in Android API,
+ * but can be desugared by R8 or their execution is prevented on Android platform.
+ */
+@Retention(AnnotationRetention.BINARY)
+@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION)
+internal annotation class SuppressJdk6SignatureCheck
@@ -647,6 +647,7 @@ public class LockBasedStorageManager implements StorageManager {
            );
        }

+        @org.jetbrains.kotlin.SuppressJdk6SignatureCheck
        private AssertionError unableToRemoveKey(K input, Throwable throwable) {
            return sanitizeStackTrace(
                    new AssertionError("Unable to remove "
@@ -22,6 +22,7 @@ fun updateCompilerXml() {
        "libraries/tools/binary-compatibility-validator",
        "libraries/tools/dukat",
        "libraries/tools/gradle",
+        "libraries/tools/jdk-api-validator",
        "libraries/tools/kotlin-allopen",
        "libraries/tools/kotlin-annotation-processing",
        "libraries/tools/kotlin-assignment",
@@ -6803,6 +6803,12 @@
            <sha256 value="cbd0fa43ebc62cb548e17f78b92b745220d9b1904f255c5916ef5a592b11e090" origin="Generated by Gradle"/>
         </artifact>
      </component>
+      <component group="org.codehaus.mojo" name="animal-sniffer" version="1.21">
+         <artifact name="animal-sniffer-1.21.jar">
+            <md5 value="48fb85ff31e5f0429b6fe3244861b88a" origin="Generated by Gradle"/>
+            <sha256 value="3b33fb52e494a78d803c3836fd4cec45fd22ca1cad62829db83912c6bfa503b9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
      <component group="org.codehaus.mojo" name="animal-sniffer-annotations" version="1.14">
         <artifact name="animal-sniffer-annotations-1.14.jar">
            <md5 value="9d42e46845c874f1710a9f6a741f6c14" origin="Generated by Gradle"/>
@@ -6833,6 +6839,12 @@
            <sha256 value="adf522f4839c35f5329ea97c407aebebfa8807b644852dc4d5cd7c97b7a6d2e0" origin="Generated by Gradle"/>
         </artifact>
      </component>
+      <component group="org.codehaus.mojo.signature" name="java16" version="1.1">
+         <artifact name="java16-1.1.signature">
+            <md5 value="f54abe9fb83f358412ad738cc46fc158" origin="Generated by Gradle"/>
+            <sha256 value="53799223a2c98dba2d0add810bed76315460df285c69e4f397ae6098f87dd619" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
      <component group="org.codehaus.mojo" name="animal-sniffer-annotations" version="1.19">
         <artifact name="animal-sniffer-annotations-1.19.jar">
            <md5 value="1d87340449dc95ca4647f933741df928" origin="Generated by Gradle"/>
@@ -0,0 +1,46 @@
+# Verifies that a library compiled with a newer JDK is compatible with older JDK API
+
+Currently, the project only checks `kotlin-reflect` for JDK 1.6 compatibility.
+The check is necessary to make sure applications using `kotlin-reflect` can run on older Android devices.
+
+## How to run
+
+Run from the root directory of the kotlin project:
+
+`./gradlew :tools:jdk-api-validator:test`
+
+## How to interpret the result
+
+Successful completion of the `test` task means the checked libraries are compatible with JDK 1.6 API.
+In case of failure, the exact location and name of the violating API references are logged as error.
+
+An example of validation error:
+
+`[ERROR] /kotlin/libraries/reflect/build/libs/kotlin-reflect-1.9.255-SNAPSHOT.jar:kotlin/reflect/jvm/internal/ComputableClassValue.class:47: Undefined reference: void ClassValue.<init>()`
+
+## How to fix a failure
+
+If the violating reference can be desugared by R8 or its execution is prevented on Android platform,
+the error can be suppressed. See `suppressAnnotations` and `undefinedReferencesToIgnore` in `JdkApiUsageTest.kt`.
+Otherwise, the API should be avoided.
+
+To check if an API can be desugared by R8:
+1. Identify the earliest R8 version that supports current Kotlin version [here](https://developer.android.com/build/kotlin-support).
+2. Download the R8 version jar artifact from Google's maven repository [here](https://maven.google.com/web/index.html#com.android.tools:r8).
+3. Run it using the `BackportedMethodList` entry point, e.g., `java -cp r8-8.0.40.jar com.android.tools.r8.BackportedMethodList`.
+4. Check if the violating API reference is in the printed list of methods.
+
+Also, you can get the list of backported methods the downloaded version supports for a given Android API level:
+```shell
+$ java -cp r8-8.0.40.jar com.android.tools.r8.BackportedMethodList --help
+Usage: BackportedMethodList [options]
+ Options are:
+  --output <file>         # Output result in <file>.
+  --min-api <number>      # Minimum Android API level for the application
+  --desugared-lib <file>  # Desugared library configuration (JSON from the
+                          # configuration)
+  --lib <file>            # The compilation SDK library (android.jar)
+  --version               # Print the version of BackportedMethodList.
+  --help                  # Print this message.
+```
+
@@ -0,0 +1,41 @@
+plugins {
+    id("kotlin")
+}
+
+repositories {
+    mavenCentral()
+}
+
+val testArtifacts by configurations.creating
+val signature by configurations.creating
+
+sourceSets {
+    "main" { none() }
+    "test" { kotlin.srcDir("src/test") }
+}
+
+dependencies {
+    implementation("org.codehaus.mojo:animal-sniffer:1.21")
+    implementation(kotlinStdlib())
+
+    testImplementation(project(":kotlin-test:kotlin-test-junit"))
+
+    testArtifacts(project(":kotlin-reflect"))
+
+    signature("org.codehaus.mojo.signature:java16:1.1@signature")
+}
+
+val signaturesDirectory = buildDir.resolve("signatures")
+
+val collectSignatures by tasks.registering(Sync::class) {
+    from(signature)
+    into(signaturesDirectory)
+}
+
+tasks.getByName<Test>("test") {
+    dependsOn(collectSignatures)
+    dependsOn(testArtifacts)
+
+    systemProperty("kotlinVersion", project.version)
+    systemProperty("signaturesDirectory", signaturesDirectory)
+}
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.tools.tests
+
+import org.codehaus.mojo.animal_sniffer.*
+import org.codehaus.mojo.animal_sniffer.logging.Logger
+import org.codehaus.mojo.animal_sniffer.logging.PrintWriterLogger
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.test.*
+
+class JdkApiUsageTest {
+
+    @Test
+    fun kotlinReflect() {
+        testApiUsage(
+            // Do not use the final jar artifact. It's shrunk by proguard.
+            jarArtifact("../../reflect/build/libs", "kotlin-reflect", "shadow"),
+            dependencies = listOf(jarArtifact("../../stdlib/jvm/build/libs", "kotlin-stdlib"))
+        )
+    }
+
+    private fun testApiUsage(artifact: Path, dependencies: List<Path>) {
+        val logger = TestLogger()
+        val additionalArtifacts = buildList {
+            add(artifact)
+            addAll(dependencies)
+        }
+
+        val signatures = buildSignatures(additionalArtifacts, logger)
+        if (logger.hasError) {
+            fail("Building signatures has failed. See console logs for details.")
+        }
+
+        checkSignatures(artifact, signatures, logger)
+        if (logger.hasError) {
+            fail("Checking signatures has failed. See console logs for details. "
+                         + "See libraries/tools/jdk-api-validator/ReadMe.md to find out how to fix the failures.")
+        }
+    }
+
+    private fun checkSignatures(artifact: Path, signatures: Path, logger: Logger) {
+        val checker = SignatureChecker(signatures.inputStream(), emptySet(), logger)
+        checker.setSourcePath(emptyList())
+        checker.setAnnotationTypes(suppressAnnotations)
+        checker.process(artifact.toFile())  // the overload that takes Path can't handle jar files
+    }
+
+    private fun buildSignatures(additionalArtifacts: List<Path>, logger: Logger): Path {
+        val signaturesDirectory = System.getProperty("signaturesDirectory")
+            .let { requireNotNull(it) { "Specify signaturesDirectory with a system property" } }
+            .let { Path(it) }
+
+        val mergedSignaturesDirectory = signaturesDirectory.resolveSibling("signatures-merged").createDirectories()
+        val mergedSignaturesFile = createTempFile(mergedSignaturesDirectory, "merged", null)
+
+        val signatureInputStreams = signaturesDirectory.listDirectoryEntries()
+            .map { it.inputStream() }
+        val mergedSignaturesOutputStream = mergedSignaturesFile.outputStream()
+
+        val signatureBuilder = SignatureBuilder(signatureInputStreams.toTypedArray(), mergedSignaturesOutputStream, logger)
+        try {
+            additionalArtifacts.forEach {
+                signatureBuilder.process(it.toFile()) // the overload that takes Path can't handle jar files
+            }
+        } finally {
+            signatureBuilder.close()
+        }
+
+        return mergedSignaturesFile
+    }
+
+    private fun jarArtifact(basePath: String, jarBaseName: String, jarClassifier: String? = null): Path {
+        val kotlinVersion = System.getProperty("kotlinVersion")
+            .let { requireNotNull(it) { "Specify kotlinVersion with a system property" } }
+
+        val jarFullName = "$jarBaseName-$kotlinVersion${jarClassifier?.let { "-$it" } ?: ""}.jar"
+        val base = Path(basePath).absolute().normalize()
+        val file = base.listDirectoryEntries()
+            .firstOrNull { it.name == jarFullName }
+
+        return file ?: throw Exception("No file with name $jarFullName in $base")
+    }
+}
+
+private val suppressAnnotations = listOf(
+    "kotlin.reflect.jvm.internal.SuppressJdk6SignatureCheck",
+
+    // The following two fqn refer to the same annotation. The first is before its relocation,
+    // the second is after. See kotlin-reflect build script.
+    "org.jetbrains.kotlin.SuppressJdk6SignatureCheck",
+    "kotlin.reflect.jvm.internal.impl.SuppressJdk6SignatureCheck",
+)
+
+private val undefinedReferencesToIgnore = listOf(
+    "int Integer.compareUnsigned(int, int)",
+    "int Integer.remainderUnsigned(int, int)",
+    "int Integer.divideUnsigned(int, int)",
+
+    "int Long.compareUnsigned(long, long)",
+    "long Long.remainderUnsigned(long, long)",
+    "long Long.divideUnsigned(long, long)",
+
+    "int Boolean.hashCode(boolean)",
+    "int Byte.hashCode(byte)",
+    "int Short.hashCode(short)",
+    "int Integer.hashCode(int)",
+    "int Long.hashCode(long)",
+    "int Float.hashCode(float)",
+    "int Double.hashCode(double)",
+)
+
+private class TestLogger : Logger {
+    private val logger = PrintWriterLogger(System.out)
+
+    var hasError: Boolean = false
+        private set
+
+    override fun info(message: String) = logger.info(message)
+    override fun info(message: String, t: Throwable?) = logger.info(message, t)
+
+    override fun debug(message: String) = logger.debug(message)
+    override fun debug(message: String, t: Throwable?) = logger.debug(message, t)
+
+    override fun warn(message: String) = logger.warn(message)
+    override fun warn(message: String, t: Throwable?) = logger.warn(message, t)
+
+    override fun error(message: String) = error(message, null)
+    override fun error(message: String, t: Throwable?) {
+        val shouldIgnore = undefinedReferencesToIgnore.any {
+            message.endsWith("Undefined reference: $it")
+        }
+        if (shouldIgnore) {
+            return
+        }
+
+        hasError = true
+        logger.error(message, t)
+    }
+}
@@ -191,11 +191,14 @@ class CodeConformanceTest : TestCase() {
                "org.jetbrains.jet" in source
            },
            FileTestCase(
-                "%d source files contain references to package kotlin.reflect.jvm.internal.impl.\n" +
+                message = "%d source files contain references to package kotlin.reflect.jvm.internal.impl.\n" +
                        "This package contains internal reflection implementation and is a result of a " +
                        "post-processing of kotlin-reflect.jar by jarjar.\n" +
                        "Most probably you meant to use classes from org.jetbrains.kotlin.**.\n" +
-                        "Please change references in these files or exclude them in this test:\n%s"
+                        "Please change references in these files or exclude them in this test:\n%s",
+                allowedFiles = listOf(
+                    "libraries/tools/jdk-api-validator/src/test/JdkApiUsageTest.kt"
+                )
            ) { _, source ->
                "kotlin.reflect.jvm.internal.impl" in source
            },
@@ -583,6 +583,7 @@ if (buildProperties.inJpsBuildIdeaSync) {
            ":kotlin-stdlib:samples",
            ":kotlin-stdlib-jvm-minimal-for-test",
            ":tools:binary-compatibility-validator",
+            ":tools:jdk-api-validator",
            ":tools:kotlin-stdlib-gen",

            ":kotlin-test",
@@ -606,6 +607,7 @@ if (buildProperties.inJpsBuildIdeaSync) {
    project(":kotlin-stdlib-jvm-minimal-for-test").projectDir = "$rootDir/libraries/stdlib/jvm-minimal-for-test" as File

    project(':tools:binary-compatibility-validator').projectDir = "$rootDir/libraries/tools/binary-compatibility-validator" as File
+    project(':tools:jdk-api-validator').projectDir = "$rootDir/libraries/tools/jdk-api-validator" as File
    project(':tools:kotlin-stdlib-gen').projectDir = "$rootDir/libraries/tools/kotlin-stdlib-gen" as File

    project(':kotlin-test').projectDir = "$rootDir/libraries/kotlin.test" as File