From 7346cf4777dd496cca705ccc3c5a2f7a2b9c4d82 Mon Sep 17 00:00:00 2001 From: Abduqodiri Qurbonzoda Date: Wed, 12 Jul 2023 05:13:08 +0000 Subject: [PATCH] Introduce jdk-api-validator to ensure kotlin-reflect uses jdk6 API Merge-request: KT-MR-6930 Merged-by: Abduqodiri Qurbonzoda --- .space/CODEOWNERS | 1 + build.gradle.kts | 1 + .../reflect/jvm/internal/CacheByClass.kt | 2 + .../internal/SuppressJdk6SignatureCheck.kt | 15 ++ .../kotlin/SuppressJdk6SignatureCheck.kt | 15 ++ .../storage/LockBasedStorageManager.java | 1 + gradle/jps.gradle.kts | 1 + gradle/verification-metadata.xml | 12 ++ libraries/tools/jdk-api-validator/ReadMe.md | 46 ++++++ .../tools/jdk-api-validator/build.gradle.kts | 41 +++++ .../src/test/JdkApiUsageTest.kt | 143 ++++++++++++++++++ .../kotlin/code/CodeConformanceTest.kt | 7 +- settings.gradle | 2 + 13 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 core/reflection.jvm/src/kotlin/reflect/jvm/internal/SuppressJdk6SignatureCheck.kt create mode 100644 core/util.runtime/src/org/jetbrains/kotlin/SuppressJdk6SignatureCheck.kt create mode 100644 libraries/tools/jdk-api-validator/ReadMe.md create mode 100644 libraries/tools/jdk-api-validator/build.gradle.kts create mode 100644 libraries/tools/jdk-api-validator/src/test/JdkApiUsageTest.kt diff --git a/.space/CODEOWNERS b/.space/CODEOWNERS index 712bd696332..028d3d5b32e 100644 --- a/.space/CODEOWNERS +++ b/.space/CODEOWNERS @@ -282,6 +282,7 @@ /libraries/kotlin-dom-api-compat/ "Kotlin JS" /libraries/tools/kotlin-build-tools-enum-compat/ "Kotlin Build Tools" /libraries/tools/gradle/ "Kotlin Build Tools" +/libraries/tools/jdk-api-validator/ "Kotlin Libraries" /libraries/tools/kotlin-allopen/ "Kotlin Build Tools" /libraries/tools/kotlin-annotations-jvm/ "Kotlin Libraries" /libraries/tools/kotlin-assignment/ "Kotlin Build Tools" diff --git a/build.gradle.kts b/build.gradle.kts index 16add5f5037..094d87371c8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -609,6 +609,7 @@ tasks { ":kotlin-test:kotlin-test-js-ir:kotlin-test-js-ir-it".takeIf { !kotlinBuildProperties.isInJpsBuildIdeaSync }, ":kotlinx-metadata-jvm", ":tools:binary-compatibility-validator", + ":tools:jdk-api-validator", //":kotlin-stdlib-wasm", )).forEach { dependsOn("$it:check") diff --git a/core/reflection.jvm/src/kotlin/reflect/jvm/internal/CacheByClass.kt b/core/reflection.jvm/src/kotlin/reflect/jvm/internal/CacheByClass.kt index b8330af62e9..38db5b93c81 100644 --- a/core/reflection.jvm/src/kotlin/reflect/jvm/internal/CacheByClass.kt +++ b/core/reflection.jvm/src/kotlin/reflect/jvm/internal/CacheByClass.kt @@ -43,6 +43,7 @@ internal fun createCache(compute: (Class<*>) -> V): CacheByClass { * ClassValue -> KPackageImpl.getClass() -> UrlClassloader -> all loaded classes by this CL -> * -> kotlin.reflect.jvm.internal.ClassValueCache -> ClassValue */ +@kotlin.reflect.jvm.internal.SuppressJdk6SignatureCheck private class ComputableClassValue(@JvmField val compute: (Class<*>) -> V) : ClassValue>() { override fun computeValue(type: Class<*>): SoftReference { return SoftReference(compute(type)) @@ -51,6 +52,7 @@ private class ComputableClassValue(@JvmField val compute: (Class<*>) -> V) : fun createNewCopy() = ComputableClassValue(compute) } +@kotlin.reflect.jvm.internal.SuppressJdk6SignatureCheck private class ClassValueCache(compute: (Class<*>) -> V) : CacheByClass() { @Volatile diff --git a/core/reflection.jvm/src/kotlin/reflect/jvm/internal/SuppressJdk6SignatureCheck.kt b/core/reflection.jvm/src/kotlin/reflect/jvm/internal/SuppressJdk6SignatureCheck.kt new file mode 100644 index 00000000000..951b86f7499 --- /dev/null +++ b/core/reflection.jvm/src/kotlin/reflect/jvm/internal/SuppressJdk6SignatureCheck.kt @@ -0,0 +1,15 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.reflect.jvm.internal + +/** + * Suppresses verification errors of the jdk-api-validator tool for certain scope. + * Such scopes include references to Java 8 API that are not available in Android API, + * but can be desugared by R8 or their execution is prevented on Android platform. + */ +@Retention(AnnotationRetention.BINARY) +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) +internal annotation class SuppressJdk6SignatureCheck diff --git a/core/util.runtime/src/org/jetbrains/kotlin/SuppressJdk6SignatureCheck.kt b/core/util.runtime/src/org/jetbrains/kotlin/SuppressJdk6SignatureCheck.kt new file mode 100644 index 00000000000..ff09a57b779 --- /dev/null +++ b/core/util.runtime/src/org/jetbrains/kotlin/SuppressJdk6SignatureCheck.kt @@ -0,0 +1,15 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +/** + * Suppresses verification errors of the jdk-api-validator tool for certain scope. + * Such scopes include references to Java 8 API that are not available in Android API, + * but can be desugared by R8 or their execution is prevented on Android platform. + */ +@Retention(AnnotationRetention.BINARY) +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) +internal annotation class SuppressJdk6SignatureCheck \ No newline at end of file diff --git a/core/util.runtime/src/org/jetbrains/kotlin/storage/LockBasedStorageManager.java b/core/util.runtime/src/org/jetbrains/kotlin/storage/LockBasedStorageManager.java index cf4348b2846..b40982d30e3 100644 --- a/core/util.runtime/src/org/jetbrains/kotlin/storage/LockBasedStorageManager.java +++ b/core/util.runtime/src/org/jetbrains/kotlin/storage/LockBasedStorageManager.java @@ -647,6 +647,7 @@ public class LockBasedStorageManager implements StorageManager { ); } + @org.jetbrains.kotlin.SuppressJdk6SignatureCheck private AssertionError unableToRemoveKey(K input, Throwable throwable) { return sanitizeStackTrace( new AssertionError("Unable to remove " diff --git a/gradle/jps.gradle.kts b/gradle/jps.gradle.kts index 212de4c5e25..da750b8f19e 100644 --- a/gradle/jps.gradle.kts +++ b/gradle/jps.gradle.kts @@ -22,6 +22,7 @@ fun updateCompilerXml() { "libraries/tools/binary-compatibility-validator", "libraries/tools/dukat", "libraries/tools/gradle", + "libraries/tools/jdk-api-validator", "libraries/tools/kotlin-allopen", "libraries/tools/kotlin-annotation-processing", "libraries/tools/kotlin-assignment", diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 25bdd27624e..f2c122f141d 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -6803,6 +6803,12 @@ + + + + + + @@ -6833,6 +6839,12 @@ + + + + + + diff --git a/libraries/tools/jdk-api-validator/ReadMe.md b/libraries/tools/jdk-api-validator/ReadMe.md new file mode 100644 index 00000000000..2685fac5658 --- /dev/null +++ b/libraries/tools/jdk-api-validator/ReadMe.md @@ -0,0 +1,46 @@ +# Verifies that a library compiled with a newer JDK is compatible with older JDK API + +Currently, the project only checks `kotlin-reflect` for JDK 1.6 compatibility. +The check is necessary to make sure applications using `kotlin-reflect` can run on older Android devices. + +## How to run + +Run from the root directory of the kotlin project: + +`./gradlew :tools:jdk-api-validator:test` + +## How to interpret the result + +Successful completion of the `test` task means the checked libraries are compatible with JDK 1.6 API. +In case of failure, the exact location and name of the violating API references are logged as error. + +An example of validation error: + +`[ERROR] /kotlin/libraries/reflect/build/libs/kotlin-reflect-1.9.255-SNAPSHOT.jar:kotlin/reflect/jvm/internal/ComputableClassValue.class:47: Undefined reference: void ClassValue.()` + +## How to fix a failure + +If the violating reference can be desugared by R8 or its execution is prevented on Android platform, +the error can be suppressed. See `suppressAnnotations` and `undefinedReferencesToIgnore` in `JdkApiUsageTest.kt`. +Otherwise, the API should be avoided. + +To check if an API can be desugared by R8: +1. Identify the earliest R8 version that supports current Kotlin version [here](https://developer.android.com/build/kotlin-support). +2. Download the R8 version jar artifact from Google's maven repository [here](https://maven.google.com/web/index.html#com.android.tools:r8). +3. Run it using the `BackportedMethodList` entry point, e.g., `java -cp r8-8.0.40.jar com.android.tools.r8.BackportedMethodList`. +4. Check if the violating API reference is in the printed list of methods. + +Also, you can get the list of backported methods the downloaded version supports for a given Android API level: +```shell +$ java -cp r8-8.0.40.jar com.android.tools.r8.BackportedMethodList --help +Usage: BackportedMethodList [options] + Options are: + --output # Output result in . + --min-api # Minimum Android API level for the application + --desugared-lib # Desugared library configuration (JSON from the + # configuration) + --lib # The compilation SDK library (android.jar) + --version # Print the version of BackportedMethodList. + --help # Print this message. +``` + diff --git a/libraries/tools/jdk-api-validator/build.gradle.kts b/libraries/tools/jdk-api-validator/build.gradle.kts new file mode 100644 index 00000000000..e5b6993b545 --- /dev/null +++ b/libraries/tools/jdk-api-validator/build.gradle.kts @@ -0,0 +1,41 @@ +plugins { + id("kotlin") +} + +repositories { + mavenCentral() +} + +val testArtifacts by configurations.creating +val signature by configurations.creating + +sourceSets { + "main" { none() } + "test" { kotlin.srcDir("src/test") } +} + +dependencies { + implementation("org.codehaus.mojo:animal-sniffer:1.21") + implementation(kotlinStdlib()) + + testImplementation(project(":kotlin-test:kotlin-test-junit")) + + testArtifacts(project(":kotlin-reflect")) + + signature("org.codehaus.mojo.signature:java16:1.1@signature") +} + +val signaturesDirectory = buildDir.resolve("signatures") + +val collectSignatures by tasks.registering(Sync::class) { + from(signature) + into(signaturesDirectory) +} + +tasks.getByName("test") { + dependsOn(collectSignatures) + dependsOn(testArtifacts) + + systemProperty("kotlinVersion", project.version) + systemProperty("signaturesDirectory", signaturesDirectory) +} diff --git a/libraries/tools/jdk-api-validator/src/test/JdkApiUsageTest.kt b/libraries/tools/jdk-api-validator/src/test/JdkApiUsageTest.kt new file mode 100644 index 00000000000..e0f0265b0e3 --- /dev/null +++ b/libraries/tools/jdk-api-validator/src/test/JdkApiUsageTest.kt @@ -0,0 +1,143 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.tools.tests + +import org.codehaus.mojo.animal_sniffer.* +import org.codehaus.mojo.animal_sniffer.logging.Logger +import org.codehaus.mojo.animal_sniffer.logging.PrintWriterLogger +import java.nio.file.Path +import kotlin.io.path.* +import kotlin.test.* + +class JdkApiUsageTest { + + @Test + fun kotlinReflect() { + testApiUsage( + // Do not use the final jar artifact. It's shrunk by proguard. + jarArtifact("../../reflect/build/libs", "kotlin-reflect", "shadow"), + dependencies = listOf(jarArtifact("../../stdlib/jvm/build/libs", "kotlin-stdlib")) + ) + } + + private fun testApiUsage(artifact: Path, dependencies: List) { + val logger = TestLogger() + val additionalArtifacts = buildList { + add(artifact) + addAll(dependencies) + } + + val signatures = buildSignatures(additionalArtifacts, logger) + if (logger.hasError) { + fail("Building signatures has failed. See console logs for details.") + } + + checkSignatures(artifact, signatures, logger) + if (logger.hasError) { + fail("Checking signatures has failed. See console logs for details. " + + "See libraries/tools/jdk-api-validator/ReadMe.md to find out how to fix the failures.") + } + } + + private fun checkSignatures(artifact: Path, signatures: Path, logger: Logger) { + val checker = SignatureChecker(signatures.inputStream(), emptySet(), logger) + checker.setSourcePath(emptyList()) + checker.setAnnotationTypes(suppressAnnotations) + checker.process(artifact.toFile()) // the overload that takes Path can't handle jar files + } + + private fun buildSignatures(additionalArtifacts: List, logger: Logger): Path { + val signaturesDirectory = System.getProperty("signaturesDirectory") + .let { requireNotNull(it) { "Specify signaturesDirectory with a system property" } } + .let { Path(it) } + + val mergedSignaturesDirectory = signaturesDirectory.resolveSibling("signatures-merged").createDirectories() + val mergedSignaturesFile = createTempFile(mergedSignaturesDirectory, "merged", null) + + val signatureInputStreams = signaturesDirectory.listDirectoryEntries() + .map { it.inputStream() } + val mergedSignaturesOutputStream = mergedSignaturesFile.outputStream() + + val signatureBuilder = SignatureBuilder(signatureInputStreams.toTypedArray(), mergedSignaturesOutputStream, logger) + try { + additionalArtifacts.forEach { + signatureBuilder.process(it.toFile()) // the overload that takes Path can't handle jar files + } + } finally { + signatureBuilder.close() + } + + return mergedSignaturesFile + } + + private fun jarArtifact(basePath: String, jarBaseName: String, jarClassifier: String? = null): Path { + val kotlinVersion = System.getProperty("kotlinVersion") + .let { requireNotNull(it) { "Specify kotlinVersion with a system property" } } + + val jarFullName = "$jarBaseName-$kotlinVersion${jarClassifier?.let { "-$it" } ?: ""}.jar" + val base = Path(basePath).absolute().normalize() + val file = base.listDirectoryEntries() + .firstOrNull { it.name == jarFullName } + + return file ?: throw Exception("No file with name $jarFullName in $base") + } +} + +private val suppressAnnotations = listOf( + "kotlin.reflect.jvm.internal.SuppressJdk6SignatureCheck", + + // The following two fqn refer to the same annotation. The first is before its relocation, + // the second is after. See kotlin-reflect build script. + "org.jetbrains.kotlin.SuppressJdk6SignatureCheck", + "kotlin.reflect.jvm.internal.impl.SuppressJdk6SignatureCheck", +) + +private val undefinedReferencesToIgnore = listOf( + "int Integer.compareUnsigned(int, int)", + "int Integer.remainderUnsigned(int, int)", + "int Integer.divideUnsigned(int, int)", + + "int Long.compareUnsigned(long, long)", + "long Long.remainderUnsigned(long, long)", + "long Long.divideUnsigned(long, long)", + + "int Boolean.hashCode(boolean)", + "int Byte.hashCode(byte)", + "int Short.hashCode(short)", + "int Integer.hashCode(int)", + "int Long.hashCode(long)", + "int Float.hashCode(float)", + "int Double.hashCode(double)", +) + +private class TestLogger : Logger { + private val logger = PrintWriterLogger(System.out) + + var hasError: Boolean = false + private set + + override fun info(message: String) = logger.info(message) + override fun info(message: String, t: Throwable?) = logger.info(message, t) + + override fun debug(message: String) = logger.debug(message) + override fun debug(message: String, t: Throwable?) = logger.debug(message, t) + + override fun warn(message: String) = logger.warn(message) + override fun warn(message: String, t: Throwable?) = logger.warn(message, t) + + override fun error(message: String) = error(message, null) + override fun error(message: String, t: Throwable?) { + val shouldIgnore = undefinedReferencesToIgnore.any { + message.endsWith("Undefined reference: $it") + } + if (shouldIgnore) { + return + } + + hasError = true + logger.error(message, t) + } +} \ No newline at end of file diff --git a/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt b/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt index c3839aacfbd..5e846d77148 100644 --- a/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt +++ b/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt @@ -191,11 +191,14 @@ class CodeConformanceTest : TestCase() { "org.jetbrains.jet" in source }, FileTestCase( - "%d source files contain references to package kotlin.reflect.jvm.internal.impl.\n" + + message = "%d source files contain references to package kotlin.reflect.jvm.internal.impl.\n" + "This package contains internal reflection implementation and is a result of a " + "post-processing of kotlin-reflect.jar by jarjar.\n" + "Most probably you meant to use classes from org.jetbrains.kotlin.**.\n" + - "Please change references in these files or exclude them in this test:\n%s" + "Please change references in these files or exclude them in this test:\n%s", + allowedFiles = listOf( + "libraries/tools/jdk-api-validator/src/test/JdkApiUsageTest.kt" + ) ) { _, source -> "kotlin.reflect.jvm.internal.impl" in source }, diff --git a/settings.gradle b/settings.gradle index 7a1d67f22ac..e3805ee45d8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -583,6 +583,7 @@ if (buildProperties.inJpsBuildIdeaSync) { ":kotlin-stdlib:samples", ":kotlin-stdlib-jvm-minimal-for-test", ":tools:binary-compatibility-validator", + ":tools:jdk-api-validator", ":tools:kotlin-stdlib-gen", ":kotlin-test", @@ -606,6 +607,7 @@ if (buildProperties.inJpsBuildIdeaSync) { project(":kotlin-stdlib-jvm-minimal-for-test").projectDir = "$rootDir/libraries/stdlib/jvm-minimal-for-test" as File project(':tools:binary-compatibility-validator').projectDir = "$rootDir/libraries/tools/binary-compatibility-validator" as File + project(':tools:jdk-api-validator').projectDir = "$rootDir/libraries/tools/jdk-api-validator" as File project(':tools:kotlin-stdlib-gen').projectDir = "$rootDir/libraries/tools/kotlin-stdlib-gen" as File project(':kotlin-test').projectDir = "$rootDir/libraries/kotlin.test" as File