Support strict metadata version semantics

Preface: Kotlin 1.3 will be able to read metadata of .class files
produced by Kotlin 1.4 (see KT-25972). Also, to simplify implementation
and to improve diagnostic messages, we're going to advance JVM metadata
version to 1.4.0 in Kotlin 1.4, and would like to keep it in sync with
the compiler version thereafter. This presents a problem: in an unlikely
event that before releasing 1.4, we find out that the metadata-reading
implementation in 1.3 was incorrect, we'd like to be able to fix the bug
in that implementation and _forbid_ 1.3 from reading metadata of 1.4.
But prior to this commit the only way to do this was to advance the
metadata version, in this case to 1.5, and that breaks the
metadata/compiler version equivalence we'd like to keep.

The solution is to add another boolean flag to the class file, called
"strict metadata version semantics", which signifies that if this class
file has metadata version 1.X, then it can only be read by the compilers
of versions 1.X and greater. This flag effectively disables the smooth
migration scenario proposed in KT-25972 (as does increasing metadata
version by 2), and will be used only in hopeless situations as in the
case described above.
This commit is contained in:
Alexander Udalov
2018-09-06 21:49:24 +03:00
parent 8c942752e8
commit 1d5d6b5b72
22 changed files with 167 additions and 38 deletions
@@ -123,7 +123,11 @@ public class ClassFileFactory implements OutputFileCollection {
generators.put(outputFilePath, new OutAndSourceFileList(CollectionsKt.toList(sourceFiles)) {
@Override
public byte[] asBytes(ClassBuilderFactory factory) {
return ModuleMappingKt.serializeToByteArray(moduleProto, state.getMetadataVersion(), 0);
int flags = 0;
if (state.getLanguageVersionSettings().getFlag(AnalysisFlag.getStrictMetadataVersionSemantics())) {
flags |= ModuleMapping.STRICT_METADATA_VERSION_SEMANTICS_FLAG;
}
return ModuleMappingKt.serializeToByteArray(moduleProto, state.getMetadataVersion(), flags);
}
@Override
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.codegen
import org.jetbrains.kotlin.codegen.state.GenerationState
import org.jetbrains.kotlin.config.AnalysisFlag
import org.jetbrains.kotlin.load.java.JvmAnnotationNames
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmBytecodeBinaryVersion
@@ -37,6 +38,9 @@ fun writeKotlinMetadata(
if (state.languageVersionSettings.isPreRelease()) {
flags = flags or JvmAnnotationNames.METADATA_PRE_RELEASE_FLAG
}
if (state.languageVersionSettings.getFlag(AnalysisFlag.strictMetadataVersionSemantics)) {
flags = flags or JvmAnnotationNames.METADATA_STRICT_VERSION_SEMANTICS_FLAG
}
if (flags != 0) {
av.visit(JvmAnnotationNames.METADATA_EXTRA_INT_FIELD_NAME, flags)
}
@@ -244,6 +244,12 @@ class K2JVMCompilerArguments : CommonCompilerArguments() {
@Argument(value = "-Xdisable-standard-script", description = "Disable standard kotlin script support")
var disableStandardScript: Boolean by FreezableVar(false)
@Argument(
value = "-Xgenerate-strict-metadata-version",
description = "Generate metadata with strict version semantics (see kdoc on Metadata.extraInt)"
)
var strictMetadataVersionSemantics: Boolean by FreezableVar(false)
@Argument(
value = "-Xfriend-paths",
valueDescription = "<path>",
@@ -253,6 +259,7 @@ class K2JVMCompilerArguments : CommonCompilerArguments() {
override fun configureAnalysisFlags(collector: MessageCollector): MutableMap<AnalysisFlag<*>, Any> {
val result = super.configureAnalysisFlags(collector)
result.put(AnalysisFlag.strictMetadataVersionSemantics, strictMetadataVersionSemantics)
result[AnalysisFlag.jsr305] = Jsr305Parser(collector).parse(
jsr305,
supportCompatqualCheckerFrameworkAnnotations
+2
View File
@@ -52,6 +52,8 @@ where advanced options include:
-Xskip-runtime-version-check Allow Kotlin runtime libraries of incompatible versions in the classpath
-Xstrict-java-nullability-assertions
Generate nullability assertions for non-null Java expressions
-Xgenerate-strict-metadata-version
Generate metadata with strict version semantics (see kdoc on Metadata.extraInt)
-Xsupport-compatqual-checker-framework-annotations=enable|disable
Specify behavior for Checker Framework compatqual annotations (NullableDecl/NonNullDecl).
Default value is 'enable'
@@ -0,0 +1,7 @@
package a
class C
fun f() {}
val v = Unit
@@ -0,0 +1,25 @@
error: incompatible classes were found in dependencies. Remove them from the classpath or use '-Xskip-metadata-version-check' to suppress errors
$TMP_DIR$/library.jar!/META-INF/main.kotlin_module: error: module was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.4.0, expected version is $ABI_VERSION$.
compiler/testData/compileKotlinAgainstCustomBinaries/strictMetadataVersionSemanticsOldVersion/source.kt:3:13: error: class 'a.C' was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.4.0, expected version is $ABI_VERSION$.
The class is loaded from $TMP_DIR$/library.jar!/a/C.class
fun test(c: C) {
^
compiler/testData/compileKotlinAgainstCustomBinaries/strictMetadataVersionSemanticsOldVersion/source.kt:4:5: error: unresolved reference: f
f()
^
compiler/testData/compileKotlinAgainstCustomBinaries/strictMetadataVersionSemanticsOldVersion/source.kt:5:5: error: unresolved reference: v
v
^
compiler/testData/compileKotlinAgainstCustomBinaries/strictMetadataVersionSemanticsOldVersion/source.kt:6:5: error: class 'a.C' was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.4.0, expected version is $ABI_VERSION$.
The class is loaded from $TMP_DIR$/library.jar!/a/C.class
c.let { C() }
^
compiler/testData/compileKotlinAgainstCustomBinaries/strictMetadataVersionSemanticsOldVersion/source.kt:6:7: error: class 'a.C' was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.4.0, expected version is $ABI_VERSION$.
The class is loaded from $TMP_DIR$/library.jar!/a/C.class
c.let { C() }
^
compiler/testData/compileKotlinAgainstCustomBinaries/strictMetadataVersionSemanticsOldVersion/source.kt:6:13: error: class 'a.C' was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.4.0, expected version is $ABI_VERSION$.
The class is loaded from $TMP_DIR$/library.jar!/a/C.class
c.let { C() }
^
COMPILATION_ERROR
@@ -0,0 +1,7 @@
import a.*
fun test(c: C) {
f()
v
c.let { C() }
}
@@ -0,0 +1,7 @@
package a
class C
fun f() {}
val v = Unit
@@ -0,0 +1,7 @@
import a.*
fun test(c: C) {
f()
v
c.let { C() }
}
@@ -332,6 +332,18 @@ class CompileKotlinAgainstCustomBinariesTest : AbstractKotlinCompilerIntegration
)
}
fun testStrictMetadataVersionSemanticsSameVersion() {
val library = compileLibrary("library", additionalOptions = listOf("-Xgenerate-strict-metadata-version"))
compileKotlin("source.kt", tmpdir, listOf(library))
}
fun testStrictMetadataVersionSemanticsOldVersion() {
val library = compileLibrary(
"library", additionalOptions = listOf("-Xgenerate-strict-metadata-version", "-Xmetadata-version=1.4.0")
)
compileKotlin("source.kt", tmpdir, listOf(library))
}
/*test source mapping generation when source info is absent*/
fun testInlineFunWithoutDebugInfo() {
compileKotlin("sourceInline.kt", tmpdir)
@@ -45,6 +45,9 @@ class AnalysisFlag<out T> internal constructor(
@JvmStatic
val skipMetadataVersionCheck by Flag.Boolean
@JvmStatic
val strictMetadataVersionSemantics by Flag.Boolean
@JvmStatic
val multiPlatformDoNotCheckActual by Flag.Boolean
@@ -38,6 +38,7 @@ public final class JvmAnnotationNames {
public static final int METADATA_MULTIFILE_PARTS_INHERIT_FLAG = 1 << 0;
public static final int METADATA_PRE_RELEASE_FLAG = 1 << 1;
public static final int METADATA_SCRIPT_FLAG = 1 << 2;
public static final int METADATA_STRICT_VERSION_SEMANTICS_FLAG = 1 << 3;
public static final Name DEFAULT_ANNOTATION_MEMBER_NAME = Name.identifier("value");
@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.load.kotlin.header;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.descriptors.SourceElement;
import org.jetbrains.kotlin.load.java.JvmAnnotationNames;
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmBytecodeBinaryVersion;
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion;
import org.jetbrains.kotlin.name.ClassId;
@@ -48,7 +49,7 @@ public class ReadKotlinClassHeaderAnnotationVisitor implements AnnotationVisitor
HEADER_KINDS.put(ClassId.topLevel(new FqName("kotlin.jvm.internal.KotlinSyntheticClass")), SYNTHETIC_CLASS);
}
private JvmMetadataVersion metadataVersion = null;
private int[] metadataVersionArray = null;
private JvmBytecodeBinaryVersion bytecodeVersion = null;
private String extraString = null;
private int extraInt = 0;
@@ -60,15 +61,15 @@ public class ReadKotlinClassHeaderAnnotationVisitor implements AnnotationVisitor
@Nullable
public KotlinClassHeader createHeader() {
if (headerKind == null) {
if (headerKind == null || metadataVersionArray == null) {
return null;
}
JvmMetadataVersion metadataVersion =
new JvmMetadataVersion(metadataVersionArray, (extraInt & JvmAnnotationNames.METADATA_STRICT_VERSION_SEMANTICS_FLAG) != 0);
if (!metadataVersion.isCompatible()) {
incompatibleData = data;
}
if (metadataVersion == null || !metadataVersion.isCompatible()) {
data = null;
}
else if (shouldHaveData() && data == null) {
@@ -79,7 +80,7 @@ public class ReadKotlinClassHeaderAnnotationVisitor implements AnnotationVisitor
return new KotlinClassHeader(
headerKind,
metadataVersion != null ? metadataVersion : JvmMetadataVersion.INVALID_VERSION,
metadataVersion,
bytecodeVersion != null ? bytecodeVersion : JvmBytecodeBinaryVersion.INVALID_VERSION,
data,
incompatibleData,
@@ -137,7 +138,7 @@ public class ReadKotlinClassHeaderAnnotationVisitor implements AnnotationVisitor
}
else if (METADATA_VERSION_FIELD_NAME.equals(string)) {
if (value instanceof int[]) {
metadataVersion = new JvmMetadataVersion((int[]) value);
metadataVersionArray = (int[]) value;
}
}
else if (BYTECODE_VERSION_FIELD_NAME.equals(string)) {
@@ -224,7 +225,7 @@ public class ReadKotlinClassHeaderAnnotationVisitor implements AnnotationVisitor
String string = name.asString();
if ("version".equals(string)) {
if (value instanceof int[]) {
metadataVersion = new JvmMetadataVersion((int[]) value);
metadataVersionArray = (int[]) value;
// If there's no bytecode binary version in the class file, we assume it to be equal to the metadata version
if (bytecodeVersion == null) {
@@ -11,12 +11,19 @@ import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
* The version of the metadata serialized by the compiler and deserialized by the compiler and reflection.
* This version includes the version of the core protobuf messages (metadata.proto) as well as JVM extensions (jvm_metadata.proto).
*/
class JvmMetadataVersion(vararg numbers: Int) : BinaryVersion(*numbers) {
// In Kotlin 1.4, JVM metadata version is going to be advanced to 1.4.0.
// Kotlin 1.3 is able to read metadata of versions up to Kotlin 1.4.
// NOTE: 1.0 is a pre-Kotlin-1.0 metadata version, with which the current compiler is incompatible
override fun isCompatible() =
major == 1 && minor in 1..4
class JvmMetadataVersion(versionArray: IntArray, val isStrictSemantics: Boolean) : BinaryVersion(*versionArray) {
constructor(vararg numbers: Int) : this(numbers, isStrictSemantics = false)
override fun isCompatible(): Boolean =
// NOTE: 1.0 is a pre-Kotlin-1.0 metadata version, with which the current compiler is incompatible
(major != 1 || minor != 0) &&
if (isStrictSemantics) {
isCompatibleTo(INSTANCE)
} else {
// In Kotlin 1.4, JVM metadata version is going to be advanced to 1.4.0.
// Kotlin 1.3 is able to read metadata of versions up to Kotlin 1.4 (unless the version has strict semantics).
major == 1 && minor <= 4
}
companion object {
@JvmField
@@ -31,6 +31,8 @@ class ModuleMapping private constructor(
@JvmField
val CORRUPTED: ModuleMapping = ModuleMapping(emptyMap(), BinaryModuleData(emptyList()), "CORRUPTED")
const val STRICT_METADATA_VERSION_SEMANTICS_FLAG = 1 shl 0
fun loadModuleMapping(
bytes: ByteArray?,
debugName: String,
@@ -50,15 +52,19 @@ class ModuleMapping private constructor(
return CORRUPTED
}
val version = JvmMetadataVersion(*versionNumber)
if (!skipMetadataVersionCheck && !version.isCompatible()) {
reportIncompatibleVersionError(version)
val preVersion = JvmMetadataVersion(*versionNumber)
if (!skipMetadataVersionCheck && !preVersion.isCompatible()) {
reportIncompatibleVersionError(preVersion)
return EMPTY
}
// Since Kotlin 1.4, we write integer flags between the version and the proto
if (isKotlin1Dot4OrLater(version)) {
stream.readInt()
val flags = if (isKotlin1Dot4OrLater(preVersion)) stream.readInt() else 0
val version = JvmMetadataVersion(versionNumber, (flags and STRICT_METADATA_VERSION_SEMANTICS_FLAG) != 0)
if (!skipMetadataVersionCheck && !version.isCompatible()) {
reportIncompatibleVersionError(version)
return EMPTY
}
val moduleProto = JvmModuleProtoBuf.Module.parseFrom(stream) ?: return EMPTY
@@ -18,6 +18,7 @@
package kotlin.reflect.jvm
import org.jetbrains.kotlin.load.java.JvmAnnotationNames
import org.jetbrains.kotlin.metadata.deserialization.TypeTable
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
@@ -36,7 +37,8 @@ fun <R> Function<R>.reflect(): KFunction<R>? {
val annotation = javaClass.getAnnotation(Metadata::class.java) ?: return null
val data = annotation.d1.takeUnless(Array<String>::isEmpty) ?: return null
val (nameResolver, proto) = JvmProtoBufUtil.readFunctionDataFrom(data, annotation.d2)
val metadataVersion = JvmMetadataVersion(*annotation.mv)
val metadataVersion =
JvmMetadataVersion(annotation.mv, (annotation.xi and JvmAnnotationNames.METADATA_STRICT_VERSION_SEMANTICS_FLAG) != 0)
val descriptor = deserializeToDescriptor(
javaClass, proto, nameResolver, TypeTable(proto.typeTable), metadataVersion, MemberDeserializer::loadFunction
@@ -26,7 +26,7 @@ import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils
import java.util.*
object KotlinJsMetadataVersionIndex : KotlinMetadataVersionIndexBase<KotlinJsMetadataVersionIndex, JsMetadataVersion>(
KotlinJsMetadataVersionIndex::class.java, ::JsMetadataVersion
KotlinJsMetadataVersionIndex::class.java, { version, _ -> JsMetadataVersion(*version) }
) {
override fun getIndexer() = INDEXER
@@ -20,6 +20,7 @@ import com.intellij.openapi.fileTypes.StdFileTypes
import com.intellij.util.indexing.DataIndexer
import com.intellij.util.indexing.FileBasedIndex
import com.intellij.util.indexing.FileContent
import org.jetbrains.kotlin.load.java.JvmAnnotationNames
import org.jetbrains.kotlin.load.java.JvmAnnotationNames.*
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
@@ -29,7 +30,8 @@ import org.jetbrains.org.objectweb.asm.ClassVisitor
import org.jetbrains.org.objectweb.asm.Opcodes
object KotlinJvmMetadataVersionIndex : KotlinMetadataVersionIndexBase<KotlinJvmMetadataVersionIndex, JvmMetadataVersion>(
KotlinJvmMetadataVersionIndex::class.java, ::JvmMetadataVersion
KotlinJvmMetadataVersionIndex::class.java,
{ version, isStrictSemantics -> JvmMetadataVersion(version, isStrictSemantics = isStrictSemantics!!) }
) {
override fun getIndexer() = INDEXER
@@ -37,16 +39,21 @@ object KotlinJvmMetadataVersionIndex : KotlinMetadataVersionIndexBase<KotlinJvmM
override fun getVersion() = VERSION
private val VERSION = 4
override fun isExtraBooleanNeeded(): Boolean = true
override fun getExtraBoolean(version: JvmMetadataVersion): Boolean = version.isStrictSemantics
private const val VERSION = 5
private val kindsToIndex = setOf(
KotlinClassHeader.Kind.CLASS,
KotlinClassHeader.Kind.FILE_FACADE,
KotlinClassHeader.Kind.MULTIFILE_CLASS
KotlinClassHeader.Kind.CLASS,
KotlinClassHeader.Kind.FILE_FACADE,
KotlinClassHeader.Kind.MULTIFILE_CLASS
)
private val INDEXER = DataIndexer<JvmMetadataVersion, Void, FileContent> { inputData: FileContent ->
var version: JvmMetadataVersion? = null
var versionArray: IntArray? = null
var isStrictSemantics = false
var annotationPresent = false
var kind: KotlinClassHeader.Kind? = null
@@ -61,11 +68,14 @@ object KotlinJvmMetadataVersionIndex : KotlinMetadataVersionIndexBase<KotlinJvmM
override fun visit(name: String, value: Any) {
when (name) {
METADATA_VERSION_FIELD_NAME -> if (value is IntArray) {
version = createBinaryVersion(value)
versionArray = value
}
KIND_FIELD_NAME -> if (value is Int) {
kind = KotlinClassHeader.Kind.getById(value)
}
METADATA_EXTRA_INT_FIELD_NAME -> if (value is Int) {
isStrictSemantics = (value and JvmAnnotationNames.METADATA_STRICT_VERSION_SEMANTICS_FLAG) != 0
}
}
}
}
@@ -73,15 +83,17 @@ object KotlinJvmMetadataVersionIndex : KotlinMetadataVersionIndexBase<KotlinJvmM
}, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES)
}
var version =
if (versionArray != null) createBinaryVersion(versionArray!!, isStrictSemantics) else null
if (kind !in kindsToIndex) {
// Do not index metadata version for synthetic classes
version = null
}
else if (annotationPresent && version == null) {
} else if (annotationPresent && version == null) {
// No version at all because the class is too old, or version is set to something weird
version = JvmMetadataVersion.INVALID_VERSION
}
if (version != null) mapOf(version!! to null) else mapOf()
if (version != null) mapOf(version to null) else emptyMap()
}
}
@@ -31,7 +31,7 @@ import java.io.DataOutput
*/
abstract class KotlinMetadataVersionIndexBase<T, V : BinaryVersion>(
private val classOfIndex: Class<T>,
protected val createBinaryVersion: (IntArray) -> V
protected val createBinaryVersion: (IntArray, Boolean?) -> V
) : ScalarIndexExtension<V>() {
override fun getName(): ID<V, Void> = ID.create<V, Void>(classOfIndex.canonicalName)
@@ -43,7 +43,9 @@ abstract class KotlinMetadataVersionIndexBase<T, V : BinaryVersion>(
override fun read(input: DataInput): V {
val size = DataInputOutputUtil.readINT(input)
return createBinaryVersion((0..size - 1).map { DataInputOutputUtil.readINT(input) }.toIntArray())
val versionArray = (0..size - 1).map { DataInputOutputUtil.readINT(input) }.toIntArray()
val extraBoolean = if (isExtraBooleanNeeded()) DataInputOutputUtil.readINT(input) == 1 else null
return createBinaryVersion(versionArray, extraBoolean)
}
override fun save(output: DataOutput, value: V) {
@@ -52,11 +54,17 @@ abstract class KotlinMetadataVersionIndexBase<T, V : BinaryVersion>(
for (number in array) {
DataInputOutputUtil.writeINT(output, number)
}
if (isExtraBooleanNeeded()) {
DataInputOutputUtil.writeINT(output, if (getExtraBoolean(value)) 1 else 0)
}
}
}
override fun dependsOnFileContent() = true
protected open fun isExtraBooleanNeeded(): Boolean = false
protected open fun getExtraBoolean(version: V): Boolean = throw UnsupportedOperationException()
protected val LOG: Logger = Logger.getInstance(classOfIndex)
protected inline fun tryBlock(inputData: FileContent, body: () -> Unit) {
@@ -313,7 +313,11 @@ sealed class KotlinClassMetadata(val header: KotlinClassHeader) {
@JvmStatic
fun read(header: KotlinClassHeader): KotlinClassMetadata? {
// We only support metadata of version 1.1.* (this is Kotlin from 1.0 until today)
if (!JvmMetadataVersion(*header.metadataVersion).isCompatible()) return null
if (!JvmMetadataVersion(
header.metadataVersion,
(header.extraInt and (1 shl 3)/* see JvmAnnotationNames.METADATA_STRICT_VERSION_SEMANTICS_FLAG */) != 0
).isCompatible()
) return null
return try {
when (header.kind) {
@@ -56,9 +56,11 @@ public annotation class Metadata(
/**
* An extra int. Bits of this number represent the following flags:
*
* 0 - this is a multi-file class facade or part, compiled with `-Xmultifile-parts-inherit`.
* 1 - this class file is compiled by a pre-release version of Kotlin and is not visible to release versions.
* 2 - this class file is a compiled Kotlin script source file (.kts).
* * 0 - this is a multi-file class facade or part, compiled with `-Xmultifile-parts-inherit`.
* * 1 - this class file is compiled by a pre-release version of Kotlin and is not visible to release versions.
* * 2 - this class file is a compiled Kotlin script source file (.kts).
* * 3 - the metadata of this class file is not supposed to be read by the compiler, whose major.minor version is less than
* the major.minor version of this metadata ([mv]).
*/
@SinceKotlin("1.1")
val xi: Int = 0