[Test] Migrate AbstractBlackBoxCodegenTest to new infrastructure

This commit is contained in:
Dmitriy Novozhilov
2020-12-25 12:41:20 +03:00
committed by TeamCityServer
parent f01122d8dc
commit 85c87f7df9
47 changed files with 7856 additions and 2932 deletions
@@ -25,6 +25,8 @@ dependencies {
testCompile(project(":compiler:fir:entrypoint"))
testCompile(projectTests(":compiler:test-infrastructure-utils"))
testCompile(project(":kotlin-preloader"))
testCompile(androidDxJar()) { isTransitive = false }
testCompile(commonDep("com.android.tools:r8"))
testCompileOnly(intellijCoreDep()) { includeJars("intellij-core") }
testCompile(intellijDep()) {
@@ -0,0 +1,294 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.codegen
import org.jetbrains.kotlin.test.KtAssert
import org.jetbrains.org.objectweb.asm.*
class BytecodeListingTextCollectingVisitor(
val filter: Filter,
val withSignatures: Boolean,
api: Int = Opcodes.API_VERSION,
val withAnnotations: Boolean = true
) : ClassVisitor(api) {
companion object {
@JvmOverloads
fun getText(
factory: ClassFileFactory,
filter: Filter = Filter.EMPTY,
withSignatures: Boolean = false,
withAnnotations: Boolean = true
) = factory.getClassFiles()
.sortedBy { it.relativePath }
.mapNotNull {
val cr = ClassReader(it.asByteArray())
val visitor = BytecodeListingTextCollectingVisitor(filter, withSignatures, withAnnotations = withAnnotations)
cr.accept(visitor, ClassReader.SKIP_CODE)
if (!filter.shouldWriteClass(cr.access, cr.className)) null else visitor.text
}.joinToString("\n\n", postfix = "\n")
private val CLASS_OR_FIELD_OR_METHOD = setOf(ModifierTarget.CLASS, ModifierTarget.FIELD, ModifierTarget.METHOD)
private val CLASS_OR_METHOD = setOf(ModifierTarget.CLASS, ModifierTarget.METHOD)
private val FIELD_ONLY = setOf(ModifierTarget.FIELD)
private val METHOD_ONLY = setOf(ModifierTarget.METHOD)
private val FIELD_OR_METHOD = setOf(ModifierTarget.FIELD, ModifierTarget.METHOD)
// TODO ACC_MANDATED - requires reading Parameters attribute, which we don't generate by default
internal val MODIFIERS =
arrayOf(
Modifier("public", Opcodes.ACC_PUBLIC, CLASS_OR_FIELD_OR_METHOD),
Modifier("protected", Opcodes.ACC_PROTECTED, CLASS_OR_FIELD_OR_METHOD),
Modifier("private", Opcodes.ACC_PRIVATE, CLASS_OR_FIELD_OR_METHOD),
Modifier("synthetic", Opcodes.ACC_SYNTHETIC, CLASS_OR_FIELD_OR_METHOD),
Modifier("bridge", Opcodes.ACC_BRIDGE, METHOD_ONLY),
Modifier("volatile", Opcodes.ACC_VOLATILE, FIELD_ONLY),
Modifier("synchronized", Opcodes.ACC_SYNCHRONIZED, METHOD_ONLY),
Modifier("varargs", Opcodes.ACC_VARARGS, METHOD_ONLY),
Modifier("transient", Opcodes.ACC_TRANSIENT, FIELD_ONLY),
Modifier("native", Opcodes.ACC_NATIVE, METHOD_ONLY),
Modifier("deprecated", Opcodes.ACC_DEPRECATED, CLASS_OR_FIELD_OR_METHOD),
Modifier("final", Opcodes.ACC_FINAL, CLASS_OR_FIELD_OR_METHOD),
Modifier("strict", Opcodes.ACC_STRICT, METHOD_ONLY),
Modifier("enum", Opcodes.ACC_ENUM, FIELD_ONLY), // ACC_ENUM modifier on class is handled in 'classOrInterface'
Modifier("abstract", Opcodes.ACC_ABSTRACT, CLASS_OR_METHOD, excludedMask = Opcodes.ACC_INTERFACE),
Modifier("static", Opcodes.ACC_STATIC, FIELD_OR_METHOD)
)
}
interface Filter {
fun shouldWriteClass(access: Int, name: String): Boolean
fun shouldWriteMethod(access: Int, name: String, desc: String): Boolean
fun shouldWriteField(access: Int, name: String, desc: String): Boolean
fun shouldWriteInnerClass(name: String, outerName: String?, innerName: String?): Boolean
object EMPTY : Filter {
override fun shouldWriteClass(access: Int, name: String) = true
override fun shouldWriteMethod(access: Int, name: String, desc: String) = true
override fun shouldWriteField(access: Int, name: String, desc: String) = true
override fun shouldWriteInnerClass(name: String, outerName: String?, innerName: String?) = true
}
object ForCodegenTests : Filter {
override fun shouldWriteClass(access: Int, name: String): Boolean = !name.startsWith("helpers/")
override fun shouldWriteMethod(access: Int, name: String, desc: String): Boolean = true
override fun shouldWriteField(access: Int, name: String, desc: String): Boolean = true
override fun shouldWriteInnerClass(name: String, outerName: String?, innerName: String?): Boolean = true
}
}
private class Declaration(val text: String, val annotations: MutableList<String> = arrayListOf())
private val declarationsInsideClass = arrayListOf<Declaration>()
private val classAnnotations = arrayListOf<String>()
private var className = ""
private var classAccess = 0
private var classSignature: String? = ""
private fun addAnnotation(desc: String, list: MutableList<String> = declarationsInsideClass.last().annotations) {
val name = Type.getType(desc).className
list.add("@$name ")
}
private fun addModifier(text: String, list: MutableList<String>) {
list.add("$text ")
}
internal enum class ModifierTarget {
CLASS, FIELD, METHOD
}
internal class Modifier(
val text: String,
private val mask: Int,
private val applicableTo: Set<ModifierTarget>,
private val excludedMask: Int = 0
) {
fun hasModifier(access: Int, target: ModifierTarget) =
access and mask != 0 &&
access and excludedMask == 0 &&
applicableTo.contains(target)
}
private fun handleModifiers(
target: ModifierTarget,
access: Int,
list: MutableList<String> = declarationsInsideClass.last().annotations
) {
for (modifier in MODIFIERS) {
if (modifier.hasModifier(access, target)) {
addModifier(modifier.text, list)
}
}
}
private fun getModifiers(target: ModifierTarget, access: Int) =
MODIFIERS.filter { it.hasModifier(access, target) }.joinToString(separator = " ") { it.text }
private fun classOrInterface(access: Int): String {
return when {
access and Opcodes.ACC_ANNOTATION != 0 -> "annotation class"
access and Opcodes.ACC_ENUM != 0 -> "enum class"
access and Opcodes.ACC_INTERFACE != 0 -> "interface"
else -> "class"
}
}
val text: String
get() = StringBuilder().apply {
if (classAnnotations.isNotEmpty()) {
append(classAnnotations.joinToString("\n", postfix = "\n"))
}
arrayListOf<String>().apply { handleModifiers(ModifierTarget.CLASS, classAccess, this) }.forEach { append(it) }
append(classOrInterface(classAccess))
if (withSignatures) {
append("<$classSignature> ")
}
append(" ")
append(className)
if (declarationsInsideClass.isNotEmpty()) {
append(" {\n")
for (declaration in declarationsInsideClass.sortedBy { it.text }) {
append(" ").append(declaration.annotations.joinToString("")).append(declaration.text).append("\n")
}
append("}")
}
}.toString()
override fun visitSource(source: String?, debug: String?) {
if (source != null) {
declarationsInsideClass.add(Declaration("// source: '$source'"))
} else {
declarationsInsideClass.add(Declaration("// source: null"))
}
}
override fun visitMethod(access: Int, name: String, desc: String, signature: String?, exceptions: Array<out String>?): MethodVisitor? {
if (!filter.shouldWriteMethod(access, name, desc)) {
return null
}
val returnType = Type.getReturnType(desc).className
val parameterTypes = Type.getArgumentTypes(desc).map { it.className }
val methodAnnotations = arrayListOf<String>()
val parameterAnnotations = hashMapOf<Int, MutableList<String>>()
handleModifiers(ModifierTarget.METHOD, access, methodAnnotations)
val methodParamCount = Type.getArgumentTypes(desc).size
return object : MethodVisitor(Opcodes.API_VERSION) {
private var visibleAnnotableParameterCount = methodParamCount
private var invisibleAnnotableParameterCount = methodParamCount
override fun visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor? {
if (withAnnotations) {
val type = Type.getType(desc).className
methodAnnotations += "@$type "
}
return super.visitAnnotation(desc, visible)
}
override fun visitParameterAnnotation(parameter: Int, desc: String, visible: Boolean): AnnotationVisitor? {
if (withAnnotations) {
val type = Type.getType(desc).className
parameterAnnotations.getOrPut(
parameter + methodParamCount - (if (visible) visibleAnnotableParameterCount else invisibleAnnotableParameterCount),
{ arrayListOf() }).add("@$type ")
}
return super.visitParameterAnnotation(parameter, desc, visible)
}
override fun visitEnd() {
val parameterWithAnnotations = parameterTypes.mapIndexed { index, parameter ->
val annotations = parameterAnnotations.getOrElse(index, { emptyList() }).joinToString("")
"${annotations}p$index: $parameter"
}.joinToString()
val signatureIfRequired = if (withSignatures) "<$signature> " else ""
declarationsInsideClass.add(
Declaration("${signatureIfRequired}method $name($parameterWithAnnotations): $returnType", methodAnnotations)
)
super.visitEnd()
}
@Suppress("NOTHING_TO_OVERRIDE")
override fun visitAnnotableParameterCount(parameterCount: Int, visible: Boolean) {
if (visible)
visibleAnnotableParameterCount = parameterCount
else {
invisibleAnnotableParameterCount = parameterCount
}
}
}
}
override fun visitField(access: Int, name: String, desc: String, signature: String?, value: Any?): FieldVisitor? {
if (!filter.shouldWriteField(access, name, desc)) {
return null
}
val type = Type.getType(desc).className
val fieldSignature = if (withSignatures) "<$signature> " else ""
val fieldDeclaration = Declaration("field $fieldSignature$name: $type")
declarationsInsideClass.add(fieldDeclaration)
handleModifiers(ModifierTarget.FIELD, access)
if (access and Opcodes.ACC_VOLATILE != 0) addModifier("volatile", fieldDeclaration.annotations)
return object : FieldVisitor(Opcodes.API_VERSION) {
override fun visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor? {
if (withAnnotations) {
addAnnotation(desc)
}
return super.visitAnnotation(desc, visible)
}
}
}
override fun visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor? {
if (withAnnotations) {
val name = Type.getType(desc).className
classAnnotations.add("@$name")
}
return super.visitAnnotation(desc, visible)
}
override fun visit(version: Int, access: Int, name: String, signature: String?, superName: String?, interfaces: Array<out String>?) {
className = name
classAccess = access
classSignature = signature
}
override fun visitOuterClass(owner: String, name: String?, descriptor: String?) {
if (name == null) {
KtAssert.assertNull("", descriptor)
declarationsInsideClass.add(Declaration("enclosing class $owner"))
} else {
KtAssert.assertNotNull("", descriptor)
declarationsInsideClass.add(Declaration("enclosing method $owner.$name$descriptor"))
}
}
override fun visitInnerClass(name: String, outerName: String?, innerName: String?, access: Int) {
if (!filter.shouldWriteInnerClass(name, outerName, innerName)) {
return
}
when {
innerName == null -> {
KtAssert.assertNull("Anonymous classes should have neither innerName nor outerName. Name=$name, outerName=$outerName", outerName)
declarationsInsideClass.add(Declaration("inner (anonymous) class $name"))
}
outerName == null -> {
declarationsInsideClass.add(Declaration("inner (local) class $name $innerName"))
}
name == "$outerName$$innerName" -> {
declarationsInsideClass.add(Declaration("${getModifiers(ModifierTarget.CLASS, access)} inner class $name"))
}
else -> {
declarationsInsideClass.add(Declaration("inner (unrecognized) class $name $outerName $innerName"))
}
}
}
}
@@ -11,17 +11,29 @@ import kotlin.collections.CollectionsKt;
import kotlin.io.FilesKt;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.backend.common.output.OutputFile;
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment;
import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime;
import org.jetbrains.kotlin.config.JvmTarget;
import org.jetbrains.kotlin.test.Assertions;
import org.jetbrains.kotlin.test.JvmCompilationUtils;
import org.jetbrains.kotlin.test.KtAssert;
import org.jetbrains.kotlin.test.util.KtTestUtil;
import org.jetbrains.kotlin.utils.ExceptionUtilsKt;
import org.jetbrains.kotlin.utils.StringsKt;
import org.jetbrains.org.objectweb.asm.ClassReader;
import org.jetbrains.org.objectweb.asm.tree.ClassNode;
import org.jetbrains.org.objectweb.asm.tree.MethodNode;
import org.jetbrains.org.objectweb.asm.tree.analysis.Analyzer;
import org.jetbrains.org.objectweb.asm.tree.analysis.AnalyzerException;
import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue;
import org.jetbrains.org.objectweb.asm.tree.analysis.SimpleVerifier;
import org.jetbrains.org.objectweb.asm.util.Textifier;
import org.jetbrains.org.objectweb.asm.util.TraceMethodVisitor;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -165,4 +177,65 @@ public class CodegenTestUtil {
return javaFilePaths;
}
private static final boolean IS_SOURCE_6_STILL_SUPPORTED =
// JDKs up to 11 do support -source/target 1.6, but later -- don't.
Arrays.asList("1.6", "1.7", "1.8", "9", "10", "11").contains(System.getProperty("java.specification.version"));
private static final String JAVA_COMPILATION_TARGET = System.getProperty("kotlin.test.java.compilation.target");
@Nullable
public static String computeJavaTarget(@NotNull List<String> javacOptions, @Nullable JvmTarget kotlinTarget) {
if (JAVA_COMPILATION_TARGET != null && !javacOptions.contains("-target"))
return JAVA_COMPILATION_TARGET;
if (kotlinTarget != null && kotlinTarget.compareTo(JvmTarget.JVM_1_6) > 0)
return kotlinTarget.getDescription();
if (IS_SOURCE_6_STILL_SUPPORTED)
return "1.6";
return null;
}
public static boolean verifyAllFilesWithAsm(ClassFileFactory factory, ClassLoader loader, boolean reportProblems) {
boolean noErrors = true;
for (OutputFile file : ClassFileUtilsKt.getClassFiles(factory)) {
noErrors &= verifyWithAsm(file, loader, reportProblems);
}
return noErrors;
}
private static boolean verifyWithAsm(@NotNull OutputFile file, ClassLoader loader, boolean reportProblems) {
ClassNode classNode = new ClassNode();
new ClassReader(file.asByteArray()).accept(classNode, 0);
SimpleVerifier verifier = new SimpleVerifier();
verifier.setClassLoader(loader);
Analyzer<BasicValue> analyzer = new Analyzer<>(verifier);
boolean noErrors = true;
for (MethodNode method : classNode.methods) {
try {
analyzer.analyze(classNode.name, method);
}
catch (Throwable e) {
if (reportProblems) {
System.err.println(file.asText());
System.err.println(classNode.name + "::" + method.name + method.desc);
//noinspection InstanceofCatchParameter
if (e instanceof AnalyzerException) {
// Print the erroneous instruction
TraceMethodVisitor tmv = new TraceMethodVisitor(new Textifier());
((AnalyzerException) e).node.accept(tmv);
PrintWriter pw = new PrintWriter(System.err);
tmv.p.print(pw);
pw.flush();
}
e.printStackTrace();
}
noErrors = false;
}
}
return noErrors;
}
}
@@ -0,0 +1,84 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.codegen;
import com.android.tools.r8.*;
import com.android.tools.r8.origin.PathOrigin;
import kotlin.Pair;
import org.jetbrains.kotlin.backend.common.output.OutputFile;
import org.jetbrains.kotlin.test.KtAssert;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
public class D8Checker {
private D8Checker() {
}
public static void check(ClassFileFactory outputFiles) {
runD8(builder -> {
for (OutputFile file : ClassFileUtilsKt.getClassFiles(outputFiles)) {
byte[] bytes = file.asByteArray();
builder.addClassProgramData(bytes, new PathOrigin(Paths.get(file.getRelativePath())));
}
});
}
public static void checkFilesWithD8(Collection<Pair<byte[], String>> classFiles) {
runD8(builder -> {
classFiles.forEach(pair -> {
builder.addClassProgramData(pair.getFirst(), new PathOrigin(Paths.get(pair.getSecond())));
});
});
}
// Compilation with D8 should proceed with no output. There should be no info, warnings, or errors.
static class TestDiagnosticsHandler implements DiagnosticsHandler {
@Override
public void error(Diagnostic diagnostic) {
KtAssert.fail("D8 dexing error: " + diagnostic.getDiagnosticMessage());
}
@Override
public void warning(Diagnostic diagnostic) {
KtAssert.fail("D8 dexing warning: " + diagnostic.getDiagnosticMessage());
}
@Override
public void info(Diagnostic diagnostic) {
KtAssert.fail("D8 dexing info: " + diagnostic.getDiagnosticMessage());
}
}
private static void runD8(Consumer<D8Command.Builder> addInput) {
ProgramConsumer ignoreOutputConsumer = new DexIndexedConsumer.ForwardingConsumer(null);
D8Command.Builder builder = D8Command.builder(new TestDiagnosticsHandler())
.setMinApiLevel(28)
.setMode(CompilationMode.DEBUG)
.setProgramConsumer(ignoreOutputConsumer);
addInput.accept(builder);
try {
D8.run(builder.build(), Executors.newSingleThreadExecutor());
}
catch (CompilationFailedException e) {
KtAssert.fail(generateExceptionMessage(e));
}
}
private static String generateExceptionMessage(Throwable e) {
StringWriter writer = new StringWriter();
try (PrintWriter printWriter = new PrintWriter(writer)) {
e.printStackTrace(printWriter);
String stackTrace = writer.toString();
return stackTrace;
}
}
}
@@ -0,0 +1,90 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.codegen;
import com.android.dx.cf.direct.DirectClassFile;
import com.android.dx.cf.direct.StdAttributeFactory;
import com.android.dx.command.dexer.Main;
import com.android.dx.dex.cf.CfTranslator;
import com.android.dx.dex.file.DexFile;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.kotlin.backend.common.output.OutputFile;
import org.jetbrains.kotlin.test.KtAssert;
import org.jetbrains.org.objectweb.asm.Opcodes;
import java.io.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DxChecker {
public static final boolean RUN_DX_CHECKER = true;
private static final Pattern STACK_TRACE_PATTERN = Pattern.compile("[\\s]+at .*");
private DxChecker() {
}
public static void check(ClassFileFactory outputFiles) {
Main.Arguments arguments = new Main.Arguments();
String[] array = new String[1];
array[0] = "testArgs";
arguments.parse(array);
for (OutputFile file : ClassFileUtilsKt.getClassFiles(outputFiles)) {
try {
byte[] bytes = file.asByteArray();
if (isJava6Class(bytes)) {
checkFileWithDx(bytes, file.getRelativePath(), arguments);
}
}
catch (Throwable e) {
KtAssert.fail(generateExceptionMessage(e));
}
}
}
private static boolean isJava6Class(byte[] bytes) throws IOException {
try (DataInputStream stream = new DataInputStream(new ByteArrayInputStream(bytes))) {
int header = stream.readInt();
if (0xCAFEBABE != header) {
throw new IOException("Invalid header class header: " + header);
}
int minor = stream.readUnsignedShort();
int major = stream.readUnsignedShort();
return major == Opcodes.V1_6;
}
}
public static void checkFileWithDx(byte[] bytes, @NotNull String relativePath) {
Main.Arguments arguments = new Main.Arguments();
String[] array = new String[1];
array[0] = "testArgs";
arguments.parse(array);
checkFileWithDx(bytes, relativePath, arguments);
}
private static void checkFileWithDx(byte[] bytes, @NotNull String relativePath, @NotNull Main.Arguments arguments) {
DirectClassFile cf = new DirectClassFile(bytes, relativePath, true);
cf.setAttributeFactory(StdAttributeFactory.THE_ONE);
CfTranslator.translate(
cf,
bytes,
arguments.cfOptions,
arguments.dexOptions,
new DexFile(arguments.dexOptions)
);
}
private static String generateExceptionMessage(Throwable e) {
StringWriter writer = new StringWriter();
try (PrintWriter printWriter = new PrintWriter(writer)) {
e.printStackTrace(printWriter);
String stackTrace = writer.toString();
Matcher matcher = STACK_TRACE_PATTERN.matcher(stackTrace);
return matcher.replaceAll("");
}
}
}
@@ -0,0 +1,26 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.codegen
import org.jetbrains.kotlin.load.java.JvmAbi
import java.net.URL
import java.net.URLClassLoader
fun clearReflectionCache(classLoader: ClassLoader) {
try {
val klass = classLoader.loadClass(JvmAbi.REFLECTION_FACTORY_IMPL.asSingleFqName().asString())
val method = klass.getDeclaredMethod("clearCaches")
method.invoke(null)
} catch (e: ClassNotFoundException) {
// This is OK for a test without kotlin-reflect in the dependencies
}
}
fun ClassLoader?.extractUrls(): List<URL> {
return (this as? URLClassLoader)?.let {
it.urLs.toList() + it.parent.extractUrls()
} ?: emptyList()
}