Support test directives in android tests

2018-01-17 11:33:16 +01:00
parent ef1a3ec32d
commit 7f1cc81d39
5 changed files with 109 additions and 143 deletions
@@ -31,11 +31,14 @@ private val packagePattern = Pattern.compile("(?m)^\\s*package[ |\t]+([\\w|\\.]*

 private val importPattern = Pattern.compile("import[ |\t]([\\w|]*\\.)")

-internal fun genFiles(file: File, fileContent: String, filesHolder: CodegenTestsOnAndroidGenerator.FilesWriter): FqName? {
-    val testFiles = createTestFiles(file, fileContent)
-    if (testFiles.filter { it.name.endsWith(".java") }.isNotEmpty()) {
+internal fun patchFiles(
+    file: File,
+    testFiles: List<CodegenTestCase.TestFile>,
+    filesHolder: CodegenTestsOnAndroidGenerator.FilesWriter
+): FqName? {
+    if (testFiles.any { it.name.endsWith(".java") }) {
        //TODO support java files
-        return null;
+        return null
    }
    val ktFiles = testFiles.filter { it.name.endsWith(".kt") }
    if (ktFiles.isEmpty()) return null
@@ -82,16 +85,6 @@ internal fun genFiles(file: File, fileContent: String, filesHolder: CodegenTests
    return boxFiles.last().newClassId
 }

-
-private fun createTestFiles(file: File, expectedText: String): List<CodegenTestCase.TestFile> {
-    val files = KotlinTestUtils.createTestFiles(file.name, expectedText, object : KotlinTestUtils.TestFileFactoryNoModules<CodegenTestCase.TestFile>() {
-        override fun create(fileName: String, text: String, directives: Map<String, String>): CodegenTestCase.TestFile {
-            return CodegenTestCase.TestFile(fileName, text)
-        }
-    })
-    return files
-}
-
 private fun hasBoxMethod(text: String): Boolean {
    return text.contains("fun box()")
 }
@@ -17,7 +17,6 @@
 package org.jetbrains.kotlin.android.tests

 import com.google.common.collect.Lists
-import com.intellij.openapi.Disposable
 import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.FileUtilRt
@@ -26,51 +25,51 @@ import org.jetbrains.kotlin.backend.common.output.OutputFileCollection
 import org.jetbrains.kotlin.cli.common.output.outputUtils.writeAllTo
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.codegen.CodegenTestCase
 import org.jetbrains.kotlin.codegen.CodegenTestFiles
 import org.jetbrains.kotlin.codegen.GenerationUtils
 import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime
 import org.jetbrains.kotlin.codegen.state.GenerationState
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
-import org.jetbrains.kotlin.config.JVMConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.name.NameUtils
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.test.*
-import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase
 import org.jetbrains.kotlin.utils.Printer
 import org.junit.Assert
 import org.junit.Ignore
 import java.io.File
+import java.io.FileWriter
 import java.io.IOException
 import java.util.*

-@Ignore
-class CodegenTestsOnAndroidGenerator private constructor(private val pathManager: PathManager) : KtUsefulTestCase() {
-    private var WRITED_FILES_COUNT = 0
+data class ConfigurationKey(val kind: ConfigurationKind, val jdkKind: TestJdkKind, val configuration: String)

-    private var MODULE_INDEX = 1
+@Ignore
+class CodegenTestsOnAndroidGenerator private constructor(private val pathManager: PathManager) : CodegenTestCase() {
+    private var writtenFilesCount = 0
+
+    private var currentModuleIndex = 1

    private val generatedTestNames = Lists.newArrayList<String>()

-    @Throws(Throwable::class)
    private fun generateOutputFiles() {
        prepareAndroidModule()
        generateAndSave()
    }

-    @Throws(IOException::class)
    private fun prepareAndroidModule() {
        println("Copying kotlin-runtime.jar and kotlin-reflect.jar in android module...")
        copyKotlinRuntimeJars()

-        println("Check \"libs\" folder in tested android module...")
+        println("Check 'libs' folder in tested android module...")
        val libsFolderInTestedModule = File(pathManager.libsFolderInAndroidTestedModuleTmpFolder)
        if (!libsFolderInTestedModule.exists()) {
            libsFolderInTestedModule.mkdirs()
        }
    }

-    @Throws(IOException::class)
    private fun copyKotlinRuntimeJars() {
        FileUtil.copy(
            ForTestCompileRuntime.runtimeJarForTests(),
@@ -87,76 +86,52 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
        )
    }

-    @Throws(Throwable::class)
    private fun generateAndSave() {
        println("Generating test files...")
-        val out = StringBuilder()
-        val p = Printer(out)
-
-        p.print(FileUtil.loadFile(File("license/LICENSE.txt")))
-        p.println("package $testClassPackage;")
-        p.println()
-        p.println("import ", baseTestClassPackage, ".", baseTestClassName, ";")
-        p.println()
-        p.println("/* This class is generated by $generatorName. DO NOT MODIFY MANUALLY */")
-        p.println("public class ", testClassName, " extends ", baseTestClassName, " {")
-        p.pushIndent()
-
-        generateTestMethodsForDirectories(p, File("compiler/testData/codegen/box"), File("compiler/testData/codegen/boxInline"))
-
-        p.popIndent()
-        p.println("}")
-
        val testSourceFilePath =
            pathManager.srcFolderInAndroidTmpFolder + "/" + testClassPackage.replace(".", "/") + "/" + testClassName + ".java"
-        FileUtil.writeToFile(File(testSourceFilePath), out.toString())
+
+        FileWriter(File(testSourceFilePath)).use {
+            val p = Printer(it)
+            p.print(FileUtil.loadFile(File("license/LICENSE.txt")))
+            p.println(
+                """package $testClassPackage;
+                |
+                |import $baseTestClassPackage.$baseTestClassName;
+                |
+                |/* This class is generated by $generatorName. DO NOT MODIFY MANUALLY */
+                |public class $testClassName extends $baseTestClassName {
+                |
+            """.trimMargin()
+            )
+            p.pushIndent()
+
+            generateTestMethodsForDirectories(p, File("compiler/testData/codegen/box"), File("compiler/testData/codegen/boxInline"))
+
+            p.popIndent()
+            p.println("}")
+        }
    }

-    @Throws(IOException::class)
    private fun generateTestMethodsForDirectories(p: Printer, vararg dirs: File) {
-        val holderMock = FilesWriter(false, false)
-        val holderFull = FilesWriter(true, false)
-        val holderInheritMFP = FilesWriter(true, true)
+        val holders = mutableMapOf<ConfigurationKey, FilesWriter>()

        for (dir in dirs) {
-            val files = dir.listFiles()
-            Assert.assertNotNull("Folder with testData is empty: " + dir.absolutePath, files)
-            processFiles(p, files!!, holderFull, holderMock, holderInheritMFP)
+            val files = dir.listFiles() ?: error("Folder with testData is empty: ${dir.absolutePath}")
+            processFiles(p, files, holders)
        }

-        holderFull.writeFilesOnDisk()
-        holderMock.writeFilesOnDisk()
-        holderInheritMFP.writeFilesOnDisk()
+        holders.values.forEach {
+            it.writeFilesOnDisk()
+        }
    }

-    internal inner class FilesWriter constructor(
-        private val isFullJdkAndRuntime: Boolean,
-        private val inheritMultifileParts: Boolean
+    internal inner class FilesWriter(
+        private val configuration: CompilerConfiguration
    ) {
+        private val rawFiles: MutableList<Pair<String, String>> = ArrayList()

-        var files: MutableList<KtFile> = ArrayList()
-        private var environment: KotlinCoreEnvironment? = null
-        private var disposable: Disposable? = null
-
-        init {
-            this.disposable = TestDisposable()
-            this.environment = createEnvironment(isFullJdkAndRuntime, disposable!!)
-        }
-
-        private fun createEnvironment(isFullJdkAndRuntime: Boolean, disposable: Disposable): KotlinCoreEnvironment {
-            val configurationKind = if (isFullJdkAndRuntime) ConfigurationKind.ALL else ConfigurationKind.NO_KOTLIN_REFLECT
-            val testJdkKind = if (isFullJdkAndRuntime) TestJdkKind.FULL_JDK else TestJdkKind.MOCK_JDK
-            val configuration = KotlinTestUtils.newConfiguration(configurationKind, testJdkKind, KotlinTestUtils.getAnnotationsJar())
-            configuration.put(CommonConfigurationKeys.MODULE_NAME, "android-module-" + MODULE_INDEX++)
-            if (inheritMultifileParts) {
-                configuration.put(JVMConfigurationKeys.INHERIT_MULTIFILE_PARTS, true)
-            }
-            return KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
-        }
-
-        fun shouldWriteFilesOnDisk(): Boolean {
-            return files.size > 300
-        }
+        private fun shouldWriteFilesOnDisk(): Boolean = rawFiles.size > 300

        fun writeFilesOnDiskIfNeeded() {
            if (shouldWriteFilesOnDisk()) {
@@ -165,50 +140,38 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
        }

        fun writeFilesOnDisk() {
-            writeFiles(files)
-            files = ArrayList()
-            if (disposable != null) {
-                Disposer.dispose(disposable!!)
-                disposable = TestDisposable()
-            }
-            environment = createEnvironment(isFullJdkAndRuntime, disposable!!)
+            val disposable = TestDisposable()
+
+            val environment = KotlinCoreEnvironment.createForTests(
+                disposable,
+                configuration.copy().apply { put(CommonConfigurationKeys.MODULE_NAME, "android-module-" + currentModuleIndex++) },
+                EnvironmentConfigFiles.JVM_CONFIG_FILES
+            )
+
+            writeFiles(
+                rawFiles.map {
+                    CodegenTestFiles.create(it.first, it.second, environment.project).psiFile
+                }, environment
+            )
+            Disposer.dispose(disposable)
+            rawFiles.clear()
        }

        fun addFile(name: String, content: String) {
-            try {
-                files.add(CodegenTestFiles.create(name, content, environment!!.project).psiFile)
-            } catch (e: Throwable) {
-                throw RuntimeException("Problem during creating file $name: \n$content", e)
-            }
-
+            rawFiles.add(name to content)
        }

-        private fun writeFiles(filesToCompile: List<KtFile>) {
+        private fun writeFiles(filesToCompile: List<KtFile>, environment: KotlinCoreEnvironment) {
            if (filesToCompile.isEmpty()) return

            //1000 files per folder, each folder would be jared by build.gradle script
            // We can't create one big jar with all test cause dex has problem with memory on teamcity
-            WRITED_FILES_COUNT += filesToCompile.size
-            val outputDir = File(pathManager.getOutputForCompiledFiles(WRITED_FILES_COUNT / 1000))
+            writtenFilesCount += filesToCompile.size
+            val outputDir = File(pathManager.getOutputForCompiledFiles(writtenFilesCount / 1000))

-            println(
-                "Generating " + filesToCompile.size + " files" +
-                        (if (inheritMultifileParts)
-                            " (JVM.INHERIT_MULTIFILE_PARTS)"
-                        else if (isFullJdkAndRuntime) " (full jdk and runtime)" else "") + " into " + outputDir.name + "..."
-            )
-            val outputFiles: OutputFileCollection
-            var state: GenerationState? = null
-            try {
-                state = GenerationUtils.compileFiles(filesToCompile, environment!!)
-                outputFiles = state.factory
-            } catch (e: Throwable) {
-                throw RuntimeException(e)
-            } finally {
-                if (state != null) {
-                    state.destroy()
-                }
-            }
+            println("Generating ${filesToCompile.size} files into ${outputDir.name}, configuration: '${environment.configuration}'...")
+
+            val outputFiles = GenerationUtils.compileFiles(filesToCompile, environment).run { destroy(); factory }

            if (!outputDir.exists()) {
                outputDir.mkdirs()
@@ -223,13 +186,11 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
    private fun processFiles(
        printer: Printer,
        files: Array<File>,
-        holderFull: FilesWriter,
-        holderMock: FilesWriter,
-        holderInheritMFP: FilesWriter
+        holders: MutableMap<ConfigurationKey, FilesWriter>
    ) {
-        holderFull.writeFilesOnDiskIfNeeded()
-        holderMock.writeFilesOnDiskIfNeeded()
-        holderInheritMFP.writeFilesOnDiskIfNeeded()
+        holders.values.forEach {
+            it.writeFilesOnDiskIfNeeded()
+        }

        for (file in files) {
            if (SpecialFiles.getExcludedFiles().contains(file.name)) {
@@ -238,34 +199,35 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
            if (file.isDirectory) {
                val listFiles = file.listFiles()
                if (listFiles != null) {
-                    processFiles(printer, listFiles, holderFull, holderMock, holderInheritMFP)
+                    processFiles(printer, listFiles, holders)
                }
-            } else if (FileUtilRt.getExtension(file.name) != KotlinFileType.INSTANCE.defaultExtension) {
+            } else if (FileUtilRt.getExtension(file.name) != KotlinFileType.EXTENSION) {
                // skip non kotlin files
            } else {
-                val fullFileText = FileUtil.loadFile(file, true)
-
                if (!InTextDirectivesUtils.isPassingTarget(TargetBackend.JVM, file)) {
                    continue
                }

-                //TODO: support LANGUAGE_VERSION
-                if (InTextDirectivesUtils.isDirectiveDefined(fullFileText, "LANGUAGE_VERSION:")) {
-                    continue
-                }
+                val fullFileText = FileUtil.loadFile(file, true)
+                //TODO support JvmPackageName
+                if (fullFileText.contains("@file:JvmPackageName(")) continue

-                //TODO: support multifile facades
-                //TODO: support multifile facades hierarchies
                if (hasBoxMethod(fullFileText)) {
-                    var filesHolder = if (InTextDirectivesUtils.isDirectiveDefined(fullFileText, "FULL_JDK") ||
-                        InTextDirectivesUtils.isDirectiveDefined(fullFileText, "WITH_RUNTIME") ||
-                        InTextDirectivesUtils.isDirectiveDefined(fullFileText, "WITH_REFLECT"))
-                        holderFull
-                    else
-                        holderMock
-                    filesHolder = if (fullFileText.contains("+JVM.INHERIT_MULTIFILE_PARTS")) holderInheritMFP else filesHolder
+                    val testFiles = createTestFiles(file, fullFileText)
+                    val kind = extractConfigurationKind(testFiles)
+                    val jdkKind = getJdkKind(testFiles)
+                    val keyConfiguration = CompilerConfiguration()
+                    updateConfigurationByDirectivesInTestFiles(testFiles, keyConfiguration)

-                    val classWithBoxMethod = genFiles(file, fullFileText, filesHolder) ?: continue
+                    val key = ConfigurationKey(kind, jdkKind, keyConfiguration.toString())
+                    val filesHolder = holders.getOrPut(key) {
+                        FilesWriter(KotlinTestUtils.newConfiguration(kind, jdkKind, KotlinTestUtils.getAnnotationsJar()).apply {
+                            println("Creating new configuration by $key")
+                            updateConfigurationByDirectivesInTestFiles(testFiles, this)
+                        })
+                    }
+
+                    val classWithBoxMethod = patchFiles(file, testFiles, filesHolder) ?: continue

                    val generatedTestName = generateTestName(file.name)
                    generateTestMethod(
@@ -279,6 +241,17 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
        }
    }

+    private fun createTestFiles(file: File, expectedText: String): List<CodegenTestCase.TestFile> =
+        KotlinTestUtils.createTestFiles(
+            file.name,
+            expectedText,
+            object : KotlinTestUtils.TestFileFactoryNoModules<CodegenTestCase.TestFile>() {
+                override fun create(fileName: String, text: String, directives: Map<String, String>): CodegenTestCase.TestFile {
+                    return CodegenTestCase.TestFile(fileName, text)
+                }
+            })
+
+
    private fun generateTestName(fileName: String): String {
        var result = NameUtils.sanitizeAsJavaIdentifier(FileUtil.getNameWithoutExtension(StringUtil.capitalize(fileName)))

@@ -48,6 +48,7 @@ public class SpecialFiles {
        excludedFiles.add("enumKClassAnnotation.kt");
        excludedFiles.add("primitivesAndArrays.kt");
        excludedFiles.add("getDelegateWithoutReflection.kt");
+        excludedFiles.add("parameterAnnotationInDefaultImpls.kt");

        // Reflection is used to check full class name
        excludedFiles.add("native");
@@ -127,9 +128,8 @@ public class SpecialFiles {
        //wrong function resolution after package renaming
        excludedFiles.add("apiVersionAtLeast1.kt");

-        //special flags
-        excludedFiles.add("inlineFunInConstructorCallWithEnabledNormalization.kt");
-        excludedFiles.add("kt9532_lv10.kt");
+        //special symbols in names
+        excludedFiles.add("nameWithWhitespace.kt");
    }

    private SpecialFiles() {
@@ -26,7 +26,7 @@ import java.util.*;
 public class CompilerConfiguration {
    public static CompilerConfiguration EMPTY = new CompilerConfiguration();

-    private final Map<Key, Object> map = new HashMap<>();
+    private final Map<Key, Object> map = new LinkedHashMap<>();
    private boolean readOnly = false;

    static {
@@ -159,7 +159,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
        return configuration;
    }

-    private static void updateConfigurationByDirectivesInTestFiles(
+    protected static void updateConfigurationByDirectivesInTestFiles(
            @NotNull List<TestFile> testFilesWithConfigurationDirectives,
            @NotNull CompilerConfiguration configuration
    ) {