Support test directives in android tests

compiler/android-tests/tests/org/jetbrains/kotlin/android/tests/AndroidTestGenerator.kt

@@ -31,11 +31,14 @@ private val packagePattern = Pattern.compile("(?m)^\\s*package[ |\t]+([\\w|\\.]*
 
 private val importPattern = Pattern.compile("import[ |\t]([\\w|]*\\.)")
 
-internal fun genFiles(file: File, fileContent: String, filesHolder: CodegenTestsOnAndroidGenerator.FilesWriter): FqName? {
+internal fun patchFiles(
-    val testFiles = createTestFiles(file, fileContent)
+    file: File,
-    if (testFiles.filter { it.name.endsWith(".java") }.isNotEmpty()) {
+    testFiles: List<CodegenTestCase.TestFile>,
+    filesHolder: CodegenTestsOnAndroidGenerator.FilesWriter
+): FqName? {
+    if (testFiles.any { it.name.endsWith(".java") }) {
         //TODO support java files
-        return null;
+        return null
     }
     val ktFiles = testFiles.filter { it.name.endsWith(".kt") }
     if (ktFiles.isEmpty()) return null
@@ -82,16 +85,6 @@ internal fun genFiles(file: File, fileContent: String, filesHolder: CodegenTests
     return boxFiles.last().newClassId
 }
 
-
-private fun createTestFiles(file: File, expectedText: String): List<CodegenTestCase.TestFile> {
-    val files = KotlinTestUtils.createTestFiles(file.name, expectedText, object : KotlinTestUtils.TestFileFactoryNoModules<CodegenTestCase.TestFile>() {
-        override fun create(fileName: String, text: String, directives: Map<String, String>): CodegenTestCase.TestFile {
-            return CodegenTestCase.TestFile(fileName, text)
-        }
-    })
-    return files
-}
-
 private fun hasBoxMethod(text: String): Boolean {
     return text.contains("fun box()")
 }

compiler/android-tests/tests/org/jetbrains/kotlin/android/tests/CodegenTestsOnAndroidGenerator.kt

@@ -17,7 +17,6 @@
 package org.jetbrains.kotlin.android.tests
 
 import com.google.common.collect.Lists
-import com.intellij.openapi.Disposable
 import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.FileUtilRt
@@ -26,51 +25,51 @@ import org.jetbrains.kotlin.backend.common.output.OutputFileCollection
 import org.jetbrains.kotlin.cli.common.output.outputUtils.writeAllTo
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.codegen.CodegenTestCase
 import org.jetbrains.kotlin.codegen.CodegenTestFiles
 import org.jetbrains.kotlin.codegen.GenerationUtils
 import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime
 import org.jetbrains.kotlin.codegen.state.GenerationState
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
-import org.jetbrains.kotlin.config.JVMConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.name.NameUtils
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.test.*
-import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase
 import org.jetbrains.kotlin.utils.Printer
 import org.junit.Assert
 import org.junit.Ignore
 import java.io.File
+import java.io.FileWriter
 import java.io.IOException
 import java.util.*
 
-@Ignore
+data class ConfigurationKey(val kind: ConfigurationKind, val jdkKind: TestJdkKind, val configuration: String)
-class CodegenTestsOnAndroidGenerator private constructor(private val pathManager: PathManager) : KtUsefulTestCase() {
-    private var WRITED_FILES_COUNT = 0
 
-    private var MODULE_INDEX = 1
+@Ignore
+class CodegenTestsOnAndroidGenerator private constructor(private val pathManager: PathManager) : CodegenTestCase() {
+    private var writtenFilesCount = 0
+
+    private var currentModuleIndex = 1
 
     private val generatedTestNames = Lists.newArrayList<String>()
 
-    @Throws(Throwable::class)
     private fun generateOutputFiles() {
         prepareAndroidModule()
         generateAndSave()
     }
 
-    @Throws(IOException::class)
     private fun prepareAndroidModule() {
         println("Copying kotlin-runtime.jar and kotlin-reflect.jar in android module...")
         copyKotlinRuntimeJars()
 
-        println("Check \"libs\" folder in tested android module...")
+        println("Check 'libs' folder in tested android module...")
         val libsFolderInTestedModule = File(pathManager.libsFolderInAndroidTestedModuleTmpFolder)
         if (!libsFolderInTestedModule.exists()) {
             libsFolderInTestedModule.mkdirs()
         }
     }
 
-    @Throws(IOException::class)
     private fun copyKotlinRuntimeJars() {
         FileUtil.copy(
             ForTestCompileRuntime.runtimeJarForTests(),
@@ -87,76 +86,52 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
         )
     }
 
-    @Throws(Throwable::class)
     private fun generateAndSave() {
         println("Generating test files...")
-        val out = StringBuilder()
-        val p = Printer(out)
-
-        p.print(FileUtil.loadFile(File("license/LICENSE.txt")))
-        p.println("package $testClassPackage;")
-        p.println()
-        p.println("import ", baseTestClassPackage, ".", baseTestClassName, ";")
-        p.println()
-        p.println("/* This class is generated by $generatorName. DO NOT MODIFY MANUALLY */")
-        p.println("public class ", testClassName, " extends ", baseTestClassName, " {")
-        p.pushIndent()
-
-        generateTestMethodsForDirectories(p, File("compiler/testData/codegen/box"), File("compiler/testData/codegen/boxInline"))
-
-        p.popIndent()
-        p.println("}")
-
         val testSourceFilePath =
             pathManager.srcFolderInAndroidTmpFolder + "/" + testClassPackage.replace(".", "/") + "/" + testClassName + ".java"
-        FileUtil.writeToFile(File(testSourceFilePath), out.toString())
+
+        FileWriter(File(testSourceFilePath)).use {
+            val p = Printer(it)
+            p.print(FileUtil.loadFile(File("license/LICENSE.txt")))
+            p.println(
+                """package $testClassPackage;
+                |
+                |import $baseTestClassPackage.$baseTestClassName;
+                |
+                |/* This class is generated by $generatorName. DO NOT MODIFY MANUALLY */
+                |public class $testClassName extends $baseTestClassName {
+                |
+            """.trimMargin()
+            )
+            p.pushIndent()
+
+            generateTestMethodsForDirectories(p, File("compiler/testData/codegen/box"), File("compiler/testData/codegen/boxInline"))
+
+            p.popIndent()
+            p.println("}")
+        }
     }
 
-    @Throws(IOException::class)
     private fun generateTestMethodsForDirectories(p: Printer, vararg dirs: File) {
-        val holderMock = FilesWriter(false, false)
+        val holders = mutableMapOf<ConfigurationKey, FilesWriter>()
-        val holderFull = FilesWriter(true, false)
-        val holderInheritMFP = FilesWriter(true, true)
 
         for (dir in dirs) {
-            val files = dir.listFiles()
+            val files = dir.listFiles() ?: error("Folder with testData is empty: ${dir.absolutePath}")
-            Assert.assertNotNull("Folder with testData is empty: " + dir.absolutePath, files)
+            processFiles(p, files, holders)
-            processFiles(p, files!!, holderFull, holderMock, holderInheritMFP)
         }
 
-        holderFull.writeFilesOnDisk()
+        holders.values.forEach {
-        holderMock.writeFilesOnDisk()
+            it.writeFilesOnDisk()
-        holderInheritMFP.writeFilesOnDisk()
+        }
     }
 
-    internal inner class FilesWriter constructor(
+    internal inner class FilesWriter(
-        private val isFullJdkAndRuntime: Boolean,
+        private val configuration: CompilerConfiguration
-        private val inheritMultifileParts: Boolean
     ) {
+        private val rawFiles: MutableList<Pair<String, String>> = ArrayList()
 
-        var files: MutableList<KtFile> = ArrayList()
+        private fun shouldWriteFilesOnDisk(): Boolean = rawFiles.size > 300
-        private var environment: KotlinCoreEnvironment? = null
-        private var disposable: Disposable? = null
-
-        init {
-            this.disposable = TestDisposable()
-            this.environment = createEnvironment(isFullJdkAndRuntime, disposable!!)
-        }
-
-        private fun createEnvironment(isFullJdkAndRuntime: Boolean, disposable: Disposable): KotlinCoreEnvironment {
-            val configurationKind = if (isFullJdkAndRuntime) ConfigurationKind.ALL else ConfigurationKind.NO_KOTLIN_REFLECT
-            val testJdkKind = if (isFullJdkAndRuntime) TestJdkKind.FULL_JDK else TestJdkKind.MOCK_JDK
-            val configuration = KotlinTestUtils.newConfiguration(configurationKind, testJdkKind, KotlinTestUtils.getAnnotationsJar())
-            configuration.put(CommonConfigurationKeys.MODULE_NAME, "android-module-" + MODULE_INDEX++)
-            if (inheritMultifileParts) {
-                configuration.put(JVMConfigurationKeys.INHERIT_MULTIFILE_PARTS, true)
-            }
-            return KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
-        }
-
-        fun shouldWriteFilesOnDisk(): Boolean {
-            return files.size > 300
-        }
 
         fun writeFilesOnDiskIfNeeded() {
             if (shouldWriteFilesOnDisk()) {
@@ -165,50 +140,38 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
         }
 
         fun writeFilesOnDisk() {
-            writeFiles(files)
+            val disposable = TestDisposable()
-            files = ArrayList()
+
-            if (disposable != null) {
+            val environment = KotlinCoreEnvironment.createForTests(
-                Disposer.dispose(disposable!!)
+                disposable,
-                disposable = TestDisposable()
+                configuration.copy().apply { put(CommonConfigurationKeys.MODULE_NAME, "android-module-" + currentModuleIndex++) },
-            }
+                EnvironmentConfigFiles.JVM_CONFIG_FILES
-            environment = createEnvironment(isFullJdkAndRuntime, disposable!!)
+            )
+
+            writeFiles(
+                rawFiles.map {
+                    CodegenTestFiles.create(it.first, it.second, environment.project).psiFile
+                }, environment
+            )
+            Disposer.dispose(disposable)
+            rawFiles.clear()
         }
 
         fun addFile(name: String, content: String) {
-            try {
+            rawFiles.add(name to content)
-                files.add(CodegenTestFiles.create(name, content, environment!!.project).psiFile)
-            } catch (e: Throwable) {
-                throw RuntimeException("Problem during creating file $name: \n$content", e)
-            }
-
         }
 
-        private fun writeFiles(filesToCompile: List<KtFile>) {
+        private fun writeFiles(filesToCompile: List<KtFile>, environment: KotlinCoreEnvironment) {
             if (filesToCompile.isEmpty()) return
 
             //1000 files per folder, each folder would be jared by build.gradle script
             // We can't create one big jar with all test cause dex has problem with memory on teamcity
-            WRITED_FILES_COUNT += filesToCompile.size
+            writtenFilesCount += filesToCompile.size
-            val outputDir = File(pathManager.getOutputForCompiledFiles(WRITED_FILES_COUNT / 1000))
+            val outputDir = File(pathManager.getOutputForCompiledFiles(writtenFilesCount / 1000))
 
-            println(
+            println("Generating ${filesToCompile.size} files into ${outputDir.name}, configuration: '${environment.configuration}'...")
-                "Generating " + filesToCompile.size + " files" +
+
-                        (if (inheritMultifileParts)
+            val outputFiles = GenerationUtils.compileFiles(filesToCompile, environment).run { destroy(); factory }
-                            " (JVM.INHERIT_MULTIFILE_PARTS)"
-                        else if (isFullJdkAndRuntime) " (full jdk and runtime)" else "") + " into " + outputDir.name + "..."
-            )
-            val outputFiles: OutputFileCollection
-            var state: GenerationState? = null
-            try {
-                state = GenerationUtils.compileFiles(filesToCompile, environment!!)
-                outputFiles = state.factory
-            } catch (e: Throwable) {
-                throw RuntimeException(e)
-            } finally {
-                if (state != null) {
-                    state.destroy()
-                }
-            }
 
             if (!outputDir.exists()) {
                 outputDir.mkdirs()
@@ -223,13 +186,11 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
     private fun processFiles(
         printer: Printer,
         files: Array<File>,
-        holderFull: FilesWriter,
+        holders: MutableMap<ConfigurationKey, FilesWriter>
-        holderMock: FilesWriter,
-        holderInheritMFP: FilesWriter
     ) {
-        holderFull.writeFilesOnDiskIfNeeded()
+        holders.values.forEach {
-        holderMock.writeFilesOnDiskIfNeeded()
+            it.writeFilesOnDiskIfNeeded()
-        holderInheritMFP.writeFilesOnDiskIfNeeded()
+        }
 
         for (file in files) {
             if (SpecialFiles.getExcludedFiles().contains(file.name)) {
@@ -238,34 +199,35 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
             if (file.isDirectory) {
                 val listFiles = file.listFiles()
                 if (listFiles != null) {
-                    processFiles(printer, listFiles, holderFull, holderMock, holderInheritMFP)
+                    processFiles(printer, listFiles, holders)
                 }
-            } else if (FileUtilRt.getExtension(file.name) != KotlinFileType.INSTANCE.defaultExtension) {
+            } else if (FileUtilRt.getExtension(file.name) != KotlinFileType.EXTENSION) {
                 // skip non kotlin files
             } else {
-                val fullFileText = FileUtil.loadFile(file, true)
-
                 if (!InTextDirectivesUtils.isPassingTarget(TargetBackend.JVM, file)) {
                     continue
                 }
 
-                //TODO: support LANGUAGE_VERSION
+                val fullFileText = FileUtil.loadFile(file, true)
-                if (InTextDirectivesUtils.isDirectiveDefined(fullFileText, "LANGUAGE_VERSION:")) {
+                //TODO support JvmPackageName
-                    continue
+                if (fullFileText.contains("@file:JvmPackageName(")) continue
-                }
 
-                //TODO: support multifile facades
-                //TODO: support multifile facades hierarchies
                 if (hasBoxMethod(fullFileText)) {
-                    var filesHolder = if (InTextDirectivesUtils.isDirectiveDefined(fullFileText, "FULL_JDK") ||
+                    val testFiles = createTestFiles(file, fullFileText)
-                        InTextDirectivesUtils.isDirectiveDefined(fullFileText, "WITH_RUNTIME") ||
+                    val kind = extractConfigurationKind(testFiles)
-                        InTextDirectivesUtils.isDirectiveDefined(fullFileText, "WITH_REFLECT"))
+                    val jdkKind = getJdkKind(testFiles)
-                        holderFull
+                    val keyConfiguration = CompilerConfiguration()
-                    else
+                    updateConfigurationByDirectivesInTestFiles(testFiles, keyConfiguration)
-                        holderMock
-                    filesHolder = if (fullFileText.contains("+JVM.INHERIT_MULTIFILE_PARTS")) holderInheritMFP else filesHolder
 
-                    val classWithBoxMethod = genFiles(file, fullFileText, filesHolder) ?: continue
+                    val key = ConfigurationKey(kind, jdkKind, keyConfiguration.toString())
+                    val filesHolder = holders.getOrPut(key) {
+                        FilesWriter(KotlinTestUtils.newConfiguration(kind, jdkKind, KotlinTestUtils.getAnnotationsJar()).apply {
+                            println("Creating new configuration by $key")
+                            updateConfigurationByDirectivesInTestFiles(testFiles, this)
+                        })
+                    }
+
+                    val classWithBoxMethod = patchFiles(file, testFiles, filesHolder) ?: continue
 
                     val generatedTestName = generateTestName(file.name)
                     generateTestMethod(
@@ -279,6 +241,17 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
         }
     }
 
+    private fun createTestFiles(file: File, expectedText: String): List<CodegenTestCase.TestFile> =
+        KotlinTestUtils.createTestFiles(
+            file.name,
+            expectedText,
+            object : KotlinTestUtils.TestFileFactoryNoModules<CodegenTestCase.TestFile>() {
+                override fun create(fileName: String, text: String, directives: Map<String, String>): CodegenTestCase.TestFile {
+                    return CodegenTestCase.TestFile(fileName, text)
+                }
+            })
+
+
     private fun generateTestName(fileName: String): String {
         var result = NameUtils.sanitizeAsJavaIdentifier(FileUtil.getNameWithoutExtension(StringUtil.capitalize(fileName)))
 

compiler/android-tests/tests/org/jetbrains/kotlin/android/tests/SpecialFiles.java

@@ -48,6 +48,7 @@ public class SpecialFiles {
         excludedFiles.add("enumKClassAnnotation.kt");
         excludedFiles.add("primitivesAndArrays.kt");
         excludedFiles.add("getDelegateWithoutReflection.kt");
+        excludedFiles.add("parameterAnnotationInDefaultImpls.kt");
 
         // Reflection is used to check full class name
         excludedFiles.add("native");
@@ -127,9 +128,8 @@ public class SpecialFiles {
         //wrong function resolution after package renaming
         excludedFiles.add("apiVersionAtLeast1.kt");
 
-        //special flags
+        //special symbols in names
-        excludedFiles.add("inlineFunInConstructorCallWithEnabledNormalization.kt");
+        excludedFiles.add("nameWithWhitespace.kt");
-        excludedFiles.add("kt9532_lv10.kt");
     }
 
     private SpecialFiles() {

compiler/frontend/src/org/jetbrains/kotlin/config/CompilerConfiguration.java

@@ -26,7 +26,7 @@ import java.util.*;
 public class CompilerConfiguration {
     public static CompilerConfiguration EMPTY = new CompilerConfiguration();
 
-    private final Map<Key, Object> map = new HashMap<>();
+    private final Map<Key, Object> map = new LinkedHashMap<>();
     private boolean readOnly = false;
 
     static {

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java

@@ -159,7 +159,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         return configuration;
     }
 
-    private static void updateConfigurationByDirectivesInTestFiles(
+    protected static void updateConfigurationByDirectivesInTestFiles(
             @NotNull List<TestFile> testFilesWithConfigurationDirectives,
             @NotNull CompilerConfiguration configuration
     ) {