Extract kotlin-reflect-api module out of kotlin-reflect

This is needed only for faster compilation of the Kotlin project itself
and has no effect on the public artifact
org.jetbrains.kotlin:kotlin-reflect.

The problem this is solving is the rebuild of the project once anything
has been changed in modules in 'core' (even inside function bodies, i.e.
a non-API change). Previously, changes in 'core' led to the compilation
of kotlin-reflect, which led to the rebuild of all modules depending on
kotlin-reflect directly or indirectly (which is almost all modules in
the project) because kotlin-reflect's artifacts are custom-built and the
changes can not be picked up incrementally. But 99.9% of the time the
initial changes in 'core' could not have any effect on the usages of
kotlin-reflect, because classes from those modules are moved to an
internal package in kotlin-reflect and thus are an internal
implementation detail.

Now, changes in 'core' still lead to the compilation of kotlin-reflect
and to the process of building the custom jar. But if a module depends
on kotlin-reflect-api, not kotlin-reflect, then the incremental
difference checker will detect that the module does not have to be
recompiled if there hasn't been any changes to the API of
kotlin-reflect-api. Which means that the module will not be rebuilt on
every change in 'core'.

This commit only introduces the new module. The dependencies
(kotlin-reflect -> kotlin-reflect-api) are replaced in the next commit.

buildSrc/build.gradle.kts

@@ -3,6 +3,7 @@ buildscript {
     val buildSrcKotlinVersion: String by extra(findProperty("buildSrc.kotlin.version")?.toString() ?: embeddedKotlinVersion)
     val buildSrcKotlinRepo: String? by extra(findProperty("buildSrc.kotlin.repo") as String?)
     extra["versions.shadow"] = "2.0.1"
+    extra["versions.protobuf-java"] = "2.6.1"
 
     repositories {
         buildSrcKotlinRepo?.let {
@@ -46,6 +47,7 @@ dependencies {
 //    compile("org.jetbrains.kotlin:kotlin-gradle-plugin:${rootProject.extra["bootstrap_kotlin_version"]}")
     compile("com.github.jengelman.gradle.plugins:shadow:${property("versions.shadow")}")
     compile("org.ow2.asm:asm-all:6.0_BETA")
+    compile("com.google.protobuf:protobuf-java:${property("versions.protobuf-java")}")
 }
 
 samWithReceiver {

buildSrc/src/main/kotlin/KotlinModuleShadowTransformer.kt

@@ -0,0 +1,62 @@
+import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
+import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
+import org.gradle.api.file.FileTreeElement
+import org.gradle.api.logging.Logger
+import org.jetbrains.kotlin.serialization.jvm.JvmPackageTable
+import shadow.org.apache.tools.zip.ZipEntry
+import shadow.org.apache.tools.zip.ZipOutputStream
+import java.io.ByteArrayOutputStream
+import java.io.DataInputStream
+import java.io.DataOutputStream
+
+class KotlinModuleShadowTransformer(private val logger: Logger) : Transformer {
+    private data class Entry(val path: String, val bytes: ByteArray)
+    private val data = mutableListOf<Entry>()
+
+    override fun canTransformResource(element: FileTreeElement): Boolean =
+            element.path.substringAfterLast(".") == KOTLIN_MODULE
+
+    override fun transform(context: TransformerContext) {
+        fun relocate(content: String): String =
+                context.relocators.fold(content) { acc, relocator -> relocator.applyToSourceContent(acc) }
+
+        val input = DataInputStream(context.`is`)
+        val version = IntArray(input.readInt()) { input.readInt() }
+        logger.info("Transforming ${context.path} with version ${version.toList()}")
+
+        val table = JvmPackageTable.PackageTable.parseFrom(context.`is`).toBuilder()
+
+        val newTable = JvmPackageTable.PackageTable.newBuilder().apply {
+            for (packageParts in table.packagePartsList + table.metadataPartsList) {
+                addPackageParts(JvmPackageTable.PackageParts.newBuilder(packageParts).apply {
+                    packageFqName = relocate(packageFqName)
+                })
+            }
+            addAllJvmPackageName(table.jvmPackageNameList.map(::relocate))
+        }
+
+        val baos = ByteArrayOutputStream()
+        val output = DataOutputStream(baos)
+        output.writeInt(version.size)
+        version.forEach(output::writeInt)
+        newTable.build().writeTo(output)
+        output.flush()
+
+        data += Entry(context.path, baos.toByteArray())
+    }
+
+    override fun hasTransformedResource(): Boolean =
+            data.isNotEmpty()
+
+    override fun modifyOutputStream(os: ZipOutputStream) {
+        for ((path, bytes) in data) {
+            os.putNextEntry(ZipEntry(path))
+            os.write(bytes)
+        }
+        data.clear()
+    }
+
+    companion object {
+        const val KOTLIN_MODULE = "kotlin_module"
+    }
+}

generators/src/org/jetbrains/kotlin/generators/protobuf/GenerateProtoBuf.kt

@@ -69,9 +69,12 @@ fun main(args: Array<String>) {
 
         for (protoPath in PROTO_PATHS) {
             execProtoc(protoPath.file, protoPath.outPath)
+            renamePackages(protoPath.file, protoPath.outPath)
             modifyAndExecProtoc(protoPath)
         }
 
+        generateJvmPackageTableToBuildSrc()
+
         println()
         println("Do not forget to run GenerateProtoBufCompare")
     }
@@ -108,8 +111,6 @@ private fun execProtoc(protoPath: String, outPath: String) {
     if (processOutput.stderr.isNotEmpty()) {
         throw AssertionError(processOutput.stderr)
     }
-
-    renamePackages(protoPath, outPath)
 }
 
 private fun renamePackages(protoPath: String, outPath: String) {
@@ -143,7 +144,9 @@ private fun modifyAndExecProtoc(protoPath: ProtoPath) {
     debugProtoFile.writeText(modifyForDebug(protoPath))
     debugProtoFile.deleteOnExit()
 
-    execProtoc(debugProtoFile.path, "build-common/test")
+    val outPath = "build-common/test"
+    execProtoc(debugProtoFile.path, outPath)
+    renamePackages(debugProtoFile.path, outPath)
 }
 
 private fun modifyForDebug(protoPath: ProtoPath): String {
@@ -158,3 +161,7 @@ private fun modifyForDebug(protoPath: ProtoPath): String {
     }
     return text
 }
+
+private fun generateJvmPackageTableToBuildSrc() {
+    execProtoc("core/descriptors.jvm/src/jvm_package_table.proto", "buildSrc/src/main/java")
+}

libraries/commonConfiguration.gradle

@@ -20,6 +20,15 @@ ext.configureJvmProject = { Project project ->
     }
 }
 
+ext.configureJavaOnlyJvm6Project = { Project project ->
+    project.tasks.withType(JavaCompile) {
+        sourceCompatibility = 1.6
+        targetCompatibility = 1.6
+        options.fork = true
+        options.forkOptions.javaHome = file(JDK_16)
+    }
+}
+
 ext.configureJvm6Project = { Project project ->
     project.configure(project) {
         project.ext.jvmTarget = "1.6"
@@ -30,12 +39,7 @@ ext.configureJvm6Project = { Project project ->
             from sourceSets.main.kotlin
         }
 
-        tasks.withType(JavaCompile) {
+        configureJavaOnlyJvm6Project(project)
-            sourceCompatibility = 1.6
-            targetCompatibility = 1.6
-            options.fork = true
-            options.forkOptions.javaHome = file(JDK_16)
-        }
 
         tasks.withType(project.compileKotlin.class) {
             kotlinOptions.jdkHome = JDK_16

libraries/reflect/api/build.gradle

@@ -0,0 +1,32 @@
+description = ''
+
+apply plugin: 'kotlin'
+
+configureJvm6Project(project)
+
+sourceSets {
+    main {
+        java {
+            srcDir "${rootDir}/core/reflection.jvm/src"
+        }
+    }
+}
+
+dependencies {
+    compileOnly project(':kotlin-stdlib')
+    compileOnly project(':core:descriptors')
+    compileOnly project(':core:descriptors.jvm')
+    compileOnly project(':core:deserialization')
+    compileOnly project(':core:descriptors.runtime')
+    compileOnly project(':core:util.runtime')
+}
+
+compileKotlin {
+    kotlinOptions {
+        freeCompilerArgs = ["-version",
+                            "-Xallow-kotlin-package",
+                            "-Xnormalize-constructor-calls=enable",
+                            "-module-name", "kotlin-reflection",
+                            "-Xdump-declarations-to=${buildDir}/reflect-declarations.json"]
+    }
+}

libraries/reflect/build.gradle

@@ -13,12 +13,11 @@ buildscript {
 }
 
 apply plugin: 'com.github.johnrengelman.shadow'
-apply plugin: 'kotlin'
+apply plugin: 'java'
 
-configureJvm6Project(project)
+configureJavaOnlyJvm6Project(project)
 configurePublishing(project)
 
-
 def core = "${rootDir}/core"
 def annotationsSrc = "${buildDir}/annotations"
 def relocatedCoreSrc = "${buildDir}/core-relocated"
@@ -27,7 +26,6 @@ sourceSets {
     main {
         java {
             srcDir annotationsSrc
-            srcDir "${core}/reflection.jvm/src"
         }
     }
 }
@@ -40,7 +38,11 @@ configurations {
 }
 
 dependencies {
+    compile project(':kotlin-stdlib')
+
     proguardDeps project(':kotlin-stdlib')
+
+    shadows project(':kotlin-reflect-api')
     shadows project(':core:descriptors')
     shadows project(':core:descriptors.jvm')
     shadows project(':core:deserialization')
@@ -48,9 +50,6 @@ dependencies {
     shadows project(':core:util.runtime')
     shadows 'javax.inject:javax.inject:1'
     shadows project(path: ':custom-dependencies:protobuf-lite', configuration: 'default')
-
-    compile project(':kotlin-stdlib')
-    compileOnly project(path: ':custom-dependencies:protobuf-lite', configuration: 'default')
 }
 
 task copyAnnotations(type: Sync) {
@@ -69,21 +68,6 @@ compileJava {
     // options.compilerArgs.addAll(["-Xlint:unchecked"])
 }
 
-compileKotlin {
-    dependsOn copyAnnotations
-    dependsOn ':custom-dependencies:protobuf-lite:prepare'
-    kotlinOptions {
-        freeCompilerArgs = ["-version",
-                            "-Xallow-kotlin-package",
-                            "-Xnormalize-constructor-calls=enable",
-                            "-module-name", "kotlin-reflection",
-                            "-Xdump-declarations-to=${buildDir}/reflect-declarations.json"]
-    }
-}
-
-kotlin.experimental.coroutines "enable"
-
-
 task reflectShadowJar(type: ShadowJar) {
     classifier = 'shadow'
     version = null
@@ -97,6 +81,8 @@ task reflectShadowJar(type: ShadowJar) {
         include 'META-INF/services/**'
     }
 
+    transform(new KotlinModuleShadowTransformer(logger))
+
     configurations = [project.configurations.shadows]
     relocate 'org.jetbrains.kotlin', 'kotlin.reflect.jvm.internal.impl'
     relocate 'javax.inject', 'kotlin.reflect.jvm.internal.impl.javax.inject'

libraries/tools/binary-compatibility-validator/src/test/kotlin/org.jetbrains.kotlin.tools.tests/RuntimePublicAPITest.kt

@@ -58,7 +58,7 @@ class RuntimePublicAPITest {
     }
 
     @Test fun kotlinReflect() {
-        snapshotAPIAndCompare("../../reflect/build/libs", "kotlin-reflect(?!-[-a-z]+)", listOf("../reflect-declarations.json"), nonPublicPackages = listOf("kotlin.reflect.jvm.internal"))
+        snapshotAPIAndCompare("../../reflect/api/build/libs", "kotlin-reflect-api(?!-[-a-z]+)", listOf("../reflect-declarations.json"), nonPublicPackages = listOf("kotlin.reflect.jvm.internal"))
     }
 
 

settings.gradle

@@ -117,6 +117,7 @@ include ":kotlin-build-common",
         ":kotlin-compiler-embeddable",
         ":kotlin-compiler-client-embeddable",
         ":kotlin-reflect",
+        ":kotlin-reflect-api",
         ":kotlin-ant",
         ":compiler:tests-java8",
         ":compiler:tests-ir-jvm",
@@ -171,6 +172,7 @@ project(':kotlin-stdlib-jdk7').projectDir = "$rootDir/libraries/stdlib/jdk7" as
 project(':kotlin-stdlib-jdk8').projectDir = "$rootDir/libraries/stdlib/jdk8" as File
 project(':kotlin-stdlib:samples').projectDir = "$rootDir/libraries/stdlib/samples" as File
 project(':kotlin-reflect').projectDir = "$rootDir/libraries/reflect" as File
+project(':kotlin-reflect-api').projectDir = "$rootDir/libraries/reflect/api" as File
 project(':kotlin-compiler').projectDir = "$rootDir/prepare/compiler" as File
 project(':kotlin-compiler-embeddable').projectDir = "$rootDir/prepare/compiler-embeddable" as File
 project(':kotlin-compiler-client-embeddable').projectDir = "$rootDir/prepare/compiler-client-embeddable" as File