Fix cooperative development setup for kt-master

build.gradle.kts

@@ -397,8 +397,14 @@ allprojects {
     val mirrorRepo: String? = findProperty("maven.repository.mirror")?.toString()
 
     repositories {
-        if (kotlinBuildProperties.getOrNull("attachedIntellijVersion") != null) {
+        when(kotlinBuildProperties.getOrNull("attachedIntellijVersion")) {
-            kotlinBuildLocalRepo(project)
+             null -> {}
+            "master" -> {
+                maven { setUrl("https://www.jetbrains.com/intellij-repository/snapshots") }
+            }
+            else -> {
+                kotlinBuildLocalRepo(project)
+            }
         }
 
         mirrorRepo?.let(::maven)

buildSrc/prepare-deps/build.gradle.kts

@@ -208,10 +208,17 @@ fun prepareDeps(
     }
 }
 
-if (intellijVersionForIde != null) {
+when(kotlinBuildProperties.getOrNull("attachedIntellijVersion")) {
-    prepareDeps(intellijForIde, intellijCoreForIde, sourcesForIde, jpsStandaloneForIde, intellijVersionForIde)
+    null -> {}
+    "master" -> {} // for intellij/kt-master, intellij maven artifacts are used instead of manual unpacked dependencies
+    else -> {
+        val intellijVersionForIde = intellijVersionForIde
+            ?: error("intellijVersionForIde should not be null as attachedIntellijVersion is present")
+        prepareDeps(intellijForIde, intellijCoreForIde, sourcesForIde, jpsStandaloneForIde, intellijVersionForIde)
+    }
 }
 
+
 tasks.named<Delete>("clean") {
     delete(customDepsRepoDir)
 }

buildSrc/src/main/kotlin/localDependencies.kt

@@ -74,14 +74,27 @@ fun RepositoryHandler.kotlinBuildLocalRepo(project: Project): IvyArtifactReposit
     }
 }
 
+/* <used only for cooperative development for non kt-master branches> */
 @JvmOverloads
 fun Project.intellijDep(module: String? = null, forIde: Boolean = false) =
     "kotlin.build:${module ?: ideModuleName()}:${ideModuleVersion(forIde)}"
 
+
 fun Project.intellijCoreDep() = "kotlin.build:intellij-core:${rootProject.extra["versions.intellijSdk"]}"
 
 fun Project.intellijPluginDep(plugin: String, forIde: Boolean = false) = intellijDep(plugin, forIde)
 
+/* </used only for cooperative development for non kt-master branches> */
+
+
+/* <used only for cooperative development for kt-master branch> */
+fun Project.intellijMavenDep(subsystem: String, artifact: String) =
+    "com.jetbrains.$subsystem:$artifact:${ideModuleVersion(forIde = true)}"
+
+
+/* </used only for cooperative development for kt-master branch> */
+
+
 fun ModuleDependency.includeJars(vararg names: String, rootProject: Project? = null) {
     names.forEach {
         var baseName = it.removeSuffix(".jar")

generators/ide-iml-to-gradle-generator/build.gradle.kts

@@ -6,6 +6,7 @@ plugins {
 dependencies {
     implementation(kotlinStdlib("jdk8"))
     implementation(jpsModel())
+    implementation("com.jetbrains.intellij.platform:util-text-matching:$intellijVersion")
     implementation(jpsModelImpl())
     implementation(jpsModelSerialization())
     implementation(commonDependency("com.google.code.gson:gson"))

generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/GradleDependencyNotation.kt

@@ -5,42 +5,11 @@
 
 package org.jetbrains.kotlin.generators.imltogradle
 
-open class GradleDependencyNotation(val dependencyNotation: String, val dependencyConfiguration: String? = null) {
+sealed class GradleDependencyNotation(val dependencyNotation: String) {
     init {
         require(dependencyNotation.isNotEmpty())
-        require(dependencyConfiguration?.isNotEmpty() ?: true)
     }
 
-    companion object {
+    data class IntellijMavenDepGradleDependencyNotation(val groupId: String, val artifactId: String) :
-        private const val artifactNameSubregex = """([a-zA-Z\-\._1-9]*?)"""
+        GradleDependencyNotation("""intellijMavenDep("$groupId", "$artifactId")""")
-
-        private val libPathToGradleNotationRegex = """^lib\/$artifactNameSubregex\.jar$""".toRegex()
-        private val pluginsPathToGradleNotationRegex = """^plugins\/$artifactNameSubregex\/.*?$""".toRegex()
-        private val jarToGradleNotationRegex = """^$artifactNameSubregex\.jar$""".toRegex()
-
-        fun fromJarPath(jarPath: String): GradleDependencyNotation? {
-            if (jarPath == "lib/cds/classesLogAgent.jar") {
-                return null // TODO remove hack?
-            }
-
-            if (jarPath.contains("intellij-core.jar")) {
-                return IntellijCoreGradleDependencyNotation
-            }
-
-            fun Regex.firstGroup() = matchEntire(jarPath)?.groupValues?.get(1)
-
-            return pluginsPathToGradleNotationRegex.firstGroup()?.let { IntellijPluginDepGradleDependencyNotation(it) }
-                ?: libPathToGradleNotationRegex.firstGroup()?.let { IntellijDepGradleDependencyNotation(it) }
-                ?: jarToGradleNotationRegex.firstGroup()?.let { IntellijDepGradleDependencyNotation(it) }
-                ?: error("Path $jarPath matches none of the regexes")
-        }
-    }
-
-    object IntellijCoreGradleDependencyNotation : GradleDependencyNotation("intellijCoreDep()", null)
-
-    data class IntellijPluginDepGradleDependencyNotation(val pluginName: String) :
-        GradleDependencyNotation("intellijPluginDep(\"$pluginName\", forIde = true)")
-
-    data class IntellijDepGradleDependencyNotation(val jarName: String) :
-        GradleDependencyNotation("intellijDep(forIde = true)", "{ includeJars(\"$jarName\") }")
 }

generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Main.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.generators.imltogradle
 
-import com.google.gson.JsonParser
 import org.jetbrains.jps.model.JpsSimpleElement
 import org.jetbrains.jps.model.java.JavaResourceRootType
 import org.jetbrains.jps.model.java.JavaSourceRootType
@@ -14,11 +13,8 @@ import org.jetbrains.jps.model.library.JpsLibrary
 import org.jetbrains.jps.model.library.JpsMavenRepositoryLibraryDescriptor
 import org.jetbrains.jps.model.library.JpsOrderRootType
 import org.jetbrains.jps.model.module.*
-import org.jetbrains.kotlin.generators.imltogradle.GradleDependencyNotation.IntellijDepGradleDependencyNotation
 import java.io.File
 
-
-private lateinit var intellijModuleNameToGradleDependencyNotationsMapping: Map<String, List<GradleDependencyNotation>>
 private val KOTLIN_REPO_ROOT = File(".").canonicalFile
 val DEFAULT_KOTLIN_SNAPSHOT_VERSION = KOTLIN_REPO_ROOT.resolve("gradle.properties")
     .readProperty("defaultSnapshotVersion")
@@ -26,11 +22,6 @@ val DEFAULT_KOTLIN_SNAPSHOT_VERSION = KOTLIN_REPO_ROOT.resolve("gradle.propertie
 private val INTELLIJ_REPO_ROOT = KOTLIN_REPO_ROOT.resolve("intellij").resolve("community").takeIf { it.exists() }
     ?: KOTLIN_REPO_ROOT.resolve("intellij")
 
-private val intellijModuleNameToGradleDependencyNotationsMappingManual: List<Pair<String, GradleDependencyNotation>> = listOf(
-    "intellij.platform.jps.build" to GradleDependencyNotation("jpsBuildTest()"),
-    "intellij.platform.structuralSearch" to IntellijDepGradleDependencyNotation("structuralsearch") // for some reason it's absent in json mapping
-)
-
 // These modules are used in Kotlin plugin and IDEA doesn't publish artifact of these modules
 private val intellijModulesForWhichGenerateBuildGradle = listOf(
     "intellij.platform.debugger.testFramework",
@@ -41,13 +32,20 @@ private val intellijModulesForWhichGenerateBuildGradle = listOf(
     "intellij.java.compiler.tests",
     "intellij.gradle.toolingExtension.tests",
     "intellij.maven",
+    "intellij.gradle.java",
+    "intellij.gradle.jps",
+    "intellij.relaxng",
+    "intellij.jvm.analysis.kotlin.tests",
+    "intellij.jvm.analysis.testFramework",
+    "intellij.platform.configurationStore.tests",
+    "intellij.statsCollector.tests",
+    "intellij.groovy.uast.tests",
 )
 
-val jsonUrlPrefixes = mapOf(
+private val intellijModulesToIgnore = listOf(
-    "202" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform202_IntellijArtifactMappings/113235432:id",
+    "kotlin.util.compiler-dependencies",
-    "203" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform203_IntellijArtifactMappings/117989041:id",
+    "intellij.gradle.java.tests",
-    "211" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform211_IntellijArtifactMappings/121258191:id",
+    "intellij.grazie.tests",
-    "212" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform211_IntellijArtifactMappings/131509697:id",
 )
 
 fun main() {
@@ -64,35 +62,6 @@ fun main() {
         }
         .toList()
 
-    val ideaMajorVersion = KOTLIN_REPO_ROOT.resolve("local.properties").readProperty("attachedIntellijVersion")
-
-    intellijModuleNameToGradleDependencyNotationsMapping = fetchJsonsFromBuildserver(ideaMajorVersion)
-        .flatMap { jsonStr ->
-            JsonParser.parseString(jsonStr).asJsonArray.mapNotNull { jsonElement ->
-                val jsonObject = jsonElement.asJsonObject
-                val moduleName = jsonObject.get("moduleName")?.asString ?: return@mapNotNull null
-                val jarPath = jsonObject.get("path")?.asString ?: return@mapNotNull null
-                moduleName to jarPath
-            }
-        }
-        .filter { (_, jarPath) -> !jarPath.contains("DatabaseTools") && !jarPath.contains("lib/openapi.jar") }
-        .groupBy(
-            keySelector = { (_, jarPath) -> jarPath },
-            valueTransform = { (moduleName, _) -> moduleName }
-        )
-        .filter { (_, moduleNames) ->
-            moduleNames.all { it in ijCommunityModuleNameToJpsModuleMapping }  // filter out ultimate jars
-        }
-        .flatMap { (jarPath, moduleNames) -> moduleNames.map { it to jarPath } }
-        .mapNotNull { (moduleName, jarPath) ->
-            moduleName to (GradleDependencyNotation.fromJarPath(jarPath) ?: return@mapNotNull null)
-        }
-        .plus(intellijModuleNameToGradleDependencyNotationsMappingManual)
-        .groupBy(
-            keySelector = { (moduleName, _) -> moduleName },
-            valueTransform = { (_, dependencyNotation) -> dependencyNotation }
-        )
-
     val imlsInSameDirectory: List<List<File>> = imlFiles.groupBy { it.parentFile }.filter { it.value.size > 1 }.map { it.value }
     if (imlsInSameDirectory.isNotEmpty()) {
         val report = imlsInSameDirectory.joinToString("\n") { "In same directory: " + it.joinToString() }
@@ -103,7 +72,7 @@ fun main() {
         .mapNotNull { imlFile ->
             ijCommunityModuleNameToJpsModuleMapping[imlFile.nameWithoutExtension]?.let { imlFile to it }
         }
-        .filter { (_, jpsModule) -> jpsModule.name != "kotlin.util.compiler-dependencies" }
+        .filter { (_, jpsModule) -> jpsModule.name !in intellijModulesToIgnore }
         .forEach { (imlFile, jpsModule) ->
             println("Processing iml ${imlFile}")
             imlFile.parentFile.resolve("build.gradle.kts").writeText(convertJpsModule(imlFile, jpsModule))
@@ -168,14 +137,22 @@ fun convertJpsLibrary(lib: JpsLibrary, scope: JpsJavaDependencyScope, exported:
 fun convertIntellijDependencyNotFollowingTransitive(dep: JpsDependencyDescriptor, exported: Boolean): List<JpsLikeDependency> {
     return when (val moduleOrLibrary = dep.moduleOrLibrary) {
         is Either.First -> {
-            val moduleName = moduleOrLibrary.value.name
+            when (val moduleName = moduleOrLibrary.value.name) {
-            if (moduleName in intellijModulesForWhichGenerateBuildGradle) {
+                in intellijModulesForWhichGenerateBuildGradle -> {
-                listOf(JpsLikeModuleDependency(":kotlin-ide.$moduleName", dep.scope, exported))
+                    listOf(JpsLikeModuleDependency(":kotlin-ide.$moduleName", dep.scope, exported))
-            } else {
+                }
-                intellijModuleNameToGradleDependencyNotationsMapping[moduleName]
+                in intellijModulesToIgnore -> emptyList()
-                    .also { if (it == null) println("WARNING: Cannot find GradleDependencyNotation for $moduleName") }
+                else -> {
-                    ?.map { JpsLikeJarDependency(it.dependencyNotation, dep.scope, it.dependencyConfiguration, exported) }
+                    val (groupId, artifactId) = MavenArtifactsBuilder.generateMavenCoordinates(moduleName)
-                    ?: emptyList()
+                    listOf(
+                        JpsLikeJarDependency(
+                            GradleDependencyNotation.IntellijMavenDepGradleDependencyNotation(groupId, artifactId).dependencyNotation,
+                            dep.scope,
+                            dependencyConfiguration = null,
+                            exported
+                        )
+                    )
+                }
             }
         }
         is Either.Second -> convertJpsLibrary(moduleOrLibrary.value, dep.scope, exported)

generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/MavenArtifactsBuilder.kt

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.generators.imltogradle
+
+import com.intellij.util.text.NameUtilCore
+import java.util.*
+
+// Copied from intellij build scripts
+object MavenArtifactsBuilder {
+    fun generateMavenCoordinates(moduleName: String): MavenArtifact {
+        val names = moduleName.split(".")
+        if (names.size < 2) {
+            error("Cannot generate Maven artifacts: incorrect module name '${moduleName}'")
+        }
+        val groupId = names.take(2).joinToString(separator = ".")
+        val firstMeaningful = if (names.size > 2 && COMMON_GROUP_NAMES.contains(names[1])) 2 else 1
+        val artifactId = names.drop(firstMeaningful).flatMap {
+            splitByCamelHumpsMergingNumbers(it).map { it.lowercase(Locale.US) }
+        }.joinToString(separator = "-")
+        return MavenArtifact(groupId, artifactId)
+    }
+
+    private fun splitByCamelHumpsMergingNumbers(s: String): List<String> {
+        val words: List<String> = NameUtilCore.splitNameIntoWords(s).toList()
+
+        val result = ArrayList<String>()
+        var i = 0
+        while (i < words.size) {
+            val next: String
+            if (i < words.size - 1 && Character.isDigit(words[i + 1][0])) {
+                next = words[i] + words[i + 1]
+                i++
+            } else {
+                next = words[i]
+            }
+            result += next
+            i++
+        }
+        return result
+    }
+
+    private val COMMON_GROUP_NAMES = setOf("platform", "vcs", "tools", "clouds")
+}
+
+data class MavenArtifact(
+    val groupId: String,
+    val artifactId: String
+)

generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Util.kt

@@ -83,22 +83,6 @@ inline fun <T> T?.orElse(block: () -> T): T = this ?: block()
 val JpsModule.dependencies: List<JpsDependencyElement>
     get() = dependenciesList.dependencies.filter { it is JpsModuleDependency || it is JpsLibraryDependency }
 
-fun fetchJsonsFromBuildserver(ideaMajorVersion: String): List<String> {
-    require(ideaMajorVersion.length == 3 && ideaMajorVersion.all { it.isDigit() }) {
-        "attachedIntellijVersion='$ideaMajorVersion' must be 3 length all digit string"
-    }
-    val urlPrefix = jsonUrlPrefixes[ideaMajorVersion] ?: error("'$ideaMajorVersion' platform is absent in mapping")
-    return listOf(
-        "$urlPrefix/ideaIU-project-structure-mapping.json",
-        "$urlPrefix/intellij-core-project-structure-mapping.json"
-    ).map { url ->
-        try {
-            URL(url).readText()
-        } catch (ex: Throwable) {
-            error("Can't access $url. Is VPN on?")
-        }
-    }
-}
 
 fun File.readProperty(propertyName: String): String {
     return inputStream().use { Properties().apply { load(it) }.getProperty(propertyName) }

gradle/versions.properties

@@ -3,6 +3,7 @@ versions.intellijSdk.forIde.202=202.7660.26
 versions.intellijSdk.forIde.203=203.6682.168
 versions.intellijSdk.forIde.211=211.7442.40
 versions.intellijSdk.forIde.212=212.4746.92
+versions.intellijSdk.forIde.master=221.4842-EAP-CANDIDATE-SNAPSHOT
 
 # versions.* should match library versions used in IDEA from versions.intellijSdk
 

settings.gradle

@@ -485,6 +485,45 @@ if (attachedIntellijVersion == "212") { // Latest available platform in scope of
     intellij "jvm-run-configurations/kotlin.jvm-run-configurations.iml"
 }
 
+if (attachedIntellijVersion == "master") {
+    logger.info("Including kotlin-ide modules in settings.gradle")
+    def excludedModules = [
+            "tools/kotlin-maven-artifacts-publishing/intellij.kotlin.util.mavenArtifactsPublishing.iml",
+            "util/project-model-updater/kotlin.util.project-model-updater.iml",
+            "util/compiler-dependencies/kotlin.util.compiler-dependencies.iml",
+            "kotlin-compiler-classpath/kotlin.util.compiler-classpath.iml"
+    ]
+    def modulesIml = new File("${rootDir}/intellij/.idea/modules.xml").text
+    def matcher = modulesIml =~ /filepath="(.+)"/ // TODO use some xml parser instead
+    def modules = (0..<matcher.count)
+            .collect { matcher[it][1] }
+            .findAll { it.startsWithAny('$PROJECT_DIR$/community/plugins/kotlin/', '$PROJECT_DIR$/plugins/kotlin/') }
+            .collect {
+                it.startsWith('$PROJECT_DIR$/community/plugins/kotlin/') ? (it - '$PROJECT_DIR$/community/plugins/kotlin/') :
+                        (it.startsWith('$PROJECT_DIR$/plugins/kotlin/') ? (it - '$PROJECT_DIR$/plugins/kotlin/') : null)
+            }
+    (modules - excludedModules).each { intellij it }
+
+    //// These modules are used in Kotlin plugin and IDEA doesn't publish artifact of these modules
+    intellij "plugins/gradle/intellij.gradle.tests.iml"
+    intellij "plugins/gradle/java/intellij.gradle.java.iml"
+    intellij "plugins/gradle/jps-plugin/intellij.gradle.jps.iml"
+    intellij "xml/relaxng/intellij.relaxng.iml"
+    intellij "plugins/maven/intellij.maven.iml"
+    intellij "jvm-run-configurations/kotlin.jvm-run-configurations.iml"
+    intellij "java/compiler/intellij.java.compiler.tests.iml"
+    intellij "platform/testFramework/extensions/intellij.platform.testExtensions.iml"
+    intellij "platform/lang-impl/intellij.platform.lang.tests.iml"
+    intellij "platform/xdebugger-testFramework/intellij.platform.debugger.testFramework.iml"
+    intellij "platform/external-system-impl/intellij.platform.externalSystem.tests.iml"
+    intellij "plugins/gradle/tooling-extension-impl/intellij.gradle.toolingExtension.tests.iml"
+    intellij "jvm/jvm-analysis-tests/intellij.jvm.analysis.testFramework.iml"
+    intellij "jvm/jvm-analysis-kotlin-tests/intellij.jvm.analysis.kotlin.tests.iml"
+    intellij "platform/configuration-store-impl/intellij.platform.configurationStore.tests.iml"
+    intellij "plugins/stats-collector/intellij.statsCollector.tests.iml"
+    intellij "plugins/groovy/groovy-uast-tests/intellij.groovy.uast.tests.iml"
+}
+
 include ":jps:jps-common",
         ":jps:jps-plugin"