From 7fbfb1bd95af12f4c5dfb28efc356f884a1ca559 Mon Sep 17 00:00:00 2001 From: Ilya Kirillov Date: Tue, 1 Mar 2022 15:54:00 +0100 Subject: [PATCH] Fix cooperative development setup for kt-master --- build.gradle.kts | 10 ++- buildSrc/prepare-deps/build.gradle.kts | 11 ++- buildSrc/src/main/kotlin/localDependencies.kt | 13 +++ .../build.gradle.kts | 1 + .../imltogradle/GradleDependencyNotation.kt | 37 +-------- .../kotlin/generators/imltogradle/Main.kt | 81 +++++++------------ .../imltogradle/MavenArtifactsBuilder.kt | 51 ++++++++++++ .../kotlin/generators/imltogradle/Util.kt | 16 ---- gradle/versions.properties | 1 + settings.gradle | 39 +++++++++ 10 files changed, 154 insertions(+), 106 deletions(-) create mode 100644 generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/MavenArtifactsBuilder.kt diff --git a/build.gradle.kts b/build.gradle.kts index 1b017f8b7d9..809f72b9dec 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -397,8 +397,14 @@ allprojects { val mirrorRepo: String? = findProperty("maven.repository.mirror")?.toString() repositories { - if (kotlinBuildProperties.getOrNull("attachedIntellijVersion") != null) { - kotlinBuildLocalRepo(project) + when(kotlinBuildProperties.getOrNull("attachedIntellijVersion")) { + null -> {} + "master" -> { + maven { setUrl("https://www.jetbrains.com/intellij-repository/snapshots") } + } + else -> { + kotlinBuildLocalRepo(project) + } } mirrorRepo?.let(::maven) diff --git a/buildSrc/prepare-deps/build.gradle.kts b/buildSrc/prepare-deps/build.gradle.kts index d39aefe0790..c88acfc72f7 100644 --- a/buildSrc/prepare-deps/build.gradle.kts +++ b/buildSrc/prepare-deps/build.gradle.kts @@ -208,10 +208,17 @@ fun prepareDeps( } } -if (intellijVersionForIde != null) { - prepareDeps(intellijForIde, intellijCoreForIde, sourcesForIde, jpsStandaloneForIde, intellijVersionForIde) +when(kotlinBuildProperties.getOrNull("attachedIntellijVersion")) { + null -> {} + "master" -> {} // for intellij/kt-master, intellij maven artifacts are used instead of manual unpacked dependencies + else -> { + val intellijVersionForIde = intellijVersionForIde + ?: error("intellijVersionForIde should not be null as attachedIntellijVersion is present") + prepareDeps(intellijForIde, intellijCoreForIde, sourcesForIde, jpsStandaloneForIde, intellijVersionForIde) + } } + tasks.named("clean") { delete(customDepsRepoDir) } diff --git a/buildSrc/src/main/kotlin/localDependencies.kt b/buildSrc/src/main/kotlin/localDependencies.kt index bcface74baf..93c645b1b40 100644 --- a/buildSrc/src/main/kotlin/localDependencies.kt +++ b/buildSrc/src/main/kotlin/localDependencies.kt @@ -74,14 +74,27 @@ fun RepositoryHandler.kotlinBuildLocalRepo(project: Project): IvyArtifactReposit } } +/* */ @JvmOverloads fun Project.intellijDep(module: String? = null, forIde: Boolean = false) = "kotlin.build:${module ?: ideModuleName()}:${ideModuleVersion(forIde)}" + fun Project.intellijCoreDep() = "kotlin.build:intellij-core:${rootProject.extra["versions.intellijSdk"]}" fun Project.intellijPluginDep(plugin: String, forIde: Boolean = false) = intellijDep(plugin, forIde) +/* */ + + +/* */ +fun Project.intellijMavenDep(subsystem: String, artifact: String) = + "com.jetbrains.$subsystem:$artifact:${ideModuleVersion(forIde = true)}" + + +/* */ + + fun ModuleDependency.includeJars(vararg names: String, rootProject: Project? = null) { names.forEach { var baseName = it.removeSuffix(".jar") diff --git a/generators/ide-iml-to-gradle-generator/build.gradle.kts b/generators/ide-iml-to-gradle-generator/build.gradle.kts index 7efd8b8da30..91d38b41a1b 100644 --- a/generators/ide-iml-to-gradle-generator/build.gradle.kts +++ b/generators/ide-iml-to-gradle-generator/build.gradle.kts @@ -6,6 +6,7 @@ plugins { dependencies { implementation(kotlinStdlib("jdk8")) implementation(jpsModel()) + implementation("com.jetbrains.intellij.platform:util-text-matching:$intellijVersion") implementation(jpsModelImpl()) implementation(jpsModelSerialization()) implementation(commonDependency("com.google.code.gson:gson")) diff --git a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/GradleDependencyNotation.kt b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/GradleDependencyNotation.kt index 382c94190f1..dddc56569d6 100644 --- a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/GradleDependencyNotation.kt +++ b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/GradleDependencyNotation.kt @@ -5,42 +5,11 @@ package org.jetbrains.kotlin.generators.imltogradle -open class GradleDependencyNotation(val dependencyNotation: String, val dependencyConfiguration: String? = null) { +sealed class GradleDependencyNotation(val dependencyNotation: String) { init { require(dependencyNotation.isNotEmpty()) - require(dependencyConfiguration?.isNotEmpty() ?: true) } - companion object { - private const val artifactNameSubregex = """([a-zA-Z\-\._1-9]*?)""" - - private val libPathToGradleNotationRegex = """^lib\/$artifactNameSubregex\.jar$""".toRegex() - private val pluginsPathToGradleNotationRegex = """^plugins\/$artifactNameSubregex\/.*?$""".toRegex() - private val jarToGradleNotationRegex = """^$artifactNameSubregex\.jar$""".toRegex() - - fun fromJarPath(jarPath: String): GradleDependencyNotation? { - if (jarPath == "lib/cds/classesLogAgent.jar") { - return null // TODO remove hack? - } - - if (jarPath.contains("intellij-core.jar")) { - return IntellijCoreGradleDependencyNotation - } - - fun Regex.firstGroup() = matchEntire(jarPath)?.groupValues?.get(1) - - return pluginsPathToGradleNotationRegex.firstGroup()?.let { IntellijPluginDepGradleDependencyNotation(it) } - ?: libPathToGradleNotationRegex.firstGroup()?.let { IntellijDepGradleDependencyNotation(it) } - ?: jarToGradleNotationRegex.firstGroup()?.let { IntellijDepGradleDependencyNotation(it) } - ?: error("Path $jarPath matches none of the regexes") - } - } - - object IntellijCoreGradleDependencyNotation : GradleDependencyNotation("intellijCoreDep()", null) - - data class IntellijPluginDepGradleDependencyNotation(val pluginName: String) : - GradleDependencyNotation("intellijPluginDep(\"$pluginName\", forIde = true)") - - data class IntellijDepGradleDependencyNotation(val jarName: String) : - GradleDependencyNotation("intellijDep(forIde = true)", "{ includeJars(\"$jarName\") }") + data class IntellijMavenDepGradleDependencyNotation(val groupId: String, val artifactId: String) : + GradleDependencyNotation("""intellijMavenDep("$groupId", "$artifactId")""") } diff --git a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Main.kt b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Main.kt index 91fe5ae2a7f..d043ef9d18e 100644 --- a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Main.kt +++ b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Main.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.generators.imltogradle -import com.google.gson.JsonParser import org.jetbrains.jps.model.JpsSimpleElement import org.jetbrains.jps.model.java.JavaResourceRootType import org.jetbrains.jps.model.java.JavaSourceRootType @@ -14,11 +13,8 @@ import org.jetbrains.jps.model.library.JpsLibrary import org.jetbrains.jps.model.library.JpsMavenRepositoryLibraryDescriptor import org.jetbrains.jps.model.library.JpsOrderRootType import org.jetbrains.jps.model.module.* -import org.jetbrains.kotlin.generators.imltogradle.GradleDependencyNotation.IntellijDepGradleDependencyNotation import java.io.File - -private lateinit var intellijModuleNameToGradleDependencyNotationsMapping: Map> private val KOTLIN_REPO_ROOT = File(".").canonicalFile val DEFAULT_KOTLIN_SNAPSHOT_VERSION = KOTLIN_REPO_ROOT.resolve("gradle.properties") .readProperty("defaultSnapshotVersion") @@ -26,11 +22,6 @@ val DEFAULT_KOTLIN_SNAPSHOT_VERSION = KOTLIN_REPO_ROOT.resolve("gradle.propertie private val INTELLIJ_REPO_ROOT = KOTLIN_REPO_ROOT.resolve("intellij").resolve("community").takeIf { it.exists() } ?: KOTLIN_REPO_ROOT.resolve("intellij") -private val intellijModuleNameToGradleDependencyNotationsMappingManual: List> = listOf( - "intellij.platform.jps.build" to GradleDependencyNotation("jpsBuildTest()"), - "intellij.platform.structuralSearch" to IntellijDepGradleDependencyNotation("structuralsearch") // for some reason it's absent in json mapping -) - // These modules are used in Kotlin plugin and IDEA doesn't publish artifact of these modules private val intellijModulesForWhichGenerateBuildGradle = listOf( "intellij.platform.debugger.testFramework", @@ -41,13 +32,20 @@ private val intellijModulesForWhichGenerateBuildGradle = listOf( "intellij.java.compiler.tests", "intellij.gradle.toolingExtension.tests", "intellij.maven", + "intellij.gradle.java", + "intellij.gradle.jps", + "intellij.relaxng", + "intellij.jvm.analysis.kotlin.tests", + "intellij.jvm.analysis.testFramework", + "intellij.platform.configurationStore.tests", + "intellij.statsCollector.tests", + "intellij.groovy.uast.tests", ) -val jsonUrlPrefixes = mapOf( - "202" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform202_IntellijArtifactMappings/113235432:id", - "203" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform203_IntellijArtifactMappings/117989041:id", - "211" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform211_IntellijArtifactMappings/121258191:id", - "212" to "https://buildserver.labs.intellij.net/guestAuth/repository/download/ijplatform_IjPlatform211_IntellijArtifactMappings/131509697:id", +private val intellijModulesToIgnore = listOf( + "kotlin.util.compiler-dependencies", + "intellij.gradle.java.tests", + "intellij.grazie.tests", ) fun main() { @@ -64,35 +62,6 @@ fun main() { } .toList() - val ideaMajorVersion = KOTLIN_REPO_ROOT.resolve("local.properties").readProperty("attachedIntellijVersion") - - intellijModuleNameToGradleDependencyNotationsMapping = fetchJsonsFromBuildserver(ideaMajorVersion) - .flatMap { jsonStr -> - JsonParser.parseString(jsonStr).asJsonArray.mapNotNull { jsonElement -> - val jsonObject = jsonElement.asJsonObject - val moduleName = jsonObject.get("moduleName")?.asString ?: return@mapNotNull null - val jarPath = jsonObject.get("path")?.asString ?: return@mapNotNull null - moduleName to jarPath - } - } - .filter { (_, jarPath) -> !jarPath.contains("DatabaseTools") && !jarPath.contains("lib/openapi.jar") } - .groupBy( - keySelector = { (_, jarPath) -> jarPath }, - valueTransform = { (moduleName, _) -> moduleName } - ) - .filter { (_, moduleNames) -> - moduleNames.all { it in ijCommunityModuleNameToJpsModuleMapping } // filter out ultimate jars - } - .flatMap { (jarPath, moduleNames) -> moduleNames.map { it to jarPath } } - .mapNotNull { (moduleName, jarPath) -> - moduleName to (GradleDependencyNotation.fromJarPath(jarPath) ?: return@mapNotNull null) - } - .plus(intellijModuleNameToGradleDependencyNotationsMappingManual) - .groupBy( - keySelector = { (moduleName, _) -> moduleName }, - valueTransform = { (_, dependencyNotation) -> dependencyNotation } - ) - val imlsInSameDirectory: List> = imlFiles.groupBy { it.parentFile }.filter { it.value.size > 1 }.map { it.value } if (imlsInSameDirectory.isNotEmpty()) { val report = imlsInSameDirectory.joinToString("\n") { "In same directory: " + it.joinToString() } @@ -103,7 +72,7 @@ fun main() { .mapNotNull { imlFile -> ijCommunityModuleNameToJpsModuleMapping[imlFile.nameWithoutExtension]?.let { imlFile to it } } - .filter { (_, jpsModule) -> jpsModule.name != "kotlin.util.compiler-dependencies" } + .filter { (_, jpsModule) -> jpsModule.name !in intellijModulesToIgnore } .forEach { (imlFile, jpsModule) -> println("Processing iml ${imlFile}") imlFile.parentFile.resolve("build.gradle.kts").writeText(convertJpsModule(imlFile, jpsModule)) @@ -168,14 +137,22 @@ fun convertJpsLibrary(lib: JpsLibrary, scope: JpsJavaDependencyScope, exported: fun convertIntellijDependencyNotFollowingTransitive(dep: JpsDependencyDescriptor, exported: Boolean): List { return when (val moduleOrLibrary = dep.moduleOrLibrary) { is Either.First -> { - val moduleName = moduleOrLibrary.value.name - if (moduleName in intellijModulesForWhichGenerateBuildGradle) { - listOf(JpsLikeModuleDependency(":kotlin-ide.$moduleName", dep.scope, exported)) - } else { - intellijModuleNameToGradleDependencyNotationsMapping[moduleName] - .also { if (it == null) println("WARNING: Cannot find GradleDependencyNotation for $moduleName") } - ?.map { JpsLikeJarDependency(it.dependencyNotation, dep.scope, it.dependencyConfiguration, exported) } - ?: emptyList() + when (val moduleName = moduleOrLibrary.value.name) { + in intellijModulesForWhichGenerateBuildGradle -> { + listOf(JpsLikeModuleDependency(":kotlin-ide.$moduleName", dep.scope, exported)) + } + in intellijModulesToIgnore -> emptyList() + else -> { + val (groupId, artifactId) = MavenArtifactsBuilder.generateMavenCoordinates(moduleName) + listOf( + JpsLikeJarDependency( + GradleDependencyNotation.IntellijMavenDepGradleDependencyNotation(groupId, artifactId).dependencyNotation, + dep.scope, + dependencyConfiguration = null, + exported + ) + ) + } } } is Either.Second -> convertJpsLibrary(moduleOrLibrary.value, dep.scope, exported) diff --git a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/MavenArtifactsBuilder.kt b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/MavenArtifactsBuilder.kt new file mode 100644 index 00000000000..fb3141069c2 --- /dev/null +++ b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/MavenArtifactsBuilder.kt @@ -0,0 +1,51 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.generators.imltogradle + +import com.intellij.util.text.NameUtilCore +import java.util.* + +// Copied from intellij build scripts +object MavenArtifactsBuilder { + fun generateMavenCoordinates(moduleName: String): MavenArtifact { + val names = moduleName.split(".") + if (names.size < 2) { + error("Cannot generate Maven artifacts: incorrect module name '${moduleName}'") + } + val groupId = names.take(2).joinToString(separator = ".") + val firstMeaningful = if (names.size > 2 && COMMON_GROUP_NAMES.contains(names[1])) 2 else 1 + val artifactId = names.drop(firstMeaningful).flatMap { + splitByCamelHumpsMergingNumbers(it).map { it.lowercase(Locale.US) } + }.joinToString(separator = "-") + return MavenArtifact(groupId, artifactId) + } + + private fun splitByCamelHumpsMergingNumbers(s: String): List { + val words: List = NameUtilCore.splitNameIntoWords(s).toList() + + val result = ArrayList() + var i = 0 + while (i < words.size) { + val next: String + if (i < words.size - 1 && Character.isDigit(words[i + 1][0])) { + next = words[i] + words[i + 1] + i++ + } else { + next = words[i] + } + result += next + i++ + } + return result + } + + private val COMMON_GROUP_NAMES = setOf("platform", "vcs", "tools", "clouds") +} + +data class MavenArtifact( + val groupId: String, + val artifactId: String +) \ No newline at end of file diff --git a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Util.kt b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Util.kt index ffc1663618c..9ab76d3a11c 100644 --- a/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Util.kt +++ b/generators/ide-iml-to-gradle-generator/src/org/jetbrains/kotlin/generators/imltogradle/Util.kt @@ -83,22 +83,6 @@ inline fun T?.orElse(block: () -> T): T = this ?: block() val JpsModule.dependencies: List get() = dependenciesList.dependencies.filter { it is JpsModuleDependency || it is JpsLibraryDependency } -fun fetchJsonsFromBuildserver(ideaMajorVersion: String): List { - require(ideaMajorVersion.length == 3 && ideaMajorVersion.all { it.isDigit() }) { - "attachedIntellijVersion='$ideaMajorVersion' must be 3 length all digit string" - } - val urlPrefix = jsonUrlPrefixes[ideaMajorVersion] ?: error("'$ideaMajorVersion' platform is absent in mapping") - return listOf( - "$urlPrefix/ideaIU-project-structure-mapping.json", - "$urlPrefix/intellij-core-project-structure-mapping.json" - ).map { url -> - try { - URL(url).readText() - } catch (ex: Throwable) { - error("Can't access $url. Is VPN on?") - } - } -} fun File.readProperty(propertyName: String): String { return inputStream().use { Properties().apply { load(it) }.getProperty(propertyName) } diff --git a/gradle/versions.properties b/gradle/versions.properties index 48497d99e64..c5e4367bec7 100644 --- a/gradle/versions.properties +++ b/gradle/versions.properties @@ -3,6 +3,7 @@ versions.intellijSdk.forIde.202=202.7660.26 versions.intellijSdk.forIde.203=203.6682.168 versions.intellijSdk.forIde.211=211.7442.40 versions.intellijSdk.forIde.212=212.4746.92 +versions.intellijSdk.forIde.master=221.4842-EAP-CANDIDATE-SNAPSHOT # versions.* should match library versions used in IDEA from versions.intellijSdk diff --git a/settings.gradle b/settings.gradle index 03d0d5e89d9..4bd09cd1958 100644 --- a/settings.gradle +++ b/settings.gradle @@ -485,6 +485,45 @@ if (attachedIntellijVersion == "212") { // Latest available platform in scope of intellij "jvm-run-configurations/kotlin.jvm-run-configurations.iml" } +if (attachedIntellijVersion == "master") { + logger.info("Including kotlin-ide modules in settings.gradle") + def excludedModules = [ + "tools/kotlin-maven-artifacts-publishing/intellij.kotlin.util.mavenArtifactsPublishing.iml", + "util/project-model-updater/kotlin.util.project-model-updater.iml", + "util/compiler-dependencies/kotlin.util.compiler-dependencies.iml", + "kotlin-compiler-classpath/kotlin.util.compiler-classpath.iml" + ] + def modulesIml = new File("${rootDir}/intellij/.idea/modules.xml").text + def matcher = modulesIml =~ /filepath="(.+)"/ // TODO use some xml parser instead + def modules = (0..