diff --git a/build.gradle.kts b/build.gradle.kts index aa40babe383..78f7508e8e4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,9 +11,15 @@ buildscript { // when updating please also update JPS artifacts configuration: https://jetbrains.quip.com/zzGUAYSJ6gv3/JPS-Build-update-bootstrap kotlinBootstrapFrom(BootstrapOption.TeamCity("1.3.50-dev-397", onlySuccessBootstrap = false)) - repositories.withRedirector(project) { + repositories { bootstrapKotlinRepo?.let(::maven) - maven("https://plugins.gradle.org/m2") + + val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true + if (cacheRedirectorEnabled) { + maven("https://cache-redirector.jetbrains.com/plugins.gradle.org/m2") + } else { + maven("https://plugins.gradle.org/m2") + } } // a workaround for kotlin compiler classpath in kotlin project: sometimes gradle substitutes @@ -410,10 +416,12 @@ allprojects { } } - if (cacheRedirectorEnabled()) { - logger.info("Redirecting repositories for $displayName") - repositories.redirect() + // Aggregate task for build related checks + tasks.register("checkBuild") { + tasks.findByName("check")?.dependsOn(this) } + + apply(from = "$rootDir/gradle/cacheRedirector.gradle.kts") } } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 68892949cb9..bf3b57c60b1 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,15 +1,17 @@ +extra["versions.shadow"] = "4.0.3" +extra["versions.native-platform"] = "0.14" buildscript { val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true val buildSrcKotlinVersion: String by extra(findProperty("buildSrc.kotlin.version")?.toString() ?: embeddedKotlinVersion) val buildSrcKotlinRepo: String? by extra(findProperty("buildSrc.kotlin.repo") as String?) - extra["versions.shadow"] = "4.0.3" - extra["versions.native-platform"] = "0.14" repositories { if (cacheRedirectorEnabled) { maven("https://cache-redirector.jetbrains.com/jcenter.bintray.com") + } else { + jcenter() } buildSrcKotlinRepo?.let { @@ -77,18 +79,13 @@ extra["versions.androidDxSources"] = "5.0.0_r2" extra["customDepsOrg"] = "kotlin.build" repositories { - if (cacheRedirectorEnabled) { - maven("https://cache-redirector.jetbrains.com/jcenter.bintray.com") - maven("https://cache-redirector.jetbrains.com/jetbrains.bintray.com/intellij-third-party-dependencies/") - } + maven("https://jetbrains.bintray.com/intellij-third-party-dependencies/") + maven("https://plugins.gradle.org/m2/") + jcenter() extra["buildSrcKotlinRepo"]?.let { maven(url = it) } - - jcenter() - maven("https://jetbrains.bintray.com/intellij-third-party-dependencies/") - maven("https://plugins.gradle.org/m2/") } dependencies { @@ -112,3 +109,10 @@ fun Project.`samWithReceiver`(configure: org.jetbrains.kotlin.samWithReceiver.gr extensions.configure("samWithReceiver", configure) tasks["build"].dependsOn(":prepare-deps:build") + +allprojects { + afterEvaluate { + apply(from = "$rootDir/../gradle/cacheRedirector.gradle.kts") + tasks["check"].dependsOn(tasks["checkRepositories"]) + } +} \ No newline at end of file diff --git a/buildSrc/prepare-deps/build.gradle.kts b/buildSrc/prepare-deps/build.gradle.kts index 30d89db8406..db5995d69bc 100644 --- a/buildSrc/prepare-deps/build.gradle.kts +++ b/buildSrc/prepare-deps/build.gradle.kts @@ -5,8 +5,12 @@ import org.gradle.api.publish.ivy.internal.publication.DefaultIvyConfiguration import org.gradle.api.publish.ivy.internal.publication.DefaultIvyPublicationIdentity import org.gradle.api.publish.ivy.internal.publisher.IvyDescriptorFileGenerator import org.gradle.internal.os.OperatingSystem +import java.net.URI + +plugins { + base +} -val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true val verifyDependencyOutput: Boolean by rootProject.extra val intellijUltimateEnabled: Boolean by rootProject.extra val intellijReleaseType: String by rootProject.extra @@ -65,11 +69,12 @@ val androidToolsOs by lazy { repositories { if (androidStudioRelease != null) { ivy { - if (cacheRedirectorEnabled) { - artifactPattern("https://cache-redirector.jetbrains.com/dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease/[artifact]-[revision]-$androidStudioOs.[ext]") + url = URI("https://dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease") + + patternLayout { + artifact("[artifact]-[revision]-$androidStudioOs.[ext]") } - artifactPattern("https://dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease/[artifact]-[revision]-$androidStudioOs.[ext]") metadataSources { artifact() } @@ -77,17 +82,28 @@ repositories { } ivy { - artifactPattern("https://dl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]") - artifactPattern("https://android.googlesource.com/platform/dalvik/+archive/android-$androidDxSourcesVersion/[artifact].[ext]") + url = URI("https://dl.google.com/android/repository") + + patternLayout { + artifact("[artifact]_[revision](-[classifier]).[ext]") + } + metadataSources { artifact() } } - if (cacheRedirectorEnabled) { - maven("https://cache-redirector.jetbrains.com/www.jetbrains.com/intellij-repository/$intellijReleaseType") - maven("https://cache-redirector.jetbrains.com/plugins.jetbrains.com/maven") - maven("https://cache-redirector.jetbrains.com/jetbrains.bintray.com/intellij-third-party-dependencies/") + + ivy { + url = URI("https://android.googlesource.com/platform/dalvik/+archive/android-$androidDxSourcesVersion") + + patternLayout { + artifact("[artifact].[ext]") + } + + metadataSources { + artifact() + } } maven("https://www.jetbrains.com/intellij-repository/$intellijReleaseType") @@ -272,19 +288,20 @@ val makeIde = if (androidStudioBuild != null) { task } -val build by tasks.creating { +val buildJpsStandalone = buildIvyRepositoryTask(jpsStandalone, customDepsOrg, customDepsRepoDir, null, sourcesFile) +val buildNodeJsPlugin = buildIvyRepositoryTask(nodeJSPlugin, customDepsOrg, customDepsRepoDir, ::skipToplevelDirectory, sourcesFile) + +tasks.named("build") { dependsOn( makeIntellijCore, makeIde, - buildIvyRepositoryTask(jpsStandalone, customDepsOrg, customDepsRepoDir, null, sourcesFile), + buildJpsStandalone, makeIntellijAnnotations, buildIvyRepoForAndroidDx ) if (installIntellijUltimate) { - dependsOn( - buildIvyRepositoryTask(nodeJSPlugin, customDepsOrg, customDepsRepoDir, ::skipToplevelDirectory, sourcesFile) - ) + dependsOn(buildNodeJsPlugin) } } @@ -294,7 +311,7 @@ tasks.create("cleanLegacy", Delete::class.java) { delete("$projectDir/intellij-sdk") } -tasks.create("clean", Delete::class.java) { +tasks.named("clean") { delete(customDepsRepoDir) } diff --git a/buildSrc/src/main/kotlin/jbCacheRedirector.kt b/buildSrc/src/main/kotlin/jbCacheRedirector.kt deleted file mode 100644 index 5beaf4bdd0f..00000000000 --- a/buildSrc/src/main/kotlin/jbCacheRedirector.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -import org.gradle.api.Project -import org.gradle.api.artifacts.dsl.RepositoryHandler -import org.gradle.api.artifacts.repositories.MavenArtifactRepository -import java.net.URI - -// https://youtrack.jetbrains.com/issue/ADM-23180 -val mirroredUrls = listOf( - "https://dl.bintray.com/groovy/maven", - "https://dl.bintray.com/kotlin/kotlin-dev", - "https://dl.bintray.com/kotlin/kotlin-eap", - "https://dl.google.com/dl/android/maven2", - "https://dl.google.com/go", - "https://download.jetbrains.com", - "https://jcenter.bintray.com", - "https://jetbrains.bintray.com/dekaf", - "https://jetbrains.bintray.com/intellij-jdk", - "https://jetbrains.bintray.com/intellij-plugin-service", - "https://jetbrains.bintray.com/intellij-third-party-dependencies", - "https://jetbrains.bintray.com/markdown", - "https://jetbrains.bintray.com/teamcity-rest-client", - "https://jetbrains.bintray.com/test-discovery", - "https://jetbrains.bintray.com/jediterm", - "https://jitpack.io", - "https://maven.exasol.com/artifactory/exasol-releases", - "https://plugins.gradle.org/m2", - "https://plugins.jetbrains.com/maven", - "https://repo.grails.org/grails/core", - "https://repo.jenkins-ci.org/releases", - "https://repo.spring.io/milestone", - "https://repo1.maven.org/maven2", - "https://services.gradle.org", - "https://www.jetbrains.com/intellij-repository", - "https://www.myget.org/F/intellij-go-snapshots/maven", - "https://www.myget.org/F/rd-snapshots/maven", - "https://www.myget.org/F/rd-model-snapshots/maven", - "https://www.python.org/ftp", - "https://dl.google.com/dl/android/studio/ide-zips", - "https://dl.bintray.com/kotlin/ktor", - "https://cdn.azul.com/zulu/bin" -) - -fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path") - -fun RepositoryHandler.redirect() = filterIsInstance().forEach { repository -> - val uri = repository.url - if (mirroredUrls.any { uri.toString().trimEnd('/').startsWith(it) }) { - repository.url = uri.toCacheRedirectorUri() - } -} - -fun Project.cacheRedirectorEnabled(): Boolean = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true - -fun RepositoryHandler.withRedirector(project: Project, configuration: RepositoryHandler.() -> Unit) { - configuration() - if (project.cacheRedirectorEnabled()) { - redirect() - } -} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/localDependencies.kt b/buildSrc/src/main/kotlin/localDependencies.kt index 11b3ad7f533..32ff3a064d6 100644 --- a/buildSrc/src/main/kotlin/localDependencies.kt +++ b/buildSrc/src/main/kotlin/localDependencies.kt @@ -42,17 +42,19 @@ private fun Project.ideModuleVersion() = when (IdeVersionConfigurator.currentIde fun RepositoryHandler.kotlinBuildLocalRepo(project: Project): IvyArtifactRepository = ivy { val baseDir = project.kotlinBuildLocalRepoDir() - setUrl(baseDir) + url = baseDir.toURI() - ivyPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/[module].ivy.xml") - ivyPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/ivy/[module].ivy.xml") - ivyPattern("${baseDir.canonicalPath}/[organisation]/${project.ideModuleName()}/[revision]/ivy/[module].ivy.xml") // bundled plugins + patternLayout { + ivy("[organisation]/[module]/[revision]/[module].ivy.xml") + ivy("[organisation]/[module]/[revision]/ivy/[module].ivy.xml") + ivy("[organisation]/${project.ideModuleName()}/[revision]/ivy/[module].ivy.xml") // bundled plugins - artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/artifacts/lib/[artifact](-[classifier]).[ext]") - artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/artifacts/[artifact](-[classifier]).[ext]") - artifactPattern("${baseDir.canonicalPath}/[organisation]/${project.ideModuleName()}/[revision]/artifacts/plugins/[module]/lib/[artifact](-[classifier]).[ext]") // bundled plugins - artifactPattern("${baseDir.canonicalPath}/[organisation]/sources/[artifact]-[revision](-[classifier]).[ext]") - artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/[artifact](-[classifier]).[ext]") + artifact("[organisation]/[module]/[revision]/artifacts/lib/[artifact](-[classifier]).[ext]") + artifact("[organisation]/[module]/[revision]/artifacts/[artifact](-[classifier]).[ext]") + artifact("[organisation]/${project.ideModuleName()}/[revision]/artifacts/plugins/[module]/lib/[artifact](-[classifier]).[ext]") // bundled plugins + artifact("[organisation]/sources/[artifact]-[revision](-[classifier]).[ext]") + artifact("[organisation]/[module]/[revision]/[artifact](-[classifier]).[ext]") + } metadataSources { ivyDescriptor() diff --git a/dependencies/android-sdk/build.gradle.kts b/dependencies/android-sdk/build.gradle.kts index 4812dd8d22b..230a33ce3fa 100644 --- a/dependencies/android-sdk/build.gradle.kts +++ b/dependencies/android-sdk/build.gradle.kts @@ -1,16 +1,17 @@ - -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar -import java.io.File import org.gradle.internal.os.OperatingSystem +import java.net.URI // TODO: consider adding dx sources (the only jar used on the compile time so far) // e.g. from "https://android.googlesource.com/platform/dalvik/+archive/android-5.0.0_r2/dx.tar.gz" repositories { ivy { - artifactPattern("https://dl-ssl.google.com/android/repository/[artifact]-[revision].[ext]") - artifactPattern("https://dl-ssl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]") - artifactPattern("https://dl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]") + url = URI("https://dl.google.com/android/repository") + patternLayout { + artifact("[artifact]-[revision].[ext]") + artifact("[artifact]_[revision](-[classifier]).[ext]") + artifact("[artifact]_[revision](-[classifier]).[ext]") + } } } diff --git a/gradle/cacheRedirector.gradle.kts b/gradle/cacheRedirector.gradle.kts new file mode 100644 index 00000000000..d1ddd859685 --- /dev/null +++ b/gradle/cacheRedirector.gradle.kts @@ -0,0 +1,169 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +import org.gradle.api.Project +import org.gradle.api.artifacts.dsl.RepositoryHandler +import org.gradle.api.artifacts.repositories.MavenArtifactRepository +import java.net.URI + +/** + * The list of repositories supported by cache redirector should be synced with the list in https://youtrack.jetbrains.com/issue/ADM-23180 + * To add a repository to the list create an issue in ADM project (example issue https://youtrack.jetbrains.com/issue/IJI-149) + * Repositories in `buildscript` blocks are *NOT* substituted by this script and should be handled manually + */ +val mirroredUrls = listOf( + "https://dl.bintray.com/groovy/maven", + "https://dl.bintray.com/kotlin/kotlin-dev", + "https://dl.bintray.com/kotlin/kotlin-eap", + "https://dl.google.com/dl/android/maven2", + "https://dl.google.com/go", + "https://download.jetbrains.com", + "https://jcenter.bintray.com", + "https://jetbrains.bintray.com/dekaf", + "https://jetbrains.bintray.com/intellij-jdk", + "https://jetbrains.bintray.com/intellij-plugin-service", + "https://jetbrains.bintray.com/intellij-third-party-dependencies", + "https://jetbrains.bintray.com/markdown", + "https://jetbrains.bintray.com/teamcity-rest-client", + "https://jetbrains.bintray.com/test-discovery", + "https://jetbrains.bintray.com/jediterm", + "https://jitpack.io", + "https://maven.exasol.com/artifactory/exasol-releases", + "https://plugins.gradle.org/m2", + "https://plugins.jetbrains.com/maven", + "https://repo.grails.org/grails/core", + "https://repo.jenkins-ci.org/releases", + "https://repo.spring.io/milestone", + "https://repo1.maven.org/maven2", + "https://services.gradle.org", + "https://www.jetbrains.com/intellij-repository", + "https://www.myget.org/F/intellij-go-snapshots/maven", + "https://www.myget.org/F/rd-snapshots/maven", + "https://www.myget.org/F/rd-model-snapshots/maven", + "https://www.python.org/ftp", + "https://dl.google.com/dl/android/studio/ide-zips", + "https://dl.bintray.com/kotlin/ktor", + "https://cdn.azul.com/zulu/bin" +) + +val aliases = mapOf( + "https://repo.maven.apache.org/maven2" to "https://repo1.maven.org/maven2" // Maven Central +) + +val isTeamcityBuild = project.hasProperty("teamcity") || System.getenv("TEAMCITY_VERSION") != null + +fun Project.cacheRedirectorEnabled(): Boolean = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true + +fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path") + +fun URI.maybeRedirect(): URI { + val url = toString().trimEnd('/') + val dealiasedUrl = aliases.getOrDefault(url, url) + + return if (mirroredUrls.any { dealiasedUrl.startsWith(it) }) { + URI(dealiasedUrl).toCacheRedirectorUri() + } else { + this + } +} + +fun RepositoryHandler.redirect() { + for (repository in this) { + when (repository) { + is MavenArtifactRepository -> repository.url = repository.url.maybeRedirect() + is IvyArtifactRepository -> if (repository.url != null) { + repository.url = repository.url.maybeRedirect() + } + } + } +} + +// teamcity.jetbrains.com is located in the same local network with build agents +fun URI.isCachedOrLocal() = host == "cache-redirector.jetbrains.com" || scheme == "file" || host == "teamcity.jetbrains.com" + +fun RepositoryHandler.findNonCachedRepositories(): List { + val mavenNonCachedRepos = filterIsInstance() + .filterNot { it.url.isCachedOrLocal() } + .map { it.url.toString() } + + val ivyNonCachedRepos = filterIsInstance() + .filterNot { it.url.isCachedOrLocal() } + .map { it.url.toString() } + + return mavenNonCachedRepos + ivyNonCachedRepos +} + +fun escape(s: String): String { + return s.replace("[\\|'\\[\\]]".toRegex(), "\\|$0").replace("\n".toRegex(), "|n").replace("\r".toRegex(), "|r") +} + +fun testStarted(testName: String) { + println("##teamcity[testStarted name='%s']".format(escape(testName))) +} + +fun testFinished(testName: String) { + println("##teamcity[testFinished name='%s']".format(escape(testName))) +} + +fun testFailed(name: String, message: String, details: String) { + println("##teamcity[testFailed name='%s' message='%s' details='%s']".format(escape(name), escape(message), escape(details))) +} + +fun Task.logNonCachedRepo(testName: String, repoUrl: String) { + val msg = "Repository $repoUrl in ${project.displayName} should be cached with cache-redirector" + val details = "Using non cached repository may lead to download failures in CI builds." + + " Check https://github.com/JetBrains/kotlin/blob/master/gradle/cacheRedirector.gradle.kts for details." + + if (isTeamcityBuild) { + testFailed(testName, msg, details) + } + + logger.warn("WARNING - $msg\n$details") +} + +fun Task.logInvalidIvyRepo(testName: String) { + val msg = "Invalid ivy repo found in ${project.displayName}" + val details = "Url must be not null" + + if (isTeamcityBuild) { + testFailed(testName, msg, details) + } + + logger.warn("WARNING - $msg: $details") +} + +val checkRepositories: TaskProvider = tasks.register("checkRepositories") { + doLast { + val testName = "$name in ${project.displayName}" + if (isTeamcityBuild) { + testStarted(testName) + } + + repositories.filterIsInstance().forEach { + if (it.url == null) { + logInvalidIvyRepo(testName) + } + } + + repositories.findNonCachedRepositories().forEach { + logNonCachedRepo(testName, it) + } + + buildscript.repositories.findNonCachedRepositories().forEach { + logNonCachedRepo(testName, it) + } + + if (isTeamcityBuild) { + testFinished(testName) + } + } +} + +tasks.findByName("checkBuild")?.dependsOn(checkRepositories) + +if (cacheRedirectorEnabled()) { + logger.info("Redirecting repositories for $displayName") + repositories.redirect() +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index fc67af84405..da2dc2f5f97 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,12 +6,15 @@ pluginManagement { url pluginRepo } } - gradlePluginPortal() - } - if (cacheRedirectorEnabled == 'true') { - logger.info("Redirecting repositories for settings.gradle pluginManagement") - JbCacheRedirectorKt.redirect(repositories) + if (cacheRedirectorEnabled == 'true') { + logger.info("Using cache redirector for settings.gradle pluginManagement") + maven { + url "https://cache-redirector.jetbrains.com/plugins.gradle.org/m2" + } + } else { + gradlePluginPortal() + } } }