Build: Extract cache redirector to script & add verification task

build.gradle.kts

@@ -11,9 +11,15 @@ buildscript {
     // when updating please also update JPS artifacts configuration: https://jetbrains.quip.com/zzGUAYSJ6gv3/JPS-Build-update-bootstrap
     kotlinBootstrapFrom(BootstrapOption.TeamCity("1.3.50-dev-397", onlySuccessBootstrap = false))
 
-    repositories.withRedirector(project) {
+    repositories {
         bootstrapKotlinRepo?.let(::maven)
-        maven("https://plugins.gradle.org/m2")
+
+        val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
+        if (cacheRedirectorEnabled) {
+            maven("https://cache-redirector.jetbrains.com/plugins.gradle.org/m2")
+        } else {
+            maven("https://plugins.gradle.org/m2")
+        }
     }
 
     // a workaround for kotlin compiler classpath in kotlin project: sometimes gradle substitutes
@@ -410,10 +416,12 @@ allprojects {
             }
         }
 
-        if (cacheRedirectorEnabled()) {
+        // Aggregate task for build related checks
-            logger.info("Redirecting repositories for $displayName")
+        tasks.register("checkBuild") {
-            repositories.redirect()
+            tasks.findByName("check")?.dependsOn(this)
         }
+
+        apply(from = "$rootDir/gradle/cacheRedirector.gradle.kts")
     }
 }
 

buildSrc/build.gradle.kts

@@ -1,15 +1,17 @@
+extra["versions.shadow"] = "4.0.3"
+extra["versions.native-platform"] = "0.14"
 
 buildscript {
     val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
 
     val buildSrcKotlinVersion: String by extra(findProperty("buildSrc.kotlin.version")?.toString() ?: embeddedKotlinVersion)
     val buildSrcKotlinRepo: String? by extra(findProperty("buildSrc.kotlin.repo") as String?)
-    extra["versions.shadow"] = "4.0.3"
-    extra["versions.native-platform"] = "0.14"
 
     repositories {
         if (cacheRedirectorEnabled) {
             maven("https://cache-redirector.jetbrains.com/jcenter.bintray.com")
+        } else {
+            jcenter()
         }
 
         buildSrcKotlinRepo?.let {
@@ -77,18 +79,13 @@ extra["versions.androidDxSources"] = "5.0.0_r2"
 extra["customDepsOrg"] = "kotlin.build"
 
 repositories {
-    if (cacheRedirectorEnabled) {
+    maven("https://jetbrains.bintray.com/intellij-third-party-dependencies/")
-        maven("https://cache-redirector.jetbrains.com/jcenter.bintray.com")
+    maven("https://plugins.gradle.org/m2/")
-        maven("https://cache-redirector.jetbrains.com/jetbrains.bintray.com/intellij-third-party-dependencies/")
+    jcenter()
-    }
 
     extra["buildSrcKotlinRepo"]?.let {
         maven(url = it)
     }
-
-    jcenter()
-    maven("https://jetbrains.bintray.com/intellij-third-party-dependencies/")
-    maven("https://plugins.gradle.org/m2/")
 }
 
 dependencies {
@@ -112,3 +109,10 @@ fun Project.`samWithReceiver`(configure: org.jetbrains.kotlin.samWithReceiver.gr
         extensions.configure("samWithReceiver", configure)
 
 tasks["build"].dependsOn(":prepare-deps:build")
+
+allprojects {
+    afterEvaluate {
+        apply(from = "$rootDir/../gradle/cacheRedirector.gradle.kts")
+        tasks["check"].dependsOn(tasks["checkRepositories"])
+    }
+}

buildSrc/prepare-deps/build.gradle.kts

@@ -5,8 +5,12 @@ import org.gradle.api.publish.ivy.internal.publication.DefaultIvyConfiguration
 import org.gradle.api.publish.ivy.internal.publication.DefaultIvyPublicationIdentity
 import org.gradle.api.publish.ivy.internal.publisher.IvyDescriptorFileGenerator
 import org.gradle.internal.os.OperatingSystem
+import java.net.URI
+
+plugins {
+    base
+}
 
-val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
 val verifyDependencyOutput: Boolean by rootProject.extra
 val intellijUltimateEnabled: Boolean by rootProject.extra
 val intellijReleaseType: String by rootProject.extra
@@ -65,11 +69,12 @@ val androidToolsOs by lazy {
 repositories {
     if (androidStudioRelease != null) {
         ivy {
-            if (cacheRedirectorEnabled) {
+            url = URI("https://dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease")
-                artifactPattern("https://cache-redirector.jetbrains.com/dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease/[artifact]-[revision]-$androidStudioOs.[ext]")
+
+            patternLayout {
+                artifact("[artifact]-[revision]-$androidStudioOs.[ext]")
             }
 
-            artifactPattern("https://dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease/[artifact]-[revision]-$androidStudioOs.[ext]")
             metadataSources {
                 artifact()
             }
@@ -77,17 +82,28 @@ repositories {
     }
 
     ivy {
-        artifactPattern("https://dl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]")
+        url = URI("https://dl.google.com/android/repository")
-        artifactPattern("https://android.googlesource.com/platform/dalvik/+archive/android-$androidDxSourcesVersion/[artifact].[ext]")
+
+        patternLayout {
+            artifact("[artifact]_[revision](-[classifier]).[ext]")
+        }
+
         metadataSources {
             artifact()
         }
     }
 
-    if (cacheRedirectorEnabled) {
+
-        maven("https://cache-redirector.jetbrains.com/www.jetbrains.com/intellij-repository/$intellijReleaseType")
+    ivy {
-        maven("https://cache-redirector.jetbrains.com/plugins.jetbrains.com/maven")
+        url = URI("https://android.googlesource.com/platform/dalvik/+archive/android-$androidDxSourcesVersion")
-        maven("https://cache-redirector.jetbrains.com/jetbrains.bintray.com/intellij-third-party-dependencies/")
+
+        patternLayout {
+            artifact("[artifact].[ext]")
+        }
+
+        metadataSources {
+            artifact()
+        }
     }
 
     maven("https://www.jetbrains.com/intellij-repository/$intellijReleaseType")
@@ -272,19 +288,20 @@ val makeIde = if (androidStudioBuild != null) {
     task
 }
 
-val build by tasks.creating {
+val buildJpsStandalone = buildIvyRepositoryTask(jpsStandalone, customDepsOrg, customDepsRepoDir, null, sourcesFile)
+val buildNodeJsPlugin = buildIvyRepositoryTask(nodeJSPlugin, customDepsOrg, customDepsRepoDir, ::skipToplevelDirectory, sourcesFile)
+
+tasks.named("build") {
     dependsOn(
         makeIntellijCore,
         makeIde,
-        buildIvyRepositoryTask(jpsStandalone, customDepsOrg, customDepsRepoDir, null, sourcesFile),
+        buildJpsStandalone,
         makeIntellijAnnotations,
         buildIvyRepoForAndroidDx
     )
 
     if (installIntellijUltimate) {
-        dependsOn(
+        dependsOn(buildNodeJsPlugin)
-            buildIvyRepositoryTask(nodeJSPlugin, customDepsOrg, customDepsRepoDir, ::skipToplevelDirectory, sourcesFile)
-        )
     }
 }
 
@@ -294,7 +311,7 @@ tasks.create("cleanLegacy", Delete::class.java) {
     delete("$projectDir/intellij-sdk")
 }
 
-tasks.create("clean", Delete::class.java) {
+tasks.named<Delete>("clean") {
     delete(customDepsRepoDir)
 }
 

buildSrc/src/main/kotlin/jbCacheRedirector.kt

@@ -1,63 +0,0 @@
-/*
- * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-import org.gradle.api.Project
-import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.artifacts.repositories.MavenArtifactRepository
-import java.net.URI
-
-// https://youtrack.jetbrains.com/issue/ADM-23180
-val mirroredUrls = listOf(
-    "https://dl.bintray.com/groovy/maven",
-    "https://dl.bintray.com/kotlin/kotlin-dev",
-    "https://dl.bintray.com/kotlin/kotlin-eap",
-    "https://dl.google.com/dl/android/maven2",
-    "https://dl.google.com/go",
-    "https://download.jetbrains.com",
-    "https://jcenter.bintray.com",
-    "https://jetbrains.bintray.com/dekaf",
-    "https://jetbrains.bintray.com/intellij-jdk",
-    "https://jetbrains.bintray.com/intellij-plugin-service",
-    "https://jetbrains.bintray.com/intellij-third-party-dependencies",
-    "https://jetbrains.bintray.com/markdown",
-    "https://jetbrains.bintray.com/teamcity-rest-client",
-    "https://jetbrains.bintray.com/test-discovery",
-    "https://jetbrains.bintray.com/jediterm",
-    "https://jitpack.io",
-    "https://maven.exasol.com/artifactory/exasol-releases",
-    "https://plugins.gradle.org/m2",
-    "https://plugins.jetbrains.com/maven",
-    "https://repo.grails.org/grails/core",
-    "https://repo.jenkins-ci.org/releases",
-    "https://repo.spring.io/milestone",
-    "https://repo1.maven.org/maven2",
-    "https://services.gradle.org",
-    "https://www.jetbrains.com/intellij-repository",
-    "https://www.myget.org/F/intellij-go-snapshots/maven",
-    "https://www.myget.org/F/rd-snapshots/maven",
-    "https://www.myget.org/F/rd-model-snapshots/maven",
-    "https://www.python.org/ftp",
-    "https://dl.google.com/dl/android/studio/ide-zips",
-    "https://dl.bintray.com/kotlin/ktor",
-    "https://cdn.azul.com/zulu/bin"
-)
-
-fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path")
-
-fun RepositoryHandler.redirect() = filterIsInstance<MavenArtifactRepository>().forEach { repository ->
-    val uri = repository.url
-    if (mirroredUrls.any { uri.toString().trimEnd('/').startsWith(it) }) {
-        repository.url = uri.toCacheRedirectorUri()
-    }
-}
-
-fun Project.cacheRedirectorEnabled(): Boolean = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
-
-fun RepositoryHandler.withRedirector(project: Project, configuration: RepositoryHandler.() -> Unit) {
-    configuration()
-    if (project.cacheRedirectorEnabled()) {
-        redirect()
-    }
-}

buildSrc/src/main/kotlin/localDependencies.kt

@@ -42,17 +42,19 @@ private fun Project.ideModuleVersion() = when (IdeVersionConfigurator.currentIde
 
 fun RepositoryHandler.kotlinBuildLocalRepo(project: Project): IvyArtifactRepository = ivy {
     val baseDir = project.kotlinBuildLocalRepoDir()
-    setUrl(baseDir)
+    url = baseDir.toURI()
 
-    ivyPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/[module].ivy.xml")
+    patternLayout {
-    ivyPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/ivy/[module].ivy.xml")
+        ivy("[organisation]/[module]/[revision]/[module].ivy.xml")
-    ivyPattern("${baseDir.canonicalPath}/[organisation]/${project.ideModuleName()}/[revision]/ivy/[module].ivy.xml") // bundled plugins
+        ivy("[organisation]/[module]/[revision]/ivy/[module].ivy.xml")
+        ivy("[organisation]/${project.ideModuleName()}/[revision]/ivy/[module].ivy.xml") // bundled plugins
 
-    artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/artifacts/lib/[artifact](-[classifier]).[ext]")
+        artifact("[organisation]/[module]/[revision]/artifacts/lib/[artifact](-[classifier]).[ext]")
-    artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/artifacts/[artifact](-[classifier]).[ext]")
+        artifact("[organisation]/[module]/[revision]/artifacts/[artifact](-[classifier]).[ext]")
-    artifactPattern("${baseDir.canonicalPath}/[organisation]/${project.ideModuleName()}/[revision]/artifacts/plugins/[module]/lib/[artifact](-[classifier]).[ext]") // bundled plugins
+        artifact("[organisation]/${project.ideModuleName()}/[revision]/artifacts/plugins/[module]/lib/[artifact](-[classifier]).[ext]") // bundled plugins
-    artifactPattern("${baseDir.canonicalPath}/[organisation]/sources/[artifact]-[revision](-[classifier]).[ext]")
+        artifact("[organisation]/sources/[artifact]-[revision](-[classifier]).[ext]")
-    artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/[artifact](-[classifier]).[ext]")
+        artifact("[organisation]/[module]/[revision]/[artifact](-[classifier]).[ext]")
+    }
 
     metadataSources {
         ivyDescriptor()

dependencies/android-sdk/build.gradle.kts

@@ -1,16 +1,17 @@
-
-import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
-import java.io.File
 import org.gradle.internal.os.OperatingSystem
+import java.net.URI
 
 // TODO: consider adding dx sources (the only jar used on the compile time so far)
 // e.g. from "https://android.googlesource.com/platform/dalvik/+archive/android-5.0.0_r2/dx.tar.gz"
 
 repositories {
     ivy {
-        artifactPattern("https://dl-ssl.google.com/android/repository/[artifact]-[revision].[ext]")
+        url = URI("https://dl.google.com/android/repository")
-        artifactPattern("https://dl-ssl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]")
+        patternLayout {
-        artifactPattern("https://dl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]")
+            artifact("[artifact]-[revision].[ext]")
+            artifact("[artifact]_[revision](-[classifier]).[ext]")
+            artifact("[artifact]_[revision](-[classifier]).[ext]")
+        }
     }
 }
 

gradle/cacheRedirector.gradle.kts

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+import org.gradle.api.Project
+import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.artifacts.repositories.MavenArtifactRepository
+import java.net.URI
+
+/**
+ *  The list of repositories supported by cache redirector should be synced with the list in https://youtrack.jetbrains.com/issue/ADM-23180
+ *  To add a repository to the list create an issue in ADM project (example issue https://youtrack.jetbrains.com/issue/IJI-149)
+ *  Repositories in `buildscript` blocks are *NOT* substituted by this script and should be handled manually
+ */
+val mirroredUrls = listOf(
+    "https://dl.bintray.com/groovy/maven",
+    "https://dl.bintray.com/kotlin/kotlin-dev",
+    "https://dl.bintray.com/kotlin/kotlin-eap",
+    "https://dl.google.com/dl/android/maven2",
+    "https://dl.google.com/go",
+    "https://download.jetbrains.com",
+    "https://jcenter.bintray.com",
+    "https://jetbrains.bintray.com/dekaf",
+    "https://jetbrains.bintray.com/intellij-jdk",
+    "https://jetbrains.bintray.com/intellij-plugin-service",
+    "https://jetbrains.bintray.com/intellij-third-party-dependencies",
+    "https://jetbrains.bintray.com/markdown",
+    "https://jetbrains.bintray.com/teamcity-rest-client",
+    "https://jetbrains.bintray.com/test-discovery",
+    "https://jetbrains.bintray.com/jediterm",
+    "https://jitpack.io",
+    "https://maven.exasol.com/artifactory/exasol-releases",
+    "https://plugins.gradle.org/m2",
+    "https://plugins.jetbrains.com/maven",
+    "https://repo.grails.org/grails/core",
+    "https://repo.jenkins-ci.org/releases",
+    "https://repo.spring.io/milestone",
+    "https://repo1.maven.org/maven2",
+    "https://services.gradle.org",
+    "https://www.jetbrains.com/intellij-repository",
+    "https://www.myget.org/F/intellij-go-snapshots/maven",
+    "https://www.myget.org/F/rd-snapshots/maven",
+    "https://www.myget.org/F/rd-model-snapshots/maven",
+    "https://www.python.org/ftp",
+    "https://dl.google.com/dl/android/studio/ide-zips",
+    "https://dl.bintray.com/kotlin/ktor",
+    "https://cdn.azul.com/zulu/bin"
+)
+
+val aliases = mapOf(
+    "https://repo.maven.apache.org/maven2" to "https://repo1.maven.org/maven2" // Maven Central
+)
+
+val isTeamcityBuild = project.hasProperty("teamcity") || System.getenv("TEAMCITY_VERSION") != null
+
+fun Project.cacheRedirectorEnabled(): Boolean = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
+
+fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path")
+
+fun URI.maybeRedirect(): URI {
+    val url = toString().trimEnd('/')
+    val dealiasedUrl = aliases.getOrDefault(url, url)
+
+    return if (mirroredUrls.any { dealiasedUrl.startsWith(it) }) {
+        URI(dealiasedUrl).toCacheRedirectorUri()
+    } else {
+        this
+    }
+}
+
+fun RepositoryHandler.redirect() {
+    for (repository in this) {
+        when (repository) {
+            is MavenArtifactRepository -> repository.url = repository.url.maybeRedirect()
+            is IvyArtifactRepository -> if (repository.url != null) {
+                repository.url = repository.url.maybeRedirect()
+            }
+        }
+    }
+}
+
+// teamcity.jetbrains.com is located in the same local network with build agents
+fun URI.isCachedOrLocal() = host == "cache-redirector.jetbrains.com" || scheme == "file" || host == "teamcity.jetbrains.com"
+
+fun RepositoryHandler.findNonCachedRepositories(): List<String> {
+    val mavenNonCachedRepos = filterIsInstance<MavenArtifactRepository>()
+        .filterNot { it.url.isCachedOrLocal() }
+        .map { it.url.toString() }
+
+    val ivyNonCachedRepos = filterIsInstance<IvyArtifactRepository>()
+        .filterNot { it.url.isCachedOrLocal() }
+        .map { it.url.toString() }
+
+    return mavenNonCachedRepos + ivyNonCachedRepos
+}
+
+fun escape(s: String): String {
+    return s.replace("[\\|'\\[\\]]".toRegex(), "\\|$0").replace("\n".toRegex(), "|n").replace("\r".toRegex(), "|r")
+}
+
+fun testStarted(testName: String) {
+    println("##teamcity[testStarted name='%s']".format(escape(testName)))
+}
+
+fun testFinished(testName: String) {
+    println("##teamcity[testFinished name='%s']".format(escape(testName)))
+}
+
+fun testFailed(name: String, message: String, details: String) {
+    println("##teamcity[testFailed name='%s' message='%s' details='%s']".format(escape(name), escape(message), escape(details)))
+}
+
+fun Task.logNonCachedRepo(testName: String, repoUrl: String) {
+    val msg = "Repository $repoUrl in ${project.displayName} should be cached with cache-redirector"
+    val details = "Using non cached repository may lead to download failures in CI builds." +
+            " Check https://github.com/JetBrains/kotlin/blob/master/gradle/cacheRedirector.gradle.kts for details."
+
+    if (isTeamcityBuild) {
+        testFailed(testName, msg, details)
+    }
+
+    logger.warn("WARNING - $msg\n$details")
+}
+
+fun Task.logInvalidIvyRepo(testName: String) {
+    val msg = "Invalid ivy repo found in ${project.displayName}"
+    val details = "Url must be not null"
+
+    if (isTeamcityBuild) {
+        testFailed(testName, msg, details)
+    }
+
+    logger.warn("WARNING - $msg: $details")
+}
+
+val checkRepositories: TaskProvider<Task> = tasks.register("checkRepositories") {
+    doLast {
+        val testName = "$name in ${project.displayName}"
+        if (isTeamcityBuild) {
+            testStarted(testName)
+        }
+
+        repositories.filterIsInstance<IvyArtifactRepository>().forEach {
+            if (it.url == null) {
+                logInvalidIvyRepo(testName)
+            }
+        }
+
+        repositories.findNonCachedRepositories().forEach {
+            logNonCachedRepo(testName, it)
+        }
+
+        buildscript.repositories.findNonCachedRepositories().forEach {
+            logNonCachedRepo(testName, it)
+        }
+
+        if (isTeamcityBuild) {
+            testFinished(testName)
+        }
+    }
+}
+
+tasks.findByName("checkBuild")?.dependsOn(checkRepositories)
+
+if (cacheRedirectorEnabled()) {
+    logger.info("Redirecting repositories for $displayName")
+    repositories.redirect()
+}

settings.gradle

@@ -6,12 +6,15 @@ pluginManagement {
                 url pluginRepo
             }
         }
-        gradlePluginPortal()
-    }
 
-    if (cacheRedirectorEnabled == 'true') {
+        if (cacheRedirectorEnabled == 'true') {
-        logger.info("Redirecting repositories for settings.gradle pluginManagement")
+            logger.info("Using cache redirector for settings.gradle pluginManagement")
-        JbCacheRedirectorKt.redirect(repositories)
+            maven {
+                url "https://cache-redirector.jetbrains.com/plugins.gradle.org/m2"
+            }
+        } else {
+            gradlePluginPortal()
+        }
     }
 }