azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build: Extract cache redirector to script & add verification task

Browse Source
This commit is contained in:
Vyacheslav Gerasimov
2019-05-21 20:41:26 +03:00
parent 2bbf92a6c3
commit b88d9d62b8
8 changed files with 255 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.gradle.kts
+13 -5
View File
@@ -11,9 +11,15 @@ buildscript {
// when updating please also update JPS artifacts configuration: https://jetbrains.quip.com/zzGUAYSJ6gv3/JPS-Build-update-bootstrap
kotlinBootstrapFrom(BootstrapOption.TeamCity("1.3.50-dev-397", onlySuccessBootstrap = false))
repositories.withRedirector(project) {
repositories {
bootstrapKotlinRepo?.let(::maven)
maven("https://plugins.gradle.org/m2")
val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
if (cacheRedirectorEnabled) {
maven("https://cache-redirector.jetbrains.com/plugins.gradle.org/m2")
} else {
maven("https://plugins.gradle.org/m2")
}
}
// a workaround for kotlin compiler classpath in kotlin project: sometimes gradle substitutes
@@ -410,10 +416,12 @@ allprojects {
}
}
if (cacheRedirectorEnabled()) {
logger.info("Redirecting repositories for $displayName")
repositories.redirect()
// Aggregate task for build related checks
tasks.register("checkBuild") {
tasks.findByName("check")?.dependsOn(this)
}
apply(from = "$rootDir/gradle/cacheRedirector.gradle.kts")
}
}
buildSrc/build.gradle.kts
+14 -10
View File
@@ -1,15 +1,17 @@
extra["versions.shadow"] = "4.0.3"
extra["versions.native-platform"] = "0.14"
buildscript {
val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
val buildSrcKotlinVersion: String by extra(findProperty("buildSrc.kotlin.version")?.toString() ?: embeddedKotlinVersion)
val buildSrcKotlinRepo: String? by extra(findProperty("buildSrc.kotlin.repo") as String?)
extra["versions.shadow"] = "4.0.3"
extra["versions.native-platform"] = "0.14"
repositories {
if (cacheRedirectorEnabled) {
maven("https://cache-redirector.jetbrains.com/jcenter.bintray.com")
} else {
jcenter()
}
buildSrcKotlinRepo?.let {
@@ -77,18 +79,13 @@ extra["versions.androidDxSources"] = "5.0.0_r2"
extra["customDepsOrg"] = "kotlin.build"
repositories {
if (cacheRedirectorEnabled) {
maven("https://cache-redirector.jetbrains.com/jcenter.bintray.com")
maven("https://cache-redirector.jetbrains.com/jetbrains.bintray.com/intellij-third-party-dependencies/")
}
maven("https://jetbrains.bintray.com/intellij-third-party-dependencies/")
maven("https://plugins.gradle.org/m2/")
jcenter()
extra["buildSrcKotlinRepo"]?.let {
maven(url = it)
}
jcenter()
maven("https://jetbrains.bintray.com/intellij-third-party-dependencies/")
maven("https://plugins.gradle.org/m2/")
}
dependencies {
@@ -112,3 +109,10 @@ fun Project.`samWithReceiver`(configure: org.jetbrains.kotlin.samWithReceiver.gr
extensions.configure("samWithReceiver", configure)
tasks["build"].dependsOn(":prepare-deps:build")
allprojects {
afterEvaluate {
apply(from = "$rootDir/../gradle/cacheRedirector.gradle.kts")
tasks["check"].dependsOn(tasks["checkRepositories"])
}
}
buildSrc/prepare-deps/build.gradle.kts
+33 -16
View File
@@ -5,8 +5,12 @@ import org.gradle.api.publish.ivy.internal.publication.DefaultIvyConfiguration
import org.gradle.api.publish.ivy.internal.publication.DefaultIvyPublicationIdentity
import org.gradle.api.publish.ivy.internal.publisher.IvyDescriptorFileGenerator
import org.gradle.internal.os.OperatingSystem
import java.net.URI
plugins {
base
}
val cacheRedirectorEnabled = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
val verifyDependencyOutput: Boolean by rootProject.extra
val intellijUltimateEnabled: Boolean by rootProject.extra
val intellijReleaseType: String by rootProject.extra
@@ -65,11 +69,12 @@ val androidToolsOs by lazy {
repositories {
if (androidStudioRelease != null) {
ivy {
if (cacheRedirectorEnabled) {
artifactPattern("https://cache-redirector.jetbrains.com/dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease/[artifact]-[revision]-$androidStudioOs.[ext]")
url = URI("https://dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease")
patternLayout {
artifact("[artifact]-[revision]-$androidStudioOs.[ext]")
}
artifactPattern("https://dl.google.com/dl/android/studio/ide-zips/$androidStudioRelease/[artifact]-[revision]-$androidStudioOs.[ext]")
metadataSources {
artifact()
}
@@ -77,17 +82,28 @@ repositories {
}
ivy {
artifactPattern("https://dl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]")
artifactPattern("https://android.googlesource.com/platform/dalvik/+archive/android-$androidDxSourcesVersion/[artifact].[ext]")
url = URI("https://dl.google.com/android/repository")
patternLayout {
artifact("[artifact]_[revision](-[classifier]).[ext]")
}
metadataSources {
artifact()
}
}
if (cacheRedirectorEnabled) {
maven("https://cache-redirector.jetbrains.com/www.jetbrains.com/intellij-repository/$intellijReleaseType")
maven("https://cache-redirector.jetbrains.com/plugins.jetbrains.com/maven")
maven("https://cache-redirector.jetbrains.com/jetbrains.bintray.com/intellij-third-party-dependencies/")
ivy {
url = URI("https://android.googlesource.com/platform/dalvik/+archive/android-$androidDxSourcesVersion")
patternLayout {
artifact("[artifact].[ext]")
}
metadataSources {
artifact()
}
}
maven("https://www.jetbrains.com/intellij-repository/$intellijReleaseType")
@@ -272,19 +288,20 @@ val makeIde = if (androidStudioBuild != null) {
task
}
val build by tasks.creating {
val buildJpsStandalone = buildIvyRepositoryTask(jpsStandalone, customDepsOrg, customDepsRepoDir, null, sourcesFile)
val buildNodeJsPlugin = buildIvyRepositoryTask(nodeJSPlugin, customDepsOrg, customDepsRepoDir, ::skipToplevelDirectory, sourcesFile)
tasks.named("build") {
dependsOn(
makeIntellijCore,
makeIde,
buildIvyRepositoryTask(jpsStandalone, customDepsOrg, customDepsRepoDir, null, sourcesFile),
buildJpsStandalone,
makeIntellijAnnotations,
buildIvyRepoForAndroidDx
)
if (installIntellijUltimate) {
dependsOn(
buildIvyRepositoryTask(nodeJSPlugin, customDepsOrg, customDepsRepoDir, ::skipToplevelDirectory, sourcesFile)
)
dependsOn(buildNodeJsPlugin)
}
}
@@ -294,7 +311,7 @@ tasks.create("cleanLegacy", Delete::class.java) {
delete("$projectDir/intellij-sdk")
}
tasks.create("clean", Delete::class.java) {
tasks.named<Delete>("clean") {
delete(customDepsRepoDir)
}
buildSrc/src/main/kotlin/jbCacheRedirector.kt
-63
View File
@@ -1,63 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
import org.gradle.api.Project
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.artifacts.repositories.MavenArtifactRepository
import java.net.URI
// https://youtrack.jetbrains.com/issue/ADM-23180
val mirroredUrls = listOf(
"https://dl.bintray.com/groovy/maven",
"https://dl.bintray.com/kotlin/kotlin-dev",
"https://dl.bintray.com/kotlin/kotlin-eap",
"https://dl.google.com/dl/android/maven2",
"https://dl.google.com/go",
"https://download.jetbrains.com",
"https://jcenter.bintray.com",
"https://jetbrains.bintray.com/dekaf",
"https://jetbrains.bintray.com/intellij-jdk",
"https://jetbrains.bintray.com/intellij-plugin-service",
"https://jetbrains.bintray.com/intellij-third-party-dependencies",
"https://jetbrains.bintray.com/markdown",
"https://jetbrains.bintray.com/teamcity-rest-client",
"https://jetbrains.bintray.com/test-discovery",
"https://jetbrains.bintray.com/jediterm",
"https://jitpack.io",
"https://maven.exasol.com/artifactory/exasol-releases",
"https://plugins.gradle.org/m2",
"https://plugins.jetbrains.com/maven",
"https://repo.grails.org/grails/core",
"https://repo.jenkins-ci.org/releases",
"https://repo.spring.io/milestone",
"https://repo1.maven.org/maven2",
"https://services.gradle.org",
"https://www.jetbrains.com/intellij-repository",
"https://www.myget.org/F/intellij-go-snapshots/maven",
"https://www.myget.org/F/rd-snapshots/maven",
"https://www.myget.org/F/rd-model-snapshots/maven",
"https://www.python.org/ftp",
"https://dl.google.com/dl/android/studio/ide-zips",
"https://dl.bintray.com/kotlin/ktor",
"https://cdn.azul.com/zulu/bin"
)
fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path")
fun RepositoryHandler.redirect() = filterIsInstance<MavenArtifactRepository>().forEach { repository ->
val uri = repository.url
if (mirroredUrls.any { uri.toString().trimEnd('/').startsWith(it) }) {
repository.url = uri.toCacheRedirectorUri()
}
}
fun Project.cacheRedirectorEnabled(): Boolean = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
fun RepositoryHandler.withRedirector(project: Project, configuration: RepositoryHandler.() -> Unit) {
configuration()
if (project.cacheRedirectorEnabled()) {
redirect()
}
}
buildSrc/src/main/kotlin/localDependencies.kt
+11 -9
View File
@@ -42,17 +42,19 @@ private fun Project.ideModuleVersion() = when (IdeVersionConfigurator.currentIde
fun RepositoryHandler.kotlinBuildLocalRepo(project: Project): IvyArtifactRepository = ivy {
val baseDir = project.kotlinBuildLocalRepoDir()
setUrl(baseDir)
url = baseDir.toURI()
ivyPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/[module].ivy.xml")
ivyPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/ivy/[module].ivy.xml")
ivyPattern("${baseDir.canonicalPath}/[organisation]/${project.ideModuleName()}/[revision]/ivy/[module].ivy.xml") // bundled plugins
patternLayout {
ivy("[organisation]/[module]/[revision]/[module].ivy.xml")
ivy("[organisation]/[module]/[revision]/ivy/[module].ivy.xml")
ivy("[organisation]/${project.ideModuleName()}/[revision]/ivy/[module].ivy.xml") // bundled plugins
artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/artifacts/lib/[artifact](-[classifier]).[ext]")
artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/artifacts/[artifact](-[classifier]).[ext]")
artifactPattern("${baseDir.canonicalPath}/[organisation]/${project.ideModuleName()}/[revision]/artifacts/plugins/[module]/lib/[artifact](-[classifier]).[ext]") // bundled plugins
artifactPattern("${baseDir.canonicalPath}/[organisation]/sources/[artifact]-[revision](-[classifier]).[ext]")
artifactPattern("${baseDir.canonicalPath}/[organisation]/[module]/[revision]/[artifact](-[classifier]).[ext]")
artifact("[organisation]/[module]/[revision]/artifacts/lib/[artifact](-[classifier]).[ext]")
artifact("[organisation]/[module]/[revision]/artifacts/[artifact](-[classifier]).[ext]")
artifact("[organisation]/${project.ideModuleName()}/[revision]/artifacts/plugins/[module]/lib/[artifact](-[classifier]).[ext]") // bundled plugins
artifact("[organisation]/sources/[artifact]-[revision](-[classifier]).[ext]")
artifact("[organisation]/[module]/[revision]/[artifact](-[classifier]).[ext]")
}
metadataSources {
ivyDescriptor()
dependencies/android-sdk/build.gradle.kts
Vendored
+7 -6
View File
@@ -1,16 +1,17 @@
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import java.io.File
import org.gradle.internal.os.OperatingSystem
import java.net.URI
// TODO: consider adding dx sources (the only jar used on the compile time so far)
// e.g. from "https://android.googlesource.com/platform/dalvik/+archive/android-5.0.0_r2/dx.tar.gz"
repositories {
ivy {
artifactPattern("https://dl-ssl.google.com/android/repository/[artifact]-[revision].[ext]")
artifactPattern("https://dl-ssl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]")
artifactPattern("https://dl.google.com/android/repository/[artifact]_[revision](-[classifier]).[ext]")
url = URI("https://dl.google.com/android/repository")
patternLayout {
artifact("[artifact]-[revision].[ext]")
artifact("[artifact]_[revision](-[classifier]).[ext]")
artifact("[artifact]_[revision](-[classifier]).[ext]")
}
}
}
gradle/cacheRedirector.gradle.kts
+169
View File
@@ -0,0 +1,169 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
import org.gradle.api.Project
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.artifacts.repositories.MavenArtifactRepository
import java.net.URI
/**
* The list of repositories supported by cache redirector should be synced with the list in https://youtrack.jetbrains.com/issue/ADM-23180
* To add a repository to the list create an issue in ADM project (example issue https://youtrack.jetbrains.com/issue/IJI-149)
* Repositories in `buildscript` blocks are *NOT* substituted by this script and should be handled manually
*/
val mirroredUrls = listOf(
"https://dl.bintray.com/groovy/maven",
"https://dl.bintray.com/kotlin/kotlin-dev",
"https://dl.bintray.com/kotlin/kotlin-eap",
"https://dl.google.com/dl/android/maven2",
"https://dl.google.com/go",
"https://download.jetbrains.com",
"https://jcenter.bintray.com",
"https://jetbrains.bintray.com/dekaf",
"https://jetbrains.bintray.com/intellij-jdk",
"https://jetbrains.bintray.com/intellij-plugin-service",
"https://jetbrains.bintray.com/intellij-third-party-dependencies",
"https://jetbrains.bintray.com/markdown",
"https://jetbrains.bintray.com/teamcity-rest-client",
"https://jetbrains.bintray.com/test-discovery",
"https://jetbrains.bintray.com/jediterm",
"https://jitpack.io",
"https://maven.exasol.com/artifactory/exasol-releases",
"https://plugins.gradle.org/m2",
"https://plugins.jetbrains.com/maven",
"https://repo.grails.org/grails/core",
"https://repo.jenkins-ci.org/releases",
"https://repo.spring.io/milestone",
"https://repo1.maven.org/maven2",
"https://services.gradle.org",
"https://www.jetbrains.com/intellij-repository",
"https://www.myget.org/F/intellij-go-snapshots/maven",
"https://www.myget.org/F/rd-snapshots/maven",
"https://www.myget.org/F/rd-model-snapshots/maven",
"https://www.python.org/ftp",
"https://dl.google.com/dl/android/studio/ide-zips",
"https://dl.bintray.com/kotlin/ktor",
"https://cdn.azul.com/zulu/bin"
)
val aliases = mapOf(
"https://repo.maven.apache.org/maven2" to "https://repo1.maven.org/maven2" // Maven Central
)
val isTeamcityBuild = project.hasProperty("teamcity") || System.getenv("TEAMCITY_VERSION") != null
fun Project.cacheRedirectorEnabled(): Boolean = findProperty("cacheRedirectorEnabled")?.toString()?.toBoolean() == true
fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path")
fun URI.maybeRedirect(): URI {
val url = toString().trimEnd('/')
val dealiasedUrl = aliases.getOrDefault(url, url)
return if (mirroredUrls.any { dealiasedUrl.startsWith(it) }) {
URI(dealiasedUrl).toCacheRedirectorUri()
} else {
this
}
}
fun RepositoryHandler.redirect() {
for (repository in this) {
when (repository) {
is MavenArtifactRepository -> repository.url = repository.url.maybeRedirect()
is IvyArtifactRepository -> if (repository.url != null) {
repository.url = repository.url.maybeRedirect()
}
}
}
}
// teamcity.jetbrains.com is located in the same local network with build agents
fun URI.isCachedOrLocal() = host == "cache-redirector.jetbrains.com" || scheme == "file" || host == "teamcity.jetbrains.com"
fun RepositoryHandler.findNonCachedRepositories(): List<String> {
val mavenNonCachedRepos = filterIsInstance<MavenArtifactRepository>()
.filterNot { it.url.isCachedOrLocal() }
.map { it.url.toString() }
val ivyNonCachedRepos = filterIsInstance<IvyArtifactRepository>()
.filterNot { it.url.isCachedOrLocal() }
.map { it.url.toString() }
return mavenNonCachedRepos + ivyNonCachedRepos
}
fun escape(s: String): String {
return s.replace("[\\|'\\[\\]]".toRegex(), "\\|$0").replace("\n".toRegex(), "|n").replace("\r".toRegex(), "|r")
}
fun testStarted(testName: String) {
println("##teamcity[testStarted name='%s']".format(escape(testName)))
}
fun testFinished(testName: String) {
println("##teamcity[testFinished name='%s']".format(escape(testName)))
}
fun testFailed(name: String, message: String, details: String) {
println("##teamcity[testFailed name='%s' message='%s' details='%s']".format(escape(name), escape(message), escape(details)))
}
fun Task.logNonCachedRepo(testName: String, repoUrl: String) {
val msg = "Repository $repoUrl in ${project.displayName} should be cached with cache-redirector"
val details = "Using non cached repository may lead to download failures in CI builds." +
" Check https://github.com/JetBrains/kotlin/blob/master/gradle/cacheRedirector.gradle.kts for details."
if (isTeamcityBuild) {
testFailed(testName, msg, details)
}
logger.warn("WARNING - $msg\n$details")
}
fun Task.logInvalidIvyRepo(testName: String) {
val msg = "Invalid ivy repo found in ${project.displayName}"
val details = "Url must be not null"
if (isTeamcityBuild) {
testFailed(testName, msg, details)
}
logger.warn("WARNING - $msg: $details")
}
val checkRepositories: TaskProvider<Task> = tasks.register("checkRepositories") {
doLast {
val testName = "$name in ${project.displayName}"
if (isTeamcityBuild) {
testStarted(testName)
}
repositories.filterIsInstance<IvyArtifactRepository>().forEach {
if (it.url == null) {
logInvalidIvyRepo(testName)
}
}
repositories.findNonCachedRepositories().forEach {
logNonCachedRepo(testName, it)
}
buildscript.repositories.findNonCachedRepositories().forEach {
logNonCachedRepo(testName, it)
}
if (isTeamcityBuild) {
testFinished(testName)
}
}
}
tasks.findByName("checkBuild")?.dependsOn(checkRepositories)
if (cacheRedirectorEnabled()) {
logger.info("Redirecting repositories for $displayName")
repositories.redirect()
}
settings.gradle
+8 -5
View File
@@ -6,12 +6,15 @@ pluginManagement {
url pluginRepo
}
}
gradlePluginPortal()
}
if (cacheRedirectorEnabled == 'true') {
logger.info("Redirecting repositories for settings.gradle pluginManagement")
JbCacheRedirectorKt.redirect(repositories)
if (cacheRedirectorEnabled == 'true') {
logger.info("Using cache redirector for settings.gradle pluginManagement")
maven {
url "https://cache-redirector.jetbrains.com/plugins.gradle.org/m2"
}
} else {
gradlePluginPortal()
}
}
}