azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move to Artifactory usage (#3513)

Browse Source
This commit is contained in:
LepilkinaElena
2019-10-29 18:46:14 +03:00
committed by GitHub
parent 1f33b27839
commit 85a812064b
10 changed files with 81 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HACKING.md
+4 -4
View File
@@ -153,15 +153,15 @@ and then a final native binary is produced from this klibrary using the -Xinclud
Output can be redirected to file with flag `--output/-o`.
To get detailed information about supported options, please use `--help/-h`.
Analyzer tool can compare both local files and files placed on Bintray/TeamCity.
Analyzer tool can compare both local files and files placed on Artifactory/TeamCity.
File description stored on Bintray
File description stored on Artifactory
bintray:<build number>:<target (Linux|Windows10|MacOSX)>:<filename>
artifactory:<build number>:<target (Linux|Windows10|MacOSX)>:<filename>
Example
bintray:1.2-dev-7942:Windows10:nativeReport.json
artifactory:1.2-dev-7942:Windows10:nativeReport.json
File description stored on TeamCity
build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt
+5 -7
View File
@@ -89,11 +89,10 @@ open class BuildRegister : DefaultTask() {
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val bintrayUser = buildProperties.getProperty("bintray.user")
val bintrayApiKey = buildProperties.getProperty("bintray.apikey")
val teamCityUser = buildProperties.getProperty("teamcity.auth.userId")
val teamCityPassword = buildProperties.getProperty("teamcity.auth.password")
val buildNumber = buildProperties.getProperty("build.number")
val apiKey = buildProperties.getProperty("artifactory.apikey")
// Get branch.
val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword)
@@ -104,8 +103,8 @@ open class BuildRegister : DefaultTask() {
// Get summary information.
val output = arrayOf("$analyzer", "summary", "--exec-samples", "all", "--compile", "samples",
"--compile-samples", "HelloWorld,Videoplayer", "--codesize-samples", "all",
"--exec-normalize", "bintray:goldenResults.csv",
"--codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
"--exec-normalize", "artifactory:builds/goldenResults.csv",
"--codesize-normalize", "artifactory:builds/goldenResults.csv", "$currentBenchmarksReportFile")
.runCommand()
// Postprocess information.
@@ -131,7 +130,7 @@ open class BuildRegister : DefaultTask() {
val frameworkOutput = arrayOf("$analyzer", "summary", "--compile", "samples",
"--compile-samples", "FrameworkBenchmarksAnalyzer", "--codesize-samples", "FrameworkBenchmarksAnalyzer",
"--codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
"--codesize-normalize", "artifactory:builds/goldenResults.csv", "$currentBenchmarksReportFile")
.runCommand()
val buildInfoPartsFramework = frameworkOutput.split(',')
@@ -154,8 +153,7 @@ open class BuildRegister : DefaultTask() {
append("{\"buildId\":\"$buildId\",")
append("\"teamCityUser\":\"$teamCityUser\",")
append("\"teamCityPassword\":\"$teamCityPassword\",")
append("\"bintrayUser\": \"$bintrayUser\", ")
append("\"bintrayPassword\":\"$bintrayApiKey\", ")
append("\"artifactoryApiKey\":\"$apiKey\",")
append("\"target\": \"$target\",")
append("\"buildType\": \"$buildType\",")
append("\"failuresNumber\": $failures,")
build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt
+9 -6
View File
@@ -141,13 +141,14 @@ fun getCompileOnlyBenchmarksOpts(project: Project, defaultCompilerOpts: List<Str
fun findFile(fileName: String, directory: String): String? =
File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()
fun uploadFileToBintray(url: String, project: String, version: String, packageName: String, bintrayFilePath: String,
filePath: String, username: String? = null, password: String? = null) {
val uploadUrl = "$url/$project/$packageName/$version/$bintrayFilePath?publish=1"
sendUploadRequest(uploadUrl, filePath, username, password)
fun uploadFileToArtifactory(url: String, project: String, artifactoryFilePath: String,
filePath: String, password: String) {
val uploadUrl = "$url/$project/$artifactoryFilePath"
sendUploadRequest(uploadUrl, filePath, extraHeaders = listOf(Pair("X-JFrog-Art-Api", password)))
}
fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null) {
fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null,
extraHeaders: List<Pair<String, String>> = emptyList()) {
val uploadingFile = File(fileName)
val connection = URL(url).openConnection() as HttpURLConnection
connection.doOutput = true
@@ -158,7 +159,9 @@ fun sendUploadRequest(url: String, fileName: String, username: String? = null, p
val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
extraHeaders.forEach {
connection.addRequestProperty(it.first, it.second)
}
try {
connection.connect()
BufferedOutputStream(connection.outputStream).use { output ->
build-tools/src/main/kotlin/org/jetbrains/kotlin/RegressionsReporter.kt
+7 -7
View File
@@ -125,8 +125,8 @@ open class RegressionsReporter : DefaultTask() {
}
}
// File name on bintray is the same as current.
val bintrayFileName = currentBenchmarksReportFile.substringAfterLast("/")
// File name on Artifactory is the same as current.
val artifactoryFileName = currentBenchmarksReportFile.substringAfterLast("/")
// Get compare to build.
val compareToBuild = getBuild(previousBuildLocator(buildTypeId, defaultBranch), user, password)
@@ -135,21 +135,21 @@ open class RegressionsReporter : DefaultTask() {
val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
// Generate comparison report.
val output = arrayOf("$analyzer", "-r", "html", "$currentBenchmarksReportFile", "bintray:$compareToBuildNumber:$target:$bintrayFileName", "-o", "$htmlReport")
val output = arrayOf("$analyzer", "-r", "html", "$currentBenchmarksReportFile", "artifactory:$compareToBuildNumber:$target:$artifactoryFileName", "-o", "$htmlReport")
.runCommand()
if (output.contains("Uncaught exception")) {
error("Error during comparasion of $currentBenchmarksReportFile and " +
"bintray:$compareToBuildNumber:$target:$bintrayFileName with $analyzer! " +
"artifactory:$compareToBuildNumber:$target:$artifactoryFileName with $analyzer! " +
"Please check files existance and their correctness.")
}
arrayOf("$analyzer", "-r", "statistics", "$currentBenchmarksReportFile", "bintray:$compareToBuildNumber:$target:$bintrayFileName", "-o", "$summaryFile")
arrayOf("$analyzer", "-r", "statistics", "$currentBenchmarksReportFile", "artifactory:$compareToBuildNumber:$target:$artifactoryFileName", "-o", "$summaryFile")
.runCommand()
val reportLink = "https://kotlin-native-perf-summary.labs.jb.gg/?target=$target&build=$buildNumber"
val detailedReportLink = "https://kotlin-native-performance.labs.jb.gg/?" +
"report=bintray:$buildNumber:$target:$bintrayFileName&" +
"compareTo=bintray:$compareToBuildNumber:$target:$bintrayFileName"
"report=artifactory:$buildNumber:$target:$artifactoryFileName&" +
"compareTo=artifactory:$compareToBuildNumber:$target:$artifactoryFileName"
val title = "\n*Performance report for target $target (build $buildNumber)* - $reportLink\n" +
"*Detailed info - * $detailedReportLink"
performance/KotlinVsSwift/build.gradle
+2 -2
View File
@@ -35,14 +35,14 @@ defaultTasks 'konanRun'
task mergeNativeReports {
doLast {
mergeReports(nativeJson)
uploadBenchmarkResultToBintray(nativeJson)
uploadBenchmarkResultToArtifactory(nativeJson)
}
}
task mergeJvmReports {
doLast {
mergeReports(jvmJson)
uploadBenchmarkResultToBintray(jvmJson)
uploadBenchmarkResultToArtifactory(jvmJson)
}
}
performance/build.gradle
+7 -9
View File
@@ -64,18 +64,16 @@ task slackSummary(type: RegressionsSummaryReporter) {
'Windows': "${rootBuildDirectory}/targetsResults/Windows 10.txt".toString()]
}
private def uploadBenchmarkResultToBintray(String fileName) {
private def uploadBenchmarkResultToArtifactory(String fileName) {
def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")
if (teamcityConfig) {
def buildProperties = new Properties()
buildProperties.load(new FileInputStream(teamcityConfig))
def user = buildProperties.getProperty("bintray.user")
def password = buildProperties.getProperty("bintray.apikey")
def password = buildProperties.getProperty("artifactory.apikey")
def buildNumber = buildProperties.getProperty("build.number")
def target = System.getProperty("os.name").replaceAll("\\s", "")
MPPTools.uploadFileToBintray("${bintrayUrl}", "${bintrayRepo}",
buildNumber, "${bintrayPackage}", "${target}/${buildNumber}/${fileName}",
"${buildDir.absolutePath}/${fileName}", user, password)
MPPTools.uploadFileToArtifactory("${artifactoryUrl}", "${artifactoryRepo}",
"${target}/${buildNumber}/${fileName}", "${buildDir.absolutePath}/${fileName}", password)
}
}
@@ -126,7 +124,7 @@ task registerExternalBenchmarks {
if (!merged.isEmpty()) {
mkdir buildDir.absolutePath
new File(buildDir, externalBenchmarksReport).write(merged)
uploadBenchmarkResultToBintray(externalBenchmarksReport)
uploadBenchmarkResultToArtifactory(externalBenchmarksReport)
}
}
}
@@ -164,14 +162,14 @@ def mergeReports(String fileName) {
task mergeNativeReports {
doLast {
mergeReports(nativeJson)
uploadBenchmarkResultToBintray(nativeJson)
uploadBenchmarkResultToArtifactory(nativeJson)
}
}
task mergeJvmReports {
doLast {
mergeReports(jvmJson)
uploadBenchmarkResultToBintray(jvmJson)
uploadBenchmarkResultToArtifactory(jvmJson)
}
}
performance/gradle.properties
+2 -3
View File
@@ -12,8 +12,7 @@ jvmJson = jvmReport.json
analyzerTool = benchmarksAnalyzer
analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin
outputReport = ../report/report.html
bintrayUrl = https://api.bintray.com/content/lepilkinaelena
bintrayRepo = KotlinNativePerformance
bintrayPackage = jsonReports
artifactoryUrl = https://repo.labs.intellij.net
artifactoryRepo = kotlin-native-benchmarks
externalReports = coroutinesReport.txt
externalBenchmarksReport = externalReport.json
tools/benchmarksAnalyzer/src/main/kotlin/main.kt
+9 -9
View File
@@ -31,27 +31,27 @@ abstract class Connector {
abstract fun getFileContent(fileLocation: String, user: String? = null): String
}
object BintrayConnector : Connector() {
override val connectorPrefix = "bintray:"
val bintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
object ArtifactoryConnector : Connector() {
override val connectorPrefix = "artifactory:"
val artifactoryUrl = "https://repo.labs.intellij.net/kotlin-native-benchmarks"
override fun getFileContent(fileLocation: String, user: String?): String {
val fileParametersSize = 3
val fileDescription = fileLocation.substringAfter(connectorPrefix)
val fileParameters = fileDescription.split(':', limit = fileParametersSize)
// Right link to bintray file.
// Right link to Artifactory file.
if (fileParameters.size == 1) {
val accessFileUrl = "$bintrayUrl/${fileParameters[0]}"
val accessFileUrl = "$artifactoryUrl/${fileParameters[0]}"
return sendGetRequest(accessFileUrl, followLocation = true)
}
// Used builds description format.
if (fileParameters.size != fileParametersSize) {
error("To get file from bintray, please, specify, build number from TeamCity and target" +
" in format bintray:build_number:target:filename")
error("To get file from Artifactory, please, specify, build number from TeamCity and target" +
" in format artifactory:build_number:target:filename")
}
val (buildNumber, target, fileName) = fileParameters
val accessFileUrl = "$bintrayUrl/$target/$buildNumber/$fileName"
val accessFileUrl = "$artifactoryUrl/$target/$buildNumber/$fileName"
return sendGetRequest(accessFileUrl, followLocation = true)
}
}
@@ -78,7 +78,7 @@ object TeamCityConnector: Connector() {
fun getFileContent(fileName: String, user: String? = null): String {
return when {
BintrayConnector.isCompatible(fileName) -> BintrayConnector.getFileContent(fileName, user)
ArtifactoryConnector.isCompatible(fileName) -> ArtifactoryConnector.getFileContent(fileName, user)
TeamCityConnector.isCompatible(fileName) -> TeamCityConnector.getFileContent(fileName, user)
else -> readFile(fileName)
}
tools/performance-server/src/main/kotlin/routes/route.kt
+34 -29
View File
@@ -21,16 +21,17 @@ import org.jetbrains.report.json.*
import org.jetbrains.build.Build
const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest"
const val downloadBintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
const val uploadBintrayUrl = "https://api.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
const val artifactoryUrl = "https://repo.labs.intellij.net/kotlin-native-benchmarks"
const val buildsFileName = "buildsSummary.csv"
const val goldenResultsFileName = "goldenResults.csv"
const val bintrayPackage = "builds"
const val artifactoryBuildsDirectory = "builds"
const val buildsInfoPartsNumber = 11
operator fun <K, V> Map<K, V>?.get(key: K) = this?.get(key)
// Local cache for saving information about builds got from Bintray.
fun getArtifactoryHeader(artifactoryApiKey: String) = Pair("X-JFrog-Art-Api", artifactoryApiKey)
// Local cache for saving information about builds got from Artifactory.
object LocalCache {
private val knownTargets = listOf("Linux", "MacOSX", "Windows10")
private val buildsInfo = mutableMapOf<String, MutableMap<String, String>>()
@@ -43,7 +44,7 @@ object LocalCache {
fun fill(onlyTarget: String? = null) {
onlyTarget?.let {
val buildsDescription = getBuildsInfoFromBintray(onlyTarget).lines().drop(1)
val buildsDescription = getBuildsInfoFromArtifactory(onlyTarget).lines().drop(1)
buildsInfo[onlyTarget] = mutableMapOf<String, String>()
buildsDescription.forEach {
if (!it.isEmpty()) {
@@ -65,19 +66,19 @@ object LocalCache {
fun buildExists(target: String, buildNumber: String) =
buildsInfo[target][buildNumber]?.let { true } ?: false
fun delete(target: String, builds: Iterable<String>, bintrayUser: String, bintrayPassword: String): Boolean {
// Delete from bintray.
val buildsDescription = getBuildsInfoFromBintray(target).lines()
fun delete(target: String, builds: Iterable<String>, apiKey: String): Boolean {
// Delete from Artifactory.
val buildsDescription = getBuildsInfoFromArtifactory(target).lines()
val newBuildsDescription = buildsDescription.filter {
val buildNumber = it.substringBefore(',')
buildNumber !in builds
}
if (newBuildsDescription.size < buildsDescription.size) {
// Upload new version of file.
val uploadUrl = "$uploadBintrayUrl/$bintrayPackage/latest/$target/$buildsFileName?publish=1&override=1"
sendUploadRequest(uploadUrl, newBuildsDescription.joinToString("\n"), bintrayUser, bintrayPassword)
val uploadUrl = "$artifactoryUrl/$artifactoryBuildsDirectory/$target/$buildsFileName"
sendUploadRequest(uploadUrl, newBuildsDescription.joinToString("\n"),
extraHeaders = listOf(getArtifactoryHeader(apiKey)))
// Reload values.
clean(target)
@@ -111,15 +112,14 @@ object LocalCache {
}
data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double)
data class GoldenResultsInfo(val bintrayUser: String, val bintrayPassword: String, val goldenResults: Array<GoldenResult>)
data class GoldenResultsInfo(val apiKey: String, val goldenResults: Array<GoldenResult>)
// Build information provided from request.
data class BuildInfo(val buildNumber: String, val branch: String, val startTime: String,
val finishTime: String)
data class BuildRegister(val buildId: String, val teamCityUser: String, val teamCityPassword: String,
val bintrayUser: String, val bintrayPassword: String,
val target: String, val buildType: String, val failuresNumber: Int,
val artifactoryApiKey: String, val target: String, val buildType: String, val failuresNumber: Int,
val executionTime: String, val compileTime: String, val codeSize: String,
val bundleSize: String?) {
companion object {
@@ -127,8 +127,8 @@ data class BuildRegister(val buildId: String, val teamCityUser: String, val team
val requestDetails = JSON.parse<BuildRegister>(json)
// Parse method doesn't create real instance with all methods. So create it by hands.
return BuildRegister(requestDetails.buildId, requestDetails.teamCityUser, requestDetails.teamCityPassword,
requestDetails.bintrayUser, requestDetails.bintrayPassword, requestDetails.target,
requestDetails.buildType, requestDetails.failuresNumber, requestDetails.executionTime, requestDetails.compileTime,
requestDetails.artifactoryApiKey, requestDetails.target, requestDetails.buildType,
requestDetails.failuresNumber, requestDetails.executionTime, requestDetails.compileTime,
requestDetails.codeSize, requestDetails.bundleSize)
}
}
@@ -189,8 +189,8 @@ class CommitsList(data: JsonElement): ConvertedFromJson {
}
}
fun getBuildsInfoFromBintray(target: String) =
sendGetRequest("$downloadBintrayUrl/$target/$buildsFileName")
fun getBuildsInfoFromArtifactory(target: String) =
sendGetRequest("$artifactoryUrl/$artifactoryBuildsDirectory/$target/$buildsFileName")
fun checkBuildType(currentType: String, targetType: String): Boolean {
val releasesBuildTypes = listOf("release", "eap", "rc1", "rc2")
@@ -226,7 +226,7 @@ fun router() {
val express = require("express")
val router = express.Router()
// Register build on Bintray.
// Register build on Artifactory.
router.post("/register", { request, response ->
val maxCommitsNumber = 5
val register = BuildRegister.create(JSON.stringify(request.body))
@@ -251,8 +251,8 @@ fun router() {
}
}
// Get summary file from Bintray.
var buildsDescription = getBuildsInfoFromBintray(register.target)
// Get summary file from Artifactory.
var buildsDescription = getBuildsInfoFromArtifactory(register.target)
// Add information about new build.
//var buildsDescription = "build, start time, finish time, branch, commits, type, failuresNumber, execution time, compile time, code size, bundle size\n"
buildsDescription += "${buildInfo.buildNumber}, ${buildInfo.startTime}, ${buildInfo.finishTime}, " +
@@ -261,8 +261,8 @@ fun router() {
"${register.bundleSize ?: "-"}\n"
// Upload new version of file.
val uploadUrl = "$uploadBintrayUrl/$bintrayPackage/latest/${register.target}/${buildsFileName}?publish=1&override=1"
sendUploadRequest(uploadUrl, buildsDescription, register.bintrayUser, register.bintrayPassword)
val uploadUrl = "$artifactoryUrl/$artifactoryBuildsDirectory/${register.target}/${buildsFileName}"
sendUploadRequest(uploadUrl, buildsDescription, extraHeaders = listOf(getArtifactoryHeader(register.artifactoryApiKey)))
LocalCache.clean(register.target)
LocalCache.fill(register.target)
@@ -271,16 +271,17 @@ fun router() {
response.sendStatus(200)
})
// Register golden results to normalize on Bintray.
// Register golden results to normalize on Artifactory.
router.post("/registerGolden", { request, response ->
val goldenResultsInfo = JSON.parse<GoldenResultsInfo>(JSON.stringify(request.body))
val buildsDescription = StringBuilder(sendGetRequest("$downloadBintrayUrl/$goldenResultsFileName"))
val buildsDescription = StringBuilder(sendGetRequest("$artifactoryUrl/$artifactoryBuildsDirectory/$goldenResultsFileName"))
goldenResultsInfo.goldenResults.forEach {
buildsDescription.append("${it.benchmarkName}, ${it.metric}, ${it.value}\n")
}
// Upload new version of file.
val uploadUrl = "$uploadBintrayUrl/$bintrayPackage/latest/$goldenResultsFileName?publish=1&override=1"
sendUploadRequest(uploadUrl, buildsDescription.toString(), goldenResultsInfo.bintrayUser, goldenResultsInfo.bintrayPassword)
val uploadUrl = "$artifactoryUrl/$artifactoryBuildsDirectory/$goldenResultsFileName"
sendUploadRequest(uploadUrl, buildsDescription.toString(),
extraHeaders = listOf(getArtifactoryHeader(goldenResultsInfo.apiKey)))
// Send response.
response.sendStatus(200)
})
@@ -318,7 +319,7 @@ fun router() {
router.get("/delete/:target", { request, response ->
val buildsToDelete: List<String> = request.query.builds.toString().split(",").map { it.trim() }
val result = LocalCache.delete(request.params.target, buildsToDelete, request.query.user, request.query.key)
val result = LocalCache.delete(request.params.target, buildsToDelete, request.query.key)
if (result) {
response.sendStatus(200)
} else {
@@ -362,12 +363,16 @@ fun sendGetRequest(url: String, user: String? = null, password: String? = null,
return response.getBody().toString()
}
fun sendUploadRequest(url: String, fileContent: String, user: String? = null, password: String? = null) {
fun sendUploadRequest(url: String, fileContent: String, user: String? = null, password: String? = null,
extraHeaders: List<Pair<String, String>> = emptyList()) {
val request = require("sync-request")
val headers = mutableListOf<Pair<String, String>>("Content-type" to "text/plain")
if (user != null && password != null) {
headers.add("Authorization" to getAuth(user, password))
}
extraHeaders.forEach {
headers.add(it.first to it.second)
}
val response = request("PUT", url,
json(
"headers" to json(*(headers.toTypedArray())),
tools/performance-server/ui/src/main/kotlin/main.kt
+2 -2
View File
@@ -182,10 +182,10 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
element = data.element.replace(rectangle)
}
// Add tooltips.
val linkToDetailedInfo = "https://kotlin-native-performance.labs.jb.gg/?report=bintray:" +
val linkToDetailedInfo = "https://kotlin-native-performance.labs.jb.gg/?report=artifactory:" +
"${currentBuild.buildNumber}:${parameters["target"]}:nativeReport.json" +
"${if (data.index - 1 >= 0)
"&compareTo=bintray:${buildsGroup.get(data.index - 1).buildNumber}:${parameters["target"]}:nativeReport.json"
"&compareTo=artifactory:${buildsGroup.get(data.index - 1).buildNumber}:${parameters["target"]}:nativeReport.json"
else ""}"
val information = buildString {
append("<a href=\"$linkToDetailedInfo\">${currentBuild.buildNumber}</a><br>")