Made benchmark application flexible to run separate benchmarks (#2649)

tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt

@@ -76,6 +76,16 @@ class BenchmarksReport(val env: Environment, benchmarksList: List<BenchmarkResul
         }
         """
     }
+
+    // Concatenate benchmarks report if they have same environment and compiler.
+    operator fun plus(other: BenchmarksReport): BenchmarksReport {
+        if (compiler != other.compiler && env != other.env) {
+            error ("It's impossible to concat reports from different machines!")
+        }
+        val mergedBenchmarks = HashMap<String, List<BenchmarkResult>>(benchmarks)
+        mergedBenchmarks.putAll(other.benchmarks)
+        return BenchmarksReport(env, mergedBenchmarks.flatMap{it.value}, compiler)
+    }
 }
 
 // Class for kotlin compiler

tools/benchmarksAnalyzer/build.gradle

@@ -40,6 +40,7 @@ kotlin {
             }
             kotlin.srcDir '../benchmarks/shared/src'
             kotlin.srcDir 'src/main/kotlin'
+            kotlin.srcDir '../kliopt'
 
         }
         commonTest {

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -221,7 +221,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
                 // Calculate metrics for showing difference.
                 val percent = current.calcPercentageDiff(previous)
                 val ratio = current.calcRatio(previous)
-                if (abs(percent.mean) >= meaningfulChangesValue) {
+                if (abs(percent.mean) - percent.variance >= meaningfulChangesValue) {
                     return Pair(name, Pair(percent, ratio))
                 }
             }