From 1a57794e8207e8bb653f8513cf81623e51d1a315 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Mon, 2 Nov 2020 15:08:42 +0100 Subject: [PATCH] Clean ups in IDE performance tests output --- .../AbstractFirHighlightingPerformanceTest.kt | 3 +- .../TestData addImport geomMean - vega.js | 404 ++++++++++++++++ .../TestData completion geomMean - vega.js | 30 +- .../resources/TestData highlight - vega.js | 66 ++- .../TestData highlight - warmup - vega.js | 450 ++++++++++++++++++ ... => TestData highlight geomMean - vega.js} | 23 +- .../resources/TestData j2k geomMean - vega.js | 411 ++++++++++++++++ ...ources build.gradle.kts geomMean - vega.js | 410 ++++++++++++++++ ...stractPerformanceCompletionHandlerTests.kt | 3 +- ...ormanceCompletionIncrementalResolveTest.kt | 3 +- .../AbstractPerformanceHighlightingTest.kt | 3 +- .../perf/AbstractPerformanceImportTest.kt | 3 +- ...anceJavaToKotlinCopyPasteConversionTest.kt | 3 +- ...manceLiteralKotlinToKotlinCopyPasteTest.kt | 3 +- ...bstractPerformanceTypingIndentationTest.kt | 3 +- .../org/jetbrains/kotlin/idea/perf/Stats.kt | 4 +- .../kotlin/idea/perf/util/aggregateResults.kt | 2 + 17 files changed, 1764 insertions(+), 60 deletions(-) create mode 100644 idea/performanceTests/resources/TestData addImport geomMean - vega.js create mode 100644 idea/performanceTests/resources/TestData highlight - warmup - vega.js rename idea/performanceTests/resources/{TestData geomMean - vega.js => TestData highlight geomMean - vega.js} (94%) create mode 100644 idea/performanceTests/resources/TestData j2k geomMean - vega.js create mode 100644 idea/performanceTests/resources/The Kotlin sources build.gradle.kts geomMean - vega.js diff --git a/idea/idea-fir-performance-tests/tests/org/jetbrains/kotlin/idea/highlighter/AbstractFirHighlightingPerformanceTest.kt b/idea/idea-fir-performance-tests/tests/org/jetbrains/kotlin/idea/highlighter/AbstractFirHighlightingPerformanceTest.kt index 3b3d0d0bd8a..8f23d1d76d6 100644 --- a/idea/idea-fir-performance-tests/tests/org/jetbrains/kotlin/idea/highlighter/AbstractFirHighlightingPerformanceTest.kt +++ b/idea/idea-fir-performance-tests/tests/org/jetbrains/kotlin/idea/highlighter/AbstractFirHighlightingPerformanceTest.kt @@ -47,8 +47,7 @@ abstract class AbstractFirHighlightingPerformanceTest : AbstractHighlightingTest override fun tearDown() { commitAllDocuments() RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats.flush() } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/resources/TestData addImport geomMean - vega.js b/idea/performanceTests/resources/TestData addImport geomMean - vega.js new file mode 100644 index 00000000000..106323c1327 --- /dev/null +++ b/idea/performanceTests/resources/TestData addImport geomMean - vega.js @@ -0,0 +1,404 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +{ + "note": "May https://vega.github.io/vega/docs/ be with you", + "$schema": "https://vega.github.io/schema/vega/v4.3.0.json", + "description": "TestData addImport geomMean", + "title": "TestData addImport geomMean", + "width": 800, + "height": 500, + "padding": 5, + "autosize": {"type": "pad", "resize": true}, + "signals": [ + { + "name": "clear", + "value": true, + "on": [ + {"events": "mouseup[!event.item]", "update": "true", "force": true} + ] + }, + { + "name": "shift", + "value": false, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "update": "event.shiftKey", + "force": true + } + ] + }, + { + "name": "clicked", + "value": null, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "comment": "note: here `datum` is `selected` data set", + "update": "{value: datum.value}", + "force": true + } + ] + }, + { + "name": "brush", + "value": 0, + "on": [ + {"events": {"signal": "clear"}, "update": "clear ? [0, 0] : brush"}, + {"events": "@xaxis:mousedown", "update": "[x(), x()]"}, + { + "events": "[@xaxis:mousedown, window:mouseup] > window:mousemove!", + "update": "[brush[0], clamp(x(), 0, width)]" + }, + { + "events": {"signal": "delta"}, + "update": "clampRange([anchor[0] + delta, anchor[1] + delta], 0, width)" + } + ] + }, + { + "name": "anchor", + "value": null, + "on": [{"events": "@brush:mousedown", "update": "slice(brush)"}] + }, + { + "name": "xdown", + "value": 0, + "on": [{"events": "@brush:mousedown", "update": "x()"}] + }, + { + "name": "delta", + "value": 0, + "on": [ + { + "events": "[@brush:mousedown, window:mouseup] > window:mousemove!", + "update": "x() - xdown" + } + ] + }, + { + "name": "domain", + "on": [ + { + "events": {"signal": "brush"}, + "update": "span(brush) ? invert('x', brush) : null" + } + ] + }, + {"name": "timestamp", "value": true, "bind": {"input": "checkbox"}} + ], + "data": [ + { + "name": "table", + "comment": "To test chart in VEGA editor https://vega.github.io/editor/#/ change `_values` to `values` and rename `url` property", + "_values": { + "hits" : { + "hits" : [ + { + "_source" : {"buildId" : 87834896, "buildTimestamp" : "2020-09-21T21:00:31+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87783396, "buildTimestamp" : "2020-09-21T12:34:19+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87809918, "buildTimestamp" : "2020-09-21T16:47:11+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87905203, "buildTimestamp" : "2020-09-22T13:23:44+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87894638, "buildTimestamp" : "2020-09-22T09:12:16+0000", "metricValue" : 37, "benchmark" : "addImport"} + } + ] + } + }, + "url": { + //"comment": "source index pattern", + "index": "kotlin_ide_benchmarks*", + //"comment": "it's a body of ES _search query to check query place it into `POST /kotlin_ide_benchmarks*/_search`", + //"comment": "it uses Kibana specific %timefilter% for time frame selection", + "body": { + "size": 1000, + "query": { + "bool": { + "must": [ + {"exists": {"field": "synthetic"}}, + { + "bool": { + "must_not": [ + {"exists": {"field": "warmUp"}} + ] + } + }, + {"term": {"benchmark.keyword": "add-import"}}, + {"term": {"name.keyword": "geomMean"}}, + {"range": {"buildTimestamp": {"%timefilter%": true}}} + ] + } + }, + "_source": ["buildId", "benchmark", "buildTimestamp", "hasError", "metricValue"], + "sort": [{"buildTimestamp": {"order": "asc"}}] + } + }, + "format": {"property": "hits.hits"}, + "comment": "we need to have follow data: \"buildId\", \"metricName\", \"metricValue\" and \"metricError\"", + "comment": "so it has to be array of {\"buildId\": \"...\", \"metricName\": \"...\", \"metricValue\": ..., \"metricError\": ...}", + "transform": [ + {"type": "collect","sort": {"field": "_source.buildTimestamp"}}, + { + "comment": "make alias: _source.buildId -> buildId", + "type": "formula", + "as": "buildId", + "expr": "datum._source.buildId" + }, + { + "comment": "make alias: _source.benchmark -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.benchmark" + }, + { + "comment": "make alias: _source.metricValue -> metricValue", + "type": "formula", + "as": "metricValue", + "expr": "datum._source.metricValue" + }, + { + "comment": "define metricError", + "type": "formula", + "as": "metricError", + "expr": "1" + }, + { + "comment": "make alias: _source.hasError -> hasError", + "type": "formula", + "as": "hasError", + "expr": "datum._source.metrics ? datum._source.hasError : false" + }, + { + "type": "formula", + "as": "timestamp", + "expr": "timeFormat(toDate(datum._source.buildTimestamp), '%Y-%m-%d %H:%M')" + }, + { + "comment": "create `url` value that points to TC build", + "type": "formula", + "as": "url", + "expr": "'https://buildserver.labs.intellij.net/buildConfiguration/Kotlin_Benchmarks_PluginPerformanceTests_IdeaPluginPerformanceTests/' + datum._source.buildId" + } + ] + }, + { + "name": "selected", + "on": [ + {"trigger": "clear", "remove": true}, + {"trigger": "!shift", "remove": true}, + {"trigger": "!shift && clicked", "insert": "clicked"}, + {"trigger": "shift && clicked", "toggle": "clicked"} + ] + } + ], + "axes": [ + { + "scale": "x", + "grid": true, + "domain": false, + "orient": "bottom", + "labelAngle": -20, + "labelAlign": "right", + "title": {"signal": "timestamp ? 'timestamp' : 'buildId'"}, + "titlePadding": 10, + "tickCount": 5, + "encode": { + "labels": { + "interactive": true, + "update": {"tooltip": {"signal": "datum.label"}} + } + } + }, + { + "scale": "y", + "grid": true, + "domain": false, + "orient": "left", + "titlePadding": 10, + "title": "ms", + "titleAnchor": "end", + "titleAngle": 0 + } + ], + "scales": [ + { + "name": "x", + "type": "point", + "range": "width", + "domain": {"data": "table", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}} + }, + { + "name": "y", + "type": "linear", + "range": "height", + "nice": true, + "zero": true, + "domain": {"data": "table", "field": "metricValue"} + }, + { + "name": "color", + "type": "ordinal", + "range": "category", + "domain": {"data": "table", "field": "metricName"} + }, + { + "name": "size", + "type": "linear", + "round": true, + "nice": false, + "zero": true, + "domain": {"data": "table", "field": "metricError"}, + "range": [1, 100] + } + ], + "legends": [ + { + "title": "Cases", + "stroke": "color", + "strokeColor": "#ccc", + "padding": 8, + "cornerRadius": 4, + "symbolLimit": 50, + "encode": { + "symbols": { + "name": "legendSymbol", + "interactive": true, + "update": { + "fill": {"value": "transparent"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 0.7 + }, + {"value": 0.15} + ], + "size": {"value": 64} + } + }, + "labels": { + "name": "legendLabel", + "interactive": true, + "update": { + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 1 + }, + {"value": 0.25} + ] + } + } + } + } + ], + "marks": [ + { + "type": "group", + "from": { + "facet": {"name": "series", "data": "table", "groupby": "metricName"} + }, + "marks": [ + { + "type": "line", + "from": {"data": "series"}, + "encode": { + "hover": {"opacity": {"value": 1}, "strokeWidth": {"value": 4}}, + "update": { + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 0.7 + }, + {"value": 0.15} + ], + "stroke": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "scale": "color", + "field": "metricName" + }, + {"value": "#ccc"} + ] + } + } + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "fill": {"value": "#B00"}, + "size": [{ "test": "datum.hasError", "value": 250 }, {"value": 0}], + "shape": {"value": "cross"}, + "angle": {"value": 45}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 1 + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "tooltip": { + "signal": "datum.metricName + ': ' + datum.metricValue + ' ms'" + }, + "href": {"field": "url"}, + "cursor": {"value": "pointer"}, + "size": {"scale": "size", "field": "metricError"}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "fill": {"scale": "color", "field": "metricName"} + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 2 + } + ] + } + ] +} diff --git a/idea/performanceTests/resources/TestData completion geomMean - vega.js b/idea/performanceTests/resources/TestData completion geomMean - vega.js index ace4400a200..d8745a587d6 100644 --- a/idea/performanceTests/resources/TestData completion geomMean - vega.js +++ b/idea/performanceTests/resources/TestData completion geomMean - vega.js @@ -173,20 +173,20 @@ "bool": { "must": [ {"range": {"buildTimestamp": {"%timefilter%": true}}}, + { + "bool": { + "should": [ + {"exists": {"field": "synthetic"}}, + {"term": {"name.keyword": "geomMean"}} + ] + } + }, { "bool": { "should": [ {"term": {"benchmark.keyword": "completion-basic"}}, - { - "term": { - "benchmark.keyword": "completion-basic-charFilter" - } - }, - { - "term": { - "benchmark.keyword": "completion-basic-keyword" - } - }, + {"term": {"benchmark.keyword": "completion-basic-charFilter"}}, + {"term": {"benchmark.keyword": "completion-basic-keyword"}}, {"term": {"benchmark.keyword": "completion-incremental"}}, {"term": {"benchmark.keyword": "completion-smart"}} ] @@ -195,7 +195,7 @@ ] } }, - "_source": ["buildId", "benchmark", "buildTimestamp", "metrics.hasError", "geomMean"], + "_source": ["buildId", "benchmark", "buildTimestamp", "name", "metricValue", "hasError"], "sort": [{"buildTimestamp": {"order": "asc"}}] } }, @@ -217,10 +217,10 @@ "expr": "datum._source.benchmark" }, { - "comment": "make alias: _source.geomMean -> metricValue", + "comment": "make alias: _source.metricValue -> metricValue", "type": "formula", "as": "metricValue", - "expr": "datum._source.geomMean" + "expr": "datum._source.metricValue" }, { "comment": "define metricError", @@ -229,10 +229,10 @@ "expr": "1" }, { - "comment": "make alias: _source.metrics[0].hasError -> hasError", + "comment": "make alias: _source.hasError -> hasError", "type": "formula", "as": "hasError", - "expr": "datum._source.metrics ? datum._source.metrics[0].hasError : false" + "expr": "datum._source.hasError ? datum._source.hasError : false" }, { "type": "formula", diff --git a/idea/performanceTests/resources/TestData highlight - vega.js b/idea/performanceTests/resources/TestData highlight - vega.js index bebf4299011..cbd6dcaa1ff 100644 --- a/idea/performanceTests/resources/TestData highlight - vega.js +++ b/idea/performanceTests/resources/TestData highlight - vega.js @@ -162,23 +162,24 @@ //"comment": "it uses Kibana specific %timefilter% for time frame selection", "body": { "size": 1000, + "query": { "bool": { "must": [ + { + "bool": { + "must_not": [ + {"exists": {"field": "warmUp"}}, + {"exists": {"field": "synthetic"}} + ] + } + }, {"term": {"benchmark.keyword": "highlight"}}, {"range": {"buildTimestamp": {"%timefilter%": true}}} ] } }, - "_source": [ - "buildId", - "benchmark", - "buildTimestamp", - "metrics.hasError", - "metrics.metricName", - "metrics.metricValue", - "metrics.metricError" - ], + "_source": ["buildId", "benchmark", "name", "buildTimestamp", "hasError", "metricValue", "metricError"], "sort": [{"buildTimestamp": {"order": "asc"}}] } }, @@ -187,23 +188,46 @@ "comment": "so it has to be array of {\"buildId\": \"...\", \"metricName\": \"...\", \"metricValue\": ..., \"metricError\": ...}", "transform": [ {"type": "collect","sort": {"field": "_source.buildTimestamp"}}, - {"type": "flatten", "fields": ["_source.metrics"], "as": ["metrics"]}, { - "type": "project", - "fields": [ - "_source.buildId", - "_source.buildTimestamp", - "metrics.hasError", - "metrics.metricName", - "metrics.metricValue", - "metrics.metricError" - ], - "as": ["buildId", "buildTimestamp", "hasError", "metricName", "metricValue", "metricError"] + "comment": "make alias: _source.buildId -> buildId", + "type": "formula", + "as": "buildId", + "expr": "datum._source.buildId" + }, + { + "comment": "make alias: _source.benchmark -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.benchmark" + }, + { + "comment": "make alias: _source.metricValue -> metricValue", + "type": "formula", + "as": "metricValue", + "expr": "datum._source.metricValue" + }, + { + "comment": "make alias: _source.name -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.name" + }, + { + "comment": "define metricError", + "type": "formula", + "as": "metricError", + "expr": "1" + }, + { + "comment": "make alias: _source.hasError -> hasError", + "type": "formula", + "as": "hasError", + "expr": "datum._source.metrics ? datum._source.hasError : false" }, { "type": "formula", "as": "timestamp", - "expr": "timeFormat(toDate(datum.buildTimestamp), '%Y-%m-%d %H:%M')" + "expr": "timeFormat(toDate(datum._source.buildTimestamp), '%Y-%m-%d %H:%M')" }, { "comment": "create `url` value that points to TC build", diff --git a/idea/performanceTests/resources/TestData highlight - warmup - vega.js b/idea/performanceTests/resources/TestData highlight - warmup - vega.js new file mode 100644 index 00000000000..b705eb39e46 --- /dev/null +++ b/idea/performanceTests/resources/TestData highlight - warmup - vega.js @@ -0,0 +1,450 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +{ + "note": "May https://vega.github.io/vega/docs/ be with you", + "$schema": "https://vega.github.io/schema/vega/v4.3.0.json", + "description": "TestData highlight - warmUp", + "title": "TestData highlight - warmUp", + "width": 800, + "height": 400, + "padding": 5, + "autosize": {"type": "pad", "resize": true}, + "signals": [ + { + "name": "clear", + "value": true, + "on": [ + {"events": "mouseup[!event.item]", "update": "true", "force": true} + ] + }, + { + "name": "shift", + "value": false, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "update": "event.shiftKey", + "force": true + } + ] + }, + { + "name": "clicked", + "value": null, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "comment": "note: here `datum` is `selected` data set", + "update": "{value: datum.value}", + "force": true + } + ] + }, + { + "name": "brush", + "value": 0, + "on": [ + {"events": {"signal": "clear"}, "update": "clear ? [0, 0] : brush"}, + {"events": "@xaxis:mousedown", "update": "[x(), x()]"}, + { + "events": "[@xaxis:mousedown, window:mouseup] > window:mousemove!", + "update": "[brush[0], clamp(x(), 0, width)]" + }, + { + "events": {"signal": "delta"}, + "update": "clampRange([anchor[0] + delta, anchor[1] + delta], 0, width)" + } + ] + }, + { + "name": "anchor", + "value": null, + "on": [{"events": "@brush:mousedown", "update": "slice(brush)"}] + }, + { + "name": "xdown", + "value": 0, + "on": [{"events": "@brush:mousedown", "update": "x()"}] + }, + { + "name": "delta", + "value": 0, + "on": [ + { + "events": "[@brush:mousedown, window:mouseup] > window:mousemove!", + "update": "x() - xdown" + } + ] + }, + { + "name": "domain", + "on": [ + { + "events": {"signal": "brush"}, + "update": "span(brush) ? invert('x', brush) : null" + } + ] + }, + {"name": "timestamp", "value": true, "bind": {"input": "checkbox"}} + ], + "data": [ + { + "name": "table", + "comment": "To test chart in VEGA editor https://vega.github.io/editor/#/ change `_values` to `values` and rename `url` property", + "_values": { + "hits" : { + "hits" : [ + { + "_source" : { + "buildId" : 87834896, + "buildTimestamp" : "2020-09-21T21:00:31+0000", + "metrics" : [ + {"metricName" : "NonNullAssertion", "metricValue" : 31, "metricError" : 1 }, + {"metricName" : "PropertiesWithPropertyDeclarations", "metricValue" : 191, "metricError" : 4 } + ], + "benchmark" : "highlight" + } + }, + { + "_source" : { + "buildId" : 87783396, + "buildTimestamp" : "2020-09-21T12:34:19+0000", + "metrics" : [ + {"metricName" : "NonNullAssertion", "metricValue" : 32, "metricError" : 1 }, + {"metricName" : "PropertiesWithPropertyDeclarations", "metricValue" : 189, "metricError" : 4 } + ], + "benchmark" : "highlight" + } + }, + { + "_source" : { + "buildId" : 87809918, + "buildTimestamp" : "2020-09-21T16:47:11+0000", + "metrics" : [ + {"metricName" : "NonNullAssertion", "metricValue" : 30, "metricError" : 1 }, + {"metricName" : "PropertiesWithPropertyDeclarations", "metricValue" : 188, "metricError" : 4 } + ], + "benchmark" : "highlight" + } + }, + { + "_source" : { + "buildId" : 87905203, + "buildTimestamp" : "2020-09-22T13:23:44+0000", + "metrics" : [ + {"metricName" : "NonNullAssertion", "metricValue" : 30, "metricError" : 1 }, + {"metricName" : "PropertiesWithPropertyDeclarations", "metricValue" : 180, "metricError" : 4 } + ], + "benchmark" : "highlight" + } + }, + { + "_source" : { + "buildId" : 87894638, + "buildTimestamp" : "2020-09-22T09:12:16+0000", + "metrics" : [ + {"metricName" : "NonNullAssertion", "metricValue" : 32, "metricError" : 1 }, + {"metricName" : "PropertiesWithPropertyDeclarations", "metricValue" : 193, "metricError" : 4 } + ], + "benchmark" : "highlight" + } + } + ] + } + }, + "url": { + //"comment": "source index pattern", + "index": "kotlin_ide_benchmarks*", + //"comment": "it's a body of ES _search query to check query place it into `POST /kotlin_ide_benchmarks*/_search`", + //"comment": "it uses Kibana specific %timefilter% for time frame selection", + "body": { + "size": 1000, + + "query": { + "bool": { + "must": [ + { + "bool": { + "must_not": [ + {"exists": {"field": "synthetic"}} + ] + } + }, + {"exists": {"field": "warmUp"}}, + {"term": {"benchmark.keyword": "highlight"}}, + {"range": {"buildTimestamp": {"%timefilter%": true}}} + ] + } + }, + "_source": ["buildId", "benchmark", "name", "buildTimestamp", "hasError", "metricValue", "metricError"], + "sort": [{"buildTimestamp": {"order": "asc"}}] + } + }, + "format": {"property": "hits.hits"}, + "comment": "we need to have follow data: \"buildId\", \"metricName\", \"metricValue\" and \"metricError\"", + "comment": "so it has to be array of {\"buildId\": \"...\", \"metricName\": \"...\", \"metricValue\": ..., \"metricError\": ...}", + "transform": [ + {"type": "collect","sort": {"field": "_source.buildTimestamp"}}, + { + "comment": "make alias: _source.buildId -> buildId", + "type": "formula", + "as": "buildId", + "expr": "datum._source.buildId" + }, + { + "comment": "make alias: _source.benchmark -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.benchmark" + }, + { + "comment": "make alias: _source.metricValue -> metricValue", + "type": "formula", + "as": "metricValue", + "expr": "datum._source.metricValue" + }, + { + "comment": "make alias: _source.name -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.name" + }, + { + "comment": "define metricError", + "type": "formula", + "as": "metricError", + "expr": "1" + }, + { + "comment": "make alias: _source.hasError -> hasError", + "type": "formula", + "as": "hasError", + "expr": "datum._source.metrics ? datum._source.hasError : false" + }, + { + "type": "formula", + "as": "timestamp", + "expr": "timeFormat(toDate(datum._source.buildTimestamp), '%Y-%m-%d %H:%M')" + }, + { + "comment": "create `url` value that points to TC build", + "type": "formula", + "as": "url", + "expr": "'https://buildserver.labs.intellij.net/buildConfiguration/Kotlin_Benchmarks_PluginPerformanceTests_IdeaPluginPerformanceTests/' + datum.buildId" + } + ] + }, + { + "name": "selected", + "on": [ + {"trigger": "clear", "remove": true}, + {"trigger": "!shift", "remove": true}, + {"trigger": "!shift && clicked", "insert": "clicked"}, + {"trigger": "shift && clicked", "toggle": "clicked"} + ] + } + ], + "axes": [ + { + "scale": "x", + "grid": true, + "domain": false, + "orient": "bottom", + "labelAngle": -20, + "labelAlign": "right", + "title": {"signal": "timestamp ? 'timestamp' : 'buildId'"}, + "titlePadding": 10, + "tickCount": 5, + "encode": { + "labels": { + "interactive": true, + "update": {"tooltip": {"signal": "datum.label"}} + } + } + }, + { + "scale": "y", + "grid": true, + "domain": false, + "orient": "left", + "titlePadding": 10, + "title": "ms", + "titleAnchor": "end", + "titleAngle": 0 + } + ], + "scales": [ + { + "name": "x", + "type": "point", + "range": "width", + "domain": {"data": "table", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}} + }, + { + "name": "y", + "type": "linear", + "range": "height", + "nice": true, + "zero": true, + "domain": {"data": "table", "field": "metricValue"} + }, + { + "name": "color", + "type": "ordinal", + "range": "category", + "domain": {"data": "table", "field": "metricName"} + }, + { + "name": "size", + "type": "linear", + "round": true, + "nice": false, + "zero": true, + "domain": {"data": "table", "field": "metricError"}, + "range": [1, 100] + } + ], + "legends": [ + { + "title": "Cases", + "stroke": "color", + "strokeColor": "#ccc", + "padding": 8, + "cornerRadius": 4, + "symbolLimit": 50, + "encode": { + "symbols": { + "name": "legendSymbol", + "interactive": true, + "update": { + "fill": {"value": "transparent"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 0.7 + }, + {"value": 0.15} + ], + "size": {"value": 64} + } + }, + "labels": { + "name": "legendLabel", + "interactive": true, + "update": { + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 1 + }, + {"value": 0.25} + ] + } + } + } + } + ], + "marks": [ + { + "type": "group", + "from": { + "facet": {"name": "series", "data": "table", "groupby": "metricName"} + }, + "marks": [ + { + "type": "line", + "from": {"data": "series"}, + "encode": { + "hover": {"opacity": {"value": 1}, "strokeWidth": {"value": 4}}, + "update": { + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 0.7 + }, + {"value": 0.15} + ], + "stroke": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "scale": "color", + "field": "metricName" + }, + {"value": "#ccc"} + ] + } + } + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "fill": {"value": "#B00"}, + "size": [{ "test": "datum.hasError", "value": 250 }, {"value": 0}], + "shape": {"value": "cross"}, + "angle": {"value": 45}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 1 + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "tooltip": { + "signal": "datum.metricName + ': ' + datum.metricValue + ' ms'" + }, + "href": {"field": "url"}, + "cursor": {"value": "pointer"}, + "size": {"scale": "size", "field": "metricError"}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "fill": {"scale": "color", "field": "metricName"} + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 2 + } + ] + } + ] +} diff --git a/idea/performanceTests/resources/TestData geomMean - vega.js b/idea/performanceTests/resources/TestData highlight geomMean - vega.js similarity index 94% rename from idea/performanceTests/resources/TestData geomMean - vega.js rename to idea/performanceTests/resources/TestData highlight geomMean - vega.js index bfccad8fcf8..56f7f1d45b4 100644 --- a/idea/performanceTests/resources/TestData geomMean - vega.js +++ b/idea/performanceTests/resources/TestData highlight geomMean - vega.js @@ -6,8 +6,8 @@ { "note": "May https://vega.github.io/vega/docs/ be with you", "$schema": "https://vega.github.io/schema/vega/v4.3.0.json", - "description": "TestData geomMean", - "title": "TestData geomMean", + "description": "TestData highlight geomMean", + "title": "TestData highlight geomMean", "width": 800, "height": 500, "padding": 5, @@ -125,12 +125,21 @@ "query": { "bool": { "must": [ + {"exists": {"field": "synthetic"}}, + { + "bool": { + "must_not": [ + {"exists": {"field": "warmUp"}} + ] + } + }, {"term": {"benchmark.keyword": "highlight"}}, + {"term": {"name.keyword": "geomMean"}}, {"range": {"buildTimestamp": {"%timefilter%": true}}} ] } }, - "_source": ["buildId", "benchmark", "buildTimestamp", "metrics.hasError", "geomMean"], + "_source": ["buildId", "benchmark", "buildTimestamp", "hasError", "metricValue"], "sort": [{"buildTimestamp": {"order": "asc"}}] } }, @@ -152,10 +161,10 @@ "expr": "datum._source.benchmark" }, { - "comment": "make alias: _source.geomMean -> metricValue", + "comment": "make alias: _source.metricValue -> metricValue", "type": "formula", "as": "metricValue", - "expr": "datum._source.geomMean" + "expr": "datum._source.metricValue" }, { "comment": "define metricError", @@ -164,10 +173,10 @@ "expr": "1" }, { - "comment": "make alias: _source.metrics[0].hasError -> hasError", + "comment": "make alias: _source.hasError -> hasError", "type": "formula", "as": "hasError", - "expr": "datum._source.metrics ? datum._source.metrics[0].hasError : false" + "expr": "datum._source.metrics ? datum._source.hasError : false" }, { "type": "formula", diff --git a/idea/performanceTests/resources/TestData j2k geomMean - vega.js b/idea/performanceTests/resources/TestData j2k geomMean - vega.js new file mode 100644 index 00000000000..23e89169b38 --- /dev/null +++ b/idea/performanceTests/resources/TestData j2k geomMean - vega.js @@ -0,0 +1,411 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +{ + "note": "May https://vega.github.io/vega/docs/ be with you", + "$schema": "https://vega.github.io/schema/vega/v4.3.0.json", + "description": "TestData j2k geomMean", + "title": "TestData j2k geomMean", + "width": 800, + "height": 500, + "padding": 5, + "autosize": {"type": "pad", "resize": true}, + "signals": [ + { + "name": "clear", + "value": true, + "on": [ + {"events": "mouseup[!event.item]", "update": "true", "force": true} + ] + }, + { + "name": "shift", + "value": false, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "update": "event.shiftKey", + "force": true + } + ] + }, + { + "name": "clicked", + "value": null, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "comment": "note: here `datum` is `selected` data set", + "update": "{value: datum.value}", + "force": true + } + ] + }, + { + "name": "brush", + "value": 0, + "on": [ + {"events": {"signal": "clear"}, "update": "clear ? [0, 0] : brush"}, + {"events": "@xaxis:mousedown", "update": "[x(), x()]"}, + { + "events": "[@xaxis:mousedown, window:mouseup] > window:mousemove!", + "update": "[brush[0], clamp(x(), 0, width)]" + }, + { + "events": {"signal": "delta"}, + "update": "clampRange([anchor[0] + delta, anchor[1] + delta], 0, width)" + } + ] + }, + { + "name": "anchor", + "value": null, + "on": [{"events": "@brush:mousedown", "update": "slice(brush)"}] + }, + { + "name": "xdown", + "value": 0, + "on": [{"events": "@brush:mousedown", "update": "x()"}] + }, + { + "name": "delta", + "value": 0, + "on": [ + { + "events": "[@brush:mousedown, window:mouseup] > window:mousemove!", + "update": "x() - xdown" + } + ] + }, + { + "name": "domain", + "on": [ + { + "events": {"signal": "brush"}, + "update": "span(brush) ? invert('x', brush) : null" + } + ] + }, + {"name": "timestamp", "value": true, "bind": {"input": "checkbox"}} + ], + "data": [ + { + "name": "table", + "comment": "To test chart in VEGA editor https://vega.github.io/editor/#/ change `_values` to `values` and rename `url` property", + "_values": { + "hits" : { + "hits" : [ + { + "_source" : {"buildId" : 87834896, "buildTimestamp" : "2020-09-21T21:00:31+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87783396, "buildTimestamp" : "2020-09-21T12:34:19+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87809918, "buildTimestamp" : "2020-09-21T16:47:11+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87905203, "buildTimestamp" : "2020-09-22T13:23:44+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87894638, "buildTimestamp" : "2020-09-22T09:12:16+0000", "metricValue" : 37, "benchmark" : "addImport"} + } + ] + } + }, + "url": { + //"comment": "source index pattern", + "index": "kotlin_ide_benchmarks*", + //"comment": "it's a body of ES _search query to check query place it into `POST /kotlin_ide_benchmarks*/_search`", + //"comment": "it uses Kibana specific %timefilter% for time frame selection", + "body": { + "size": 1000, + "query": { + "bool": { + "must": [ + {"exists": {"field": "synthetic"}}, + { + "bool": { + "must_not": [ + {"exists": {"field": "warmUp"}} + ] + } + }, + { + "bool": { + "should": [ + {"term": {"benchmark.keyword": "old j2k"}}, + {"term": {"benchmark.keyword": "new j2k"}}, + ] + } + } + {"term": {"name.keyword": "geomMean"}}, + {"range": {"buildTimestamp": {"%timefilter%": true}}} + ] + } + }, + "_source": ["buildId", "benchmark", "buildTimestamp", "hasError", "metricValue"], + "sort": [{"buildTimestamp": {"order": "asc"}}] + } + }, + "format": {"property": "hits.hits"}, + "comment": "we need to have follow data: \"buildId\", \"metricName\", \"metricValue\" and \"metricError\"", + "comment": "so it has to be array of {\"buildId\": \"...\", \"metricName\": \"...\", \"metricValue\": ..., \"metricError\": ...}", + "transform": [ + {"type": "collect","sort": {"field": "_source.buildTimestamp"}}, + { + "comment": "make alias: _source.buildId -> buildId", + "type": "formula", + "as": "buildId", + "expr": "datum._source.buildId" + }, + { + "comment": "make alias: _source.benchmark -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.benchmark" + }, + { + "comment": "make alias: _source.metricValue -> metricValue", + "type": "formula", + "as": "metricValue", + "expr": "datum._source.metricValue" + }, + { + "comment": "define metricError", + "type": "formula", + "as": "metricError", + "expr": "1" + }, + { + "comment": "make alias: _source.hasError -> hasError", + "type": "formula", + "as": "hasError", + "expr": "datum._source.metrics ? datum._source.hasError : false" + }, + { + "type": "formula", + "as": "timestamp", + "expr": "timeFormat(toDate(datum._source.buildTimestamp), '%Y-%m-%d %H:%M')" + }, + { + "comment": "create `url` value that points to TC build", + "type": "formula", + "as": "url", + "expr": "'https://buildserver.labs.intellij.net/buildConfiguration/Kotlin_Benchmarks_PluginPerformanceTests_IdeaPluginPerformanceTests/' + datum._source.buildId" + } + ] + }, + { + "name": "selected", + "on": [ + {"trigger": "clear", "remove": true}, + {"trigger": "!shift", "remove": true}, + {"trigger": "!shift && clicked", "insert": "clicked"}, + {"trigger": "shift && clicked", "toggle": "clicked"} + ] + } + ], + "axes": [ + { + "scale": "x", + "grid": true, + "domain": false, + "orient": "bottom", + "labelAngle": -20, + "labelAlign": "right", + "title": {"signal": "timestamp ? 'timestamp' : 'buildId'"}, + "titlePadding": 10, + "tickCount": 5, + "encode": { + "labels": { + "interactive": true, + "update": {"tooltip": {"signal": "datum.label"}} + } + } + }, + { + "scale": "y", + "grid": true, + "domain": false, + "orient": "left", + "titlePadding": 10, + "title": "ms", + "titleAnchor": "end", + "titleAngle": 0 + } + ], + "scales": [ + { + "name": "x", + "type": "point", + "range": "width", + "domain": {"data": "table", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}} + }, + { + "name": "y", + "type": "linear", + "range": "height", + "nice": true, + "zero": true, + "domain": {"data": "table", "field": "metricValue"} + }, + { + "name": "color", + "type": "ordinal", + "range": "category", + "domain": {"data": "table", "field": "metricName"} + }, + { + "name": "size", + "type": "linear", + "round": true, + "nice": false, + "zero": true, + "domain": {"data": "table", "field": "metricError"}, + "range": [1, 100] + } + ], + "legends": [ + { + "title": "Cases", + "stroke": "color", + "strokeColor": "#ccc", + "padding": 8, + "cornerRadius": 4, + "symbolLimit": 50, + "encode": { + "symbols": { + "name": "legendSymbol", + "interactive": true, + "update": { + "fill": {"value": "transparent"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 0.7 + }, + {"value": 0.15} + ], + "size": {"value": 64} + } + }, + "labels": { + "name": "legendLabel", + "interactive": true, + "update": { + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 1 + }, + {"value": 0.25} + ] + } + } + } + } + ], + "marks": [ + { + "type": "group", + "from": { + "facet": {"name": "series", "data": "table", "groupby": "metricName"} + }, + "marks": [ + { + "type": "line", + "from": {"data": "series"}, + "encode": { + "hover": {"opacity": {"value": 1}, "strokeWidth": {"value": 4}}, + "update": { + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 0.7 + }, + {"value": 0.15} + ], + "stroke": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "scale": "color", + "field": "metricName" + }, + {"value": "#ccc"} + ] + } + } + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "fill": {"value": "#B00"}, + "size": [{ "test": "datum.hasError", "value": 250 }, {"value": 0}], + "shape": {"value": "cross"}, + "angle": {"value": 45}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 1 + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "tooltip": { + "signal": "datum.metricName + ': ' + datum.metricValue + ' ms'" + }, + "href": {"field": "url"}, + "cursor": {"value": "pointer"}, + "size": {"scale": "size", "field": "metricError"}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "fill": {"scale": "color", "field": "metricName"} + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 2 + } + ] + } + ] +} diff --git a/idea/performanceTests/resources/The Kotlin sources build.gradle.kts geomMean - vega.js b/idea/performanceTests/resources/The Kotlin sources build.gradle.kts geomMean - vega.js new file mode 100644 index 00000000000..3c3c4ee92bd --- /dev/null +++ b/idea/performanceTests/resources/The Kotlin sources build.gradle.kts geomMean - vega.js @@ -0,0 +1,410 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +{ + "note": "May https://vega.github.io/vega/docs/ be with you", + "$schema": "https://vega.github.io/schema/vega/v4.3.0.json", + "description": "The Kotlin sources: highlight build.gradle.kts (geom mean)", + "title": "The Kotlin sources: highlight build.gradle.kts (geom mean)", + "width": 800, + "height": 500, + "padding": 5, + "autosize": {"type": "pad", "resize": true}, + "signals": [ + { + "name": "clear", + "value": true, + "on": [ + {"events": "mouseup[!event.item]", "update": "true", "force": true} + ] + }, + { + "name": "shift", + "value": false, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "update": "event.shiftKey", + "force": true + } + ] + }, + { + "name": "clicked", + "value": null, + "on": [ + { + "events": "@legendSymbol:click, @legendLabel:click", + "comment": "note: here `datum` is `selected` data set", + "update": "{value: datum.value}", + "force": true + } + ] + }, + { + "name": "brush", + "value": 0, + "on": [ + {"events": {"signal": "clear"}, "update": "clear ? [0, 0] : brush"}, + {"events": "@xaxis:mousedown", "update": "[x(), x()]"}, + { + "events": "[@xaxis:mousedown, window:mouseup] > window:mousemove!", + "update": "[brush[0], clamp(x(), 0, width)]" + }, + { + "events": {"signal": "delta"}, + "update": "clampRange([anchor[0] + delta, anchor[1] + delta], 0, width)" + } + ] + }, + { + "name": "anchor", + "value": null, + "on": [{"events": "@brush:mousedown", "update": "slice(brush)"}] + }, + { + "name": "xdown", + "value": 0, + "on": [{"events": "@brush:mousedown", "update": "x()"}] + }, + { + "name": "delta", + "value": 0, + "on": [ + { + "events": "[@brush:mousedown, window:mouseup] > window:mousemove!", + "update": "x() - xdown" + } + ] + }, + { + "name": "domain", + "on": [ + { + "events": {"signal": "brush"}, + "update": "span(brush) ? invert('x', brush) : null" + } + ] + }, + {"name": "timestamp", "value": true, "bind": {"input": "checkbox"}} + ], + "data": [ + { + "name": "table", + "comment": "To test chart in VEGA editor https://vega.github.io/editor/#/ change `_values` to `values` and rename `url` property", + "_values": { + "hits" : { + "hits" : [ + { + "_source" : {"buildId" : 87834896, "buildTimestamp" : "2020-09-21T21:00:31+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87783396, "buildTimestamp" : "2020-09-21T12:34:19+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87809918, "buildTimestamp" : "2020-09-21T16:47:11+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87905203, "buildTimestamp" : "2020-09-22T13:23:44+0000", "metricValue" : 37, "benchmark" : "addImport"} + }, + { + "_source" : {"buildId" : 87894638, "buildTimestamp" : "2020-09-22T09:12:16+0000", "metricValue" : 37, "benchmark" : "addImport"} + } + ] + } + }, + "url": { + //"comment": "source index pattern", + "index": "kotlin_ide_benchmarks*", + //"comment": "it's a body of ES _search query to check query place it into `POST /kotlin_ide_benchmarks*/_search`", + //"comment": "it uses Kibana specific %timefilter% for time frame selection", + "body": { + "size": 1000, + "query": { + "bool": { + "must": [ + {"exists": {"field": "synthetic"}}, + { + "bool": { + "must_not": [ + {"exists": {"field": "warmUp"}} + ] + } + }, + { + "bool": { + "should": [ + {"term": {"benchmark.keyword": "kotlin gradle.kts"}} + ] + } + } + {"term": {"name.keyword": "geomMean"}}, + {"range": {"buildTimestamp": {"%timefilter%": true}}} + ] + } + }, + "_source": ["buildId", "benchmark", "buildTimestamp", "hasError", "metricValue"], + "sort": [{"buildTimestamp": {"order": "asc"}}] + } + }, + "format": {"property": "hits.hits"}, + "comment": "we need to have follow data: \"buildId\", \"metricName\", \"metricValue\" and \"metricError\"", + "comment": "so it has to be array of {\"buildId\": \"...\", \"metricName\": \"...\", \"metricValue\": ..., \"metricError\": ...}", + "transform": [ + {"type": "collect","sort": {"field": "_source.buildTimestamp"}}, + { + "comment": "make alias: _source.buildId -> buildId", + "type": "formula", + "as": "buildId", + "expr": "datum._source.buildId" + }, + { + "comment": "make alias: _source.benchmark -> metricName", + "type": "formula", + "as": "metricName", + "expr": "datum._source.benchmark" + }, + { + "comment": "make alias: _source.metricValue -> metricValue", + "type": "formula", + "as": "metricValue", + "expr": "datum._source.metricValue" + }, + { + "comment": "define metricError", + "type": "formula", + "as": "metricError", + "expr": "1" + }, + { + "comment": "make alias: _source.hasError -> hasError", + "type": "formula", + "as": "hasError", + "expr": "datum._source.metrics ? datum._source.hasError : false" + }, + { + "type": "formula", + "as": "timestamp", + "expr": "timeFormat(toDate(datum._source.buildTimestamp), '%Y-%m-%d %H:%M')" + }, + { + "comment": "create `url` value that points to TC build", + "type": "formula", + "as": "url", + "expr": "'https://buildserver.labs.intellij.net/buildConfiguration/Kotlin_Benchmarks_PluginPerformanceTests_IdeaPluginPerformanceTests/' + datum._source.buildId" + } + ] + }, + { + "name": "selected", + "on": [ + {"trigger": "clear", "remove": true}, + {"trigger": "!shift", "remove": true}, + {"trigger": "!shift && clicked", "insert": "clicked"}, + {"trigger": "shift && clicked", "toggle": "clicked"} + ] + } + ], + "axes": [ + { + "scale": "x", + "grid": true, + "domain": false, + "orient": "bottom", + "labelAngle": -20, + "labelAlign": "right", + "title": {"signal": "timestamp ? 'timestamp' : 'buildId'"}, + "titlePadding": 10, + "tickCount": 5, + "encode": { + "labels": { + "interactive": true, + "update": {"tooltip": {"signal": "datum.label"}} + } + } + }, + { + "scale": "y", + "grid": true, + "domain": false, + "orient": "left", + "titlePadding": 10, + "title": "ms", + "titleAnchor": "end", + "titleAngle": 0 + } + ], + "scales": [ + { + "name": "x", + "type": "point", + "range": "width", + "domain": {"data": "table", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}} + }, + { + "name": "y", + "type": "linear", + "range": "height", + "nice": true, + "zero": true, + "domain": {"data": "table", "field": "metricValue"} + }, + { + "name": "color", + "type": "ordinal", + "range": "category", + "domain": {"data": "table", "field": "metricName"} + }, + { + "name": "size", + "type": "linear", + "round": true, + "nice": false, + "zero": true, + "domain": {"data": "table", "field": "metricError"}, + "range": [1, 100] + } + ], + "legends": [ + { + "title": "Cases", + "stroke": "color", + "strokeColor": "#ccc", + "padding": 8, + "cornerRadius": 4, + "symbolLimit": 50, + "encode": { + "symbols": { + "name": "legendSymbol", + "interactive": true, + "update": { + "fill": {"value": "transparent"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 0.7 + }, + {"value": 0.15} + ], + "size": {"value": 64} + } + }, + "labels": { + "name": "legendLabel", + "interactive": true, + "update": { + "opacity": [ + { + "comment": "here `datum` is `selected` data set", + "test": "!length(data('selected')) || indata('selected', 'value', datum.value)", + "value": 1 + }, + {"value": 0.25} + ] + } + } + } + } + ], + "marks": [ + { + "type": "group", + "from": { + "facet": {"name": "series", "data": "table", "groupby": "metricName"} + }, + "marks": [ + { + "type": "line", + "from": {"data": "series"}, + "encode": { + "hover": {"opacity": {"value": 1}, "strokeWidth": {"value": 4}}, + "update": { + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 2}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 0.7 + }, + {"value": 0.15} + ], + "stroke": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "scale": "color", + "field": "metricName" + }, + {"value": "#ccc"} + ] + } + } + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "fill": {"value": "#B00"}, + "size": [{ "test": "datum.hasError", "value": 250 }, {"value": 0}], + "shape": {"value": "cross"}, + "angle": {"value": 45}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && datum.hasError && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 1 + }, + { + "type": "symbol", + "from": {"data": "series"}, + "encode": { + "enter": { + "tooltip": { + "signal": "datum.metricName + ': ' + datum.metricValue + ' ms'" + }, + "href": {"field": "url"}, + "cursor": {"value": "pointer"}, + "size": {"scale": "size", "field": "metricError"}, + "x": {"scale": "x", "field": {"signal": "timestamp ? 'timestamp' : 'buildId'"}}, + "y": {"scale": "y", "field": "metricValue"}, + "strokeWidth": {"value": 1}, + "fill": {"scale": "color", "field": "metricName"} + }, + "update": { + "opacity": [ + { + "test": "(!domain || inrange(datum.buildId, domain)) && (!length(data('selected')) || indata('selected', 'value', datum.metricName))", + "value": 1 + }, + {"value": 0.15} + ] + } + }, + "zindex": 2 + } + ] + } + ] +} diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt index 24e4d2aeed2..6a0c52c4070 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt @@ -52,8 +52,7 @@ abstract class AbstractPerformanceCompletionHandlerTests( override fun tearDown() { commitAllDocuments() RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { statsMap.values.forEach(Stats::flush) } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt index ea2d39b6c49..bc3a3694516 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt @@ -51,8 +51,7 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight override fun tearDown() { commitAllDocuments() RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats.flush() } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt index b884798ea76..11c297ccb0b 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt @@ -42,8 +42,7 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu override fun tearDown() { commitAllDocuments() RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats.flush() } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt index 298b5913e8e..02872ca0fa7 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt @@ -34,8 +34,7 @@ abstract class AbstractPerformanceImportTest : KotlinLightCodeInsightFixtureTest override fun tearDown() { RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats().flush() } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt index b1842bceb58..ec74c425b6e 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt @@ -46,8 +46,7 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va override fun tearDown() { RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats.forEach(Stats::flush) } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt index a3ab7d07df4..7959f0b9806 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt @@ -44,8 +44,7 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC override fun tearDown() { RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats.flush() } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt index 558f1e28421..b32d4116e92 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt @@ -29,8 +29,7 @@ abstract class AbstractPerformanceTypingIndentationTest : KotlinLightCodeInsight override fun tearDown() { RunAll( - ThrowableRunnable { super.tearDown() }, - ThrowableRunnable { stats.flush() } + ThrowableRunnable { super.tearDown() } ).run() } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt index 60039ae27c2..be9d0fe8653 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -158,6 +158,8 @@ class Stats( } else { block() } + + flush() } private fun convertStatInfoIntoMetrics( @@ -346,7 +348,7 @@ class Stats( flush() } - fun flush() { + private fun flush() { val simpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ") simpleDateFormat.timeZone = TimeZone.getTimeZone("UTC") // properties["buildTimestamp"] = simpleDateFormat.format(Date()) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt index fefce6700e6..9ad67448fe7 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt @@ -32,6 +32,8 @@ fun main(args: Array) { index = it.index, hasError = it.hasError, buildTimestamp = benchmark.buildTimestamp, + metricValue = it.metricValue, + metricError = it.metricError, metrics = it.metrics ?: emptyList() ) warmUpBenchmark.writeJson()