Add test for consistency check between the Kotlin specification and spec tests

compiler/tests-spec/build.gradle.kts

@@ -9,6 +9,7 @@ dependencies {
         testCompileOnly(intellijCoreDep()) { includeJars("intellij-core") }
         testRuntimeOnly(intellijPluginDep("java"))
     }
+    compile("org.jsoup:jsoup:1.10.3")
 }
 
 sourceSets {
@@ -45,3 +46,11 @@ val remoteRunTests by task<Test> {
         includeTests.forEach { includeTestsMatching(packagePrefix + it) }
     }
 }
+
+val specConsistencyTests by task<Test> {
+    workingDir = rootDir
+
+    filter {
+        includeTestsMatching("org.jetbrains.kotlin.spec.consistency.SpecTestsConsistencyTest")
+    }
+}

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/consistency/SpecTestsConsistencyTest.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.spec.consistency
+
+import com.intellij.testFramework.TestDataPath
+import junit.framework.TestCase
+import org.jetbrains.kotlin.spec.utils.GeneralConfiguration
+import org.jetbrains.kotlin.spec.utils.SpecTestLinkedType
+import org.jetbrains.kotlin.spec.utils.TestArea
+import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.parseLinkedSpecTest
+import org.jetbrains.kotlin.spec.utils.spec.SpecSentencesStorage
+import org.junit.Test
+import org.junit.runner.RunWith
+import java.io.File
+import kotlin.io.walkTopDown
+
+@TestDataPath("\$PROJECT_ROOT/compiler/tests-spec/testData/")
+@RunWith(com.intellij.testFramework.Parameterized::class)
+class SpecTestsConsistencyTest : TestCase() {
+    @org.junit.runners.Parameterized.Parameter
+    lateinit var testFilePath: String
+
+    companion object {
+        private val specSentencesStorage = SpecSentencesStorage()
+
+        private const val CURRENT_VERSION = "0.1-99"
+
+        @org.junit.runners.Parameterized.Parameters(name = "{0}")
+        @JvmStatic
+        fun getTestFiles() = emptyList<Array<Any>>()
+
+        @com.intellij.testFramework.Parameterized.Parameters(name = "{0}")
+        @JvmStatic
+        fun getTestFiles(klass: Class<*>): List<Array<String>> {
+            val testFiles = mutableListOf<Array<String>>()
+
+            TestArea.values().forEach { testArea ->
+                val testDataPath =
+                    "${GeneralConfiguration.TESTDATA_PATH}/${testArea.testDataPath}/${SpecTestLinkedType.LINKED.testDataPath}"
+
+                testFiles += File(testDataPath).let { testsDir ->
+                    testsDir.walkTopDown().filter { it.extension == "kt" }.map {
+                        arrayOf(it.relativeTo(File(GeneralConfiguration.TESTDATA_PATH)).path.replace("/", "$"))
+                    }.toList()
+                }
+            }
+
+            return testFiles
+        }
+    }
+
+    @Test
+    fun doTest() {
+        val file = File("${GeneralConfiguration.TESTDATA_PATH}/${testFilePath.replace("$", "/")}")
+        val specSentences = specSentencesStorage.getLatest() ?: return
+        val test = parseLinkedSpecTest(file.canonicalPath, mapOf("main" to file.readText()))
+        val sectionsPath = setOf(*test.place.sections.toTypedArray(), test.place.paragraphNumber).joinToString()
+        val sentenceNumber = test.place.sentenceNumber
+        val paragraphSentences = specSentences[sectionsPath]
+
+        if (paragraphSentences != null && paragraphSentences.size >= sentenceNumber) {
+            val specSentencesForCurrentTest =
+                specSentencesStorage[test.specVersion] ?: throw Exception("spec ${test.specVersion} not found")
+            val paragraphForTestSentences =
+                specSentencesForCurrentTest[sectionsPath] ?: throw Exception("$sectionsPath not found")
+            if (paragraphForTestSentences.size < sentenceNumber) {
+                throw Exception("$sentenceNumber not found")
+            }
+            val expectedSentence = paragraphForTestSentences[sentenceNumber - 1]
+            val actualSentence = paragraphSentences[sentenceNumber - 1]
+            val locationSentenceText = "$sectionsPath paragraph, $sentenceNumber sentence"
+
+            println("Comparing versions: ${test.specVersion} (for expected) and ${specSentencesStorage.latestSpecVersion} (for actual)")
+            println("Expected location: $locationSentenceText")
+
+            assertEquals(expectedSentence, actualSentence)
+        }
+    }
+}

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/utils/spec/HtmlSpecLoader.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.spec.utils.spec
+
+import groovy.util.Node
+import groovy.util.XmlParser
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import java.io.BufferedInputStream
+import java.net.URL
+import java.nio.charset.Charset
+import java.util.regex.Pattern
+
+object HtmlSpecLoader {
+    private const val SPEC_DOCS_TC_CONFIGURATION_ID = "Kotlin_Spec_DocsMaster"
+    private const val TC_URL = "https://teamcity.jetbrains.com"
+    private const val TC_PATH_PREFIX = "guestAuth/app/rest/builds"
+    private const val HTML_SPEC_PATH = "/html/kotlin-spec.html"
+    private const val STABLE_BRANCH = "master"
+
+    private fun loadRawHtmlSpec(specVersion: String, buildNumber: String): String {
+        val htmlSpecLink =
+            "$TC_URL/$TC_PATH_PREFIX/buildType:(id:$SPEC_DOCS_TC_CONFIGURATION_ID),number:$buildNumber,branch:default:any/artifacts/content/kotlin-spec-$specVersion-$buildNumber.zip%21$HTML_SPEC_PATH"
+
+        return BufferedInputStream(URL(htmlSpecLink).openStream()).readBytes().toString(Charset.forName("UTF-8"))
+    }
+
+    private fun getLastSpecVersion(): Pair<String, String> {
+        val xmlParser = XmlParser()
+        val buildInfo =
+            xmlParser.parse("$TC_URL/$TC_PATH_PREFIX/buildType:(id:$SPEC_DOCS_TC_CONFIGURATION_ID),count:1,status:SUCCESS?branch=$STABLE_BRANCH")
+        val artifactsLink = (buildInfo.children().find { (it as Node).name() == "artifacts" } as Node).attribute("href").toString()
+        val artifacts = xmlParser.parse(TC_URL + artifactsLink)
+        val artifactName = (artifacts.children().single() as Node).attribute("name").toString()
+        val artifactNameMatches =
+            Pattern.compile("""kotlin-spec-(?<specVersion>\d+\.\d+)-(?<buildNumber>[1-9]\d*)\.zip""").matcher(artifactName).apply { find() }
+
+        return Pair(artifactNameMatches.group("specVersion"), artifactNameMatches.group("buildNumber"))
+    }
+
+    private fun parseHtmlSpec(htmlSpecContent: String) = Jsoup.parse(htmlSpecContent).body()
+
+    fun loadSpec(version: String): Element? {
+        val specVersion = version.substringBefore("-")
+        val buildNumber = version.substringAfter("-")
+
+        return parseHtmlSpec(loadRawHtmlSpec(specVersion, buildNumber))
+    }
+
+    fun loadLatestSpec(): Pair<String, Element?> {
+        val (specVersion, buildNumber) = getLastSpecVersion()
+
+        return Pair("$specVersion-$buildNumber", parseHtmlSpec(loadRawHtmlSpec(specVersion, buildNumber)))
+    }
+}

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/utils/spec/HtmlSpecSentencesMapBuilder.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.spec.utils.spec
+
+import org.jsoup.nodes.Element
+import java.util.*
+
+typealias SentencesByLocation = MutableMap<String, MutableList<String>>
+
+object HtmlSpecSentencesMapBuilder {
+    enum class SectionTag(val level: Int) { h1(1), h2(2), h3(3), h4(4), h5(5) }
+
+    private const val PARAGRAPH_SELECTORS = ".paragraph, ul, ol"
+    private const val SECTION_SELECTORS = "h2, h3, h4, h5"
+
+    fun build(spec: Element): SentencesByLocation {
+        var paragraphCounter = 0
+        val currentSectionsPath = Stack<Pair<SectionTag, String>>()
+        val sentencesByLocation: SentencesByLocation = mutableMapOf()
+
+        spec.select("$SECTION_SELECTORS, $PARAGRAPH_SELECTORS, .sentence").forEach { element ->
+            when {
+                element.`is`(SECTION_SELECTORS) -> {
+                    val sectionTag = SectionTag.valueOf(element.tagName().toLowerCase())
+                    while (!currentSectionsPath.empty() && currentSectionsPath.peek().first.level >= sectionTag.level) {
+                        currentSectionsPath.pop()
+                    }
+
+                    currentSectionsPath.push(Pair(sectionTag, element.attr("id")))
+                    paragraphCounter = 0
+                }
+                element.`is`(PARAGRAPH_SELECTORS) -> paragraphCounter++
+                else -> {
+                    if (!element.parents().`is`(PARAGRAPH_SELECTORS)) return@forEach
+                    val sentenceLocation =
+                        currentSectionsPath.map { it.second }.toMutableSet().apply { add(paragraphCounter.toString()) }.joinToString()
+                    sentencesByLocation.putIfAbsent(sentenceLocation, mutableListOf())
+                    sentencesByLocation[sentenceLocation]!!.add(element.text())
+                }
+            }
+        }
+
+        return sentencesByLocation
+    }
+}

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/utils/spec/SpecSentencesStorage.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.spec.utils.spec
+
+class SpecSentencesStorage {
+    lateinit var latestSpecVersion: String
+    private val specSentences = mutableMapOf<String, SentencesByLocation>()
+
+    operator fun get(version: String): SentencesByLocation? {
+        return specSentences.getOrPut(version) {
+            val htmlSpec = HtmlSpecLoader.loadSpec(version) ?: return null
+            HtmlSpecSentencesMapBuilder.build(htmlSpec)
+        }
+    }
+
+    fun getLatest(): SentencesByLocation? {
+        return specSentences.getOrPut("latest") {
+            val (version, htmlSpec) = HtmlSpecLoader.loadLatestSpec()
+            if (htmlSpec == null) return null
+            latestSpecVersion = version
+            HtmlSpecSentencesMapBuilder.build(htmlSpec)
+        }
+    }
+}